You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by ch...@apache.org on 2017/09/27 22:10:36 UTC
[01/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Repository: incubator-predictionio
Updated Branches:
refs/heads/master e34a853d0 -> 018ea8e34
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/42d6d428
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/42d6d428
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/42d6d428
Branch: refs/heads/master
Commit: 42d6d428efef2f23270c77615ec4634cf191ce79
Parents: fdcd921 5aea927
Author: Naoki Takezoe <ta...@apache.org>
Authored: Sun Jul 16 02:02:07 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Sun Jul 16 02:02:07 2017 +0900
----------------------------------------------------------------------
.../templates/classification/add-algorithm.html.md | 2 +-
.../templates/classification/dase.html.md.erb | 16 ++++++++--------
.../recommendation/blacklist-items.html.md | 6 +++---
.../templates/recommendation/dase.html.md.erb | 10 +++++-----
.../templates/recommendation/evaluation.html.md.erb | 2 +-
5 files changed, 18 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
[23/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by ch...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/f7526901
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/f7526901
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/f7526901
Branch: refs/heads/master
Commit: f7526901ef1585d73a9b358c61850310c8b01f2a
Parents: cfa3f5d 55ce7b8
Author: Naoki Takezoe <ta...@apache.org>
Authored: Thu Aug 3 15:05:29 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Thu Aug 3 15:05:29 2017 +0900
----------------------------------------------------------------------
.../source/images/logos/apache_incubator.png | Bin 0 -> 14557 bytes
docs/manual/source/partials/_footer.html.slim | 9 +++++++--
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[38/50] [abbrv] incubator-predictionio git commit: [PIO-116] PySpark
Support
Posted by ch...@apache.org.
[PIO-116] PySpark Support
Closes #427
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/df406bf9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/df406bf9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/df406bf9
Branch: refs/heads/master
Commit: df406bf92463da4a79c8d84ec0ca439feaa0ec7f
Parents: 69c5e3b
Author: Shinsuke Sugaya <sh...@apache.org>
Authored: Mon Sep 11 12:05:18 2017 +0900
Committer: Shinsuke Sugaya <sh...@apache.org>
Committed: Mon Sep 11 12:05:18 2017 +0900
----------------------------------------------------------------------
bin/pio-shell | 9 ++
.../data/store/python/PPythonEventStore.scala | 146 +++++++++++++++++++
make-distribution.sh | 2 +
python/pypio/__init__.py | 20 +++
python/pypio/data/__init__.py | 25 ++++
python/pypio/data/eventstore.py | 50 +++++++
python/pypio/shell.py | 21 +++
python/pypio/utils.py | 27 ++++
.../apache/predictionio/tools/RunWorkflow.scala | 7 +-
.../org/apache/predictionio/tools/Runner.scala | 47 ++++--
.../predictionio/tools/console/Console.scala | 3 +
11 files changed, 346 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/bin/pio-shell
----------------------------------------------------------------------
diff --git a/bin/pio-shell b/bin/pio-shell
index 5a5745a..cd119cd 100755
--- a/bin/pio-shell
+++ b/bin/pio-shell
@@ -59,6 +59,15 @@ then
. ${PIO_HOME}/bin/compute-classpath.sh
shift
${SPARK_HOME}/bin/spark-shell --jars ${ASSEMBLY_JARS} $@
+elif [[ "$1" == "--with-pyspark" ]]
+then
+ echo "Starting the PIO shell with the Apache Spark Shell."
+ # Get paths of assembly jars to pass to pyspark
+ . ${PIO_HOME}/bin/compute-classpath.sh
+ shift
+ export PYTHONSTARTUP=${PIO_HOME}/python/pypio/shell.py
+ export PYTHONPATH=${PIO_HOME}/python
+ ${SPARK_HOME}/bin/pyspark --jars ${ASSEMBLY_JARS} $@
else
echo -e "\033[0;33mStarting the PIO shell without Apache Spark.\033[0m"
echo -e "\033[0;33mIf you need the Apache Spark library, run 'pio-shell --with-spark [spark-submit arguments...]'.\033[0m"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/data/src/main/spark-2/org/apache/predictionio/data/store/python/PPythonEventStore.scala
----------------------------------------------------------------------
diff --git a/data/src/main/spark-2/org/apache/predictionio/data/store/python/PPythonEventStore.scala b/data/src/main/spark-2/org/apache/predictionio/data/store/python/PPythonEventStore.scala
new file mode 100644
index 0000000..1d03634
--- /dev/null
+++ b/data/src/main/spark-2/org/apache/predictionio/data/store/python/PPythonEventStore.scala
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.predictionio.data.store.python
+
+import java.sql.Timestamp
+
+import org.apache.predictionio.data.store.PEventStore
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.joda.time.DateTime
+
+
+/** This object provides a set of operation to access Event Store
+ * with Spark's parallelization
+ */
+object PPythonEventStore {
+
+
+ /** Read events from Event Store
+ *
+ * @param appName return events of this app
+ * @param channelName return events of this channel (default channel if it's None)
+ * @param startTime return events with eventTime >= startTime
+ * @param untilTime return events with eventTime < untilTime
+ * @param entityType return events of this entityType
+ * @param entityId return events of this entityId
+ * @param eventNames return events with any of these event names.
+ * @param targetEntityType return events of this targetEntityType:
+ * - None means no restriction on targetEntityType
+ * - Some(None) means no targetEntityType for this event
+ * - Some(Some(x)) means targetEntityType should match x.
+ * @param targetEntityId return events of this targetEntityId
+ * - None means no restriction on targetEntityId
+ * - Some(None) means no targetEntityId for this event
+ * - Some(Some(x)) means targetEntityId should match x.
+ * @param spark Spark context
+ * @return DataFrame
+ */
+ def find(
+ appName: String,
+ channelName: String,
+ startTime: Timestamp,
+ untilTime: Timestamp,
+ entityType: String,
+ entityId: String,
+ eventNames: Array[String],
+ targetEntityType: String,
+ targetEntityId: String
+ )(spark: SparkSession): DataFrame = {
+ import spark.implicits._
+ val colNames: Seq[String] =
+ Seq(
+ "eventId",
+ "event",
+ "entityType",
+ "entityId",
+ "targetEntityType",
+ "targetEntityId",
+ "eventTime",
+ "tags",
+ "prId",
+ "creationTime",
+ "fields"
+ )
+ PEventStore.find(appName,
+ Option(channelName),
+ Option(startTime).map(t => new DateTime(t.getTime)),
+ Option(untilTime).map(t => new DateTime(t.getTime)),
+ Option(entityType),
+ Option(entityId),
+ Option(eventNames),
+ Option(Option(targetEntityType)),
+ Option(Option(targetEntityId)))(spark.sparkContext).map { e =>
+ (
+ e.eventId,
+ e.event,
+ e.entityType,
+ e.entityId,
+ e.targetEntityType.orNull,
+ e.targetEntityId.orNull,
+ new Timestamp(e.eventTime.getMillis),
+ e.tags.mkString("\t"),
+ e.prId.orNull,
+ new Timestamp(e.creationTime.getMillis),
+ e.properties.fields.mapValues(_.values.toString)
+ )
+ }.toDF(colNames: _*)
+ }
+
+ /** Aggregate properties of entities based on these special events:
+ * \$set, \$unset, \$delete events.
+ *
+ * @param appName use events of this app
+ * @param entityType aggregate properties of the entities of this entityType
+ * @param channelName use events of this channel (default channel if it's None)
+ * @param startTime use events with eventTime >= startTime
+ * @param untilTime use events with eventTime < untilTime
+ * @param required only keep entities with these required properties defined
+ * @param spark Spark session
+ * @return DataFrame DataFrame of entityId and PropetyMap pair
+ */
+ def aggregateProperties(
+ appName: String,
+ entityType: String,
+ channelName: String,
+ startTime: Timestamp,
+ untilTime: Timestamp,
+ required: Array[String]
+ )
+ (spark: SparkSession): DataFrame = {
+ import spark.implicits._
+ val colNames: Seq[String] =
+ Seq(
+ "entityId",
+ "firstUpdated",
+ "lastUpdated",
+ "fields"
+ )
+ PEventStore.aggregateProperties(appName,
+ entityType,
+ Option(channelName),
+ Option(startTime).map(t => new DateTime(t.getTime)),
+ Option(untilTime).map(t => new DateTime(t.getTime)),
+ Option(required.toSeq))(spark.sparkContext).map { x =>
+ val m = x._2
+ (x._1,
+ new Timestamp(m.firstUpdated.getMillis),
+ new Timestamp(m.lastUpdated.getMillis),
+ m.fields.mapValues(_.values.toString)
+ )
+ }.toDF(colNames: _*)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index bf4c7ce..7a34274 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -83,6 +83,7 @@ cd ${FWDIR}
rm -rf ${DISTDIR}
mkdir -p ${DISTDIR}/bin
mkdir -p ${DISTDIR}/conf
+mkdir -p ${DISTDIR}/python
mkdir -p ${DISTDIR}/lib
mkdir -p ${DISTDIR}/lib/spark
mkdir -p ${DISTDIR}/project
@@ -91,6 +92,7 @@ mkdir -p ${DISTDIR}/sbt
cp ${FWDIR}/bin/* ${DISTDIR}/bin || :
cp ${FWDIR}/conf/* ${DISTDIR}/conf
+cp -r ${FWDIR}/python/* ${DISTDIR}/python
cp ${FWDIR}/project/build.properties ${DISTDIR}/project
cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
cp ${FWDIR}/assembly/src/universal/lib/*assembly*jar ${DISTDIR}/lib
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/python/pypio/__init__.py
----------------------------------------------------------------------
diff --git a/python/pypio/__init__.py b/python/pypio/__init__.py
new file mode 100644
index 0000000..04d8ac3
--- /dev/null
+++ b/python/pypio/__init__.py
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+PyPIO is the Python API for PredictionIO.
+"""
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/python/pypio/data/__init__.py
----------------------------------------------------------------------
diff --git a/python/pypio/data/__init__.py b/python/pypio/data/__init__.py
new file mode 100644
index 0000000..63a6442
--- /dev/null
+++ b/python/pypio/data/__init__.py
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+
+from pypio.data.eventstore import PEventStore
+
+
+__all__ = [
+ 'PEventStore'
+]
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/python/pypio/data/eventstore.py
----------------------------------------------------------------------
diff --git a/python/pypio/data/eventstore.py b/python/pypio/data/eventstore.py
new file mode 100644
index 0000000..4eb73df
--- /dev/null
+++ b/python/pypio/data/eventstore.py
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+
+from pypio.utils import new_string_array
+from pyspark.sql.dataframe import DataFrame
+
+__all__ = ["PEventStore"]
+
+
+class PEventStore(object):
+
+ def __init__(self, jss, sql_ctx):
+ self._jss = jss
+ self.sql_ctx = sql_ctx
+ self._sc = sql_ctx and sql_ctx._sc
+
+ def find(self, app_name, channel_name=None, start_time=None, until_time=None,
+ entity_type=None, entity_id=None, event_names=None, target_entity_type=None,
+ target_entity_id=None):
+ pes = self._sc._jvm.org.apache.predictionio.data.store.python.PPythonEventStore
+ jdf = pes.find(app_name, channel_name, start_time, until_time, entity_type, entity_id,
+ event_names, target_entity_type, target_entity_id, self._jss)
+ return DataFrame(jdf, self.sql_ctx)
+
+ def aggregate_properties(self, app_name, entity_type, channel_name=None,
+ start_time=None, until_time=None, required=None):
+ pes = self._sc._jvm.org.apache.predictionio.data.store.python.PPythonEventStore
+ jdf = pes.aggregateProperties(app_name, entity_type, channel_name,
+ start_time, until_time,
+ new_string_array(required, self._sc._gateway),
+ self._jss)
+ return DataFrame(jdf, self.sql_ctx)
+
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/python/pypio/shell.py
----------------------------------------------------------------------
diff --git a/python/pypio/shell.py b/python/pypio/shell.py
new file mode 100644
index 0000000..94c1e1a
--- /dev/null
+++ b/python/pypio/shell.py
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pypio.data import PEventStore
+
+p_event_store = PEventStore(spark._jsparkSession, sqlContext)
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/python/pypio/utils.py
----------------------------------------------------------------------
diff --git a/python/pypio/utils.py b/python/pypio/utils.py
new file mode 100644
index 0000000..76900c3
--- /dev/null
+++ b/python/pypio/utils.py
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+def new_string_array(list_data, gateway):
+ if list_data is None:
+ return None
+ string_class = gateway.jvm.String
+ args = gateway.new_array(string_class, len(list_data))
+ for i in range(len(list_data)):
+ args[i] = list_data[i]
+ return args
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
index a25f4e0..236d3ba 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
@@ -39,6 +39,7 @@ case class WorkflowArgs(
stopAfterRead: Boolean = false,
stopAfterPrepare: Boolean = false,
skipSanityCheck: Boolean = false,
+ mainPyFile: Option[String] = None,
jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
object RunWorkflow extends Logging {
@@ -85,8 +86,12 @@ object RunWorkflow extends Logging {
(if (wa.batch != "") Seq("--batch", wa.batch) else Nil) ++
Seq("--json-extractor", wa.jsonExtractor.toString)
+ val resourceName = wa.mainPyFile match {
+ case Some(x) => x
+ case _ => "org.apache.predictionio.workflow.CreateWorkflow"
+ }
Runner.runOnSpark(
- "org.apache.predictionio.workflow.CreateWorkflow",
+ resourceName,
args,
sa,
jarFiles,
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index 4e266c8..8e08b21 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -183,7 +183,7 @@ object Runner extends EitherLogging {
}
def runOnSpark(
- className: String,
+ resourceName: String,
classArgs: Seq[String],
sa: SparkArgs,
extraJars: Seq[URI],
@@ -194,6 +194,10 @@ object Runner extends EitherLogging {
argumentValue(sa.sparkPassThrough, "--deploy-mode").getOrElse("client")
val master =
argumentValue(sa.sparkPassThrough, "--master").getOrElse("local")
+ val isPython = resourceName match {
+ case x if x.endsWith(".py") => true
+ case _ => false
+ }
(sa.scratchUri, deployMode, master) match {
case (Some(u), "client", m) if m != "yarn-cluster" =>
@@ -219,10 +223,15 @@ object Runner extends EitherLogging {
sys.env.getOrElse("SPARK_HOME", "."))
// Local path to PredictionIO assembly JAR
- val mainJar = Common.coreAssembly(pioHome) fold(
- errStr => return Left(errStr),
- assembly => handleScratchFile(fs, sa.scratchUri, assembly)
- )
+ val assemblyJar = Common.coreAssembly(pioHome) fold(
+ errStr => return Left(errStr),
+ assembly => handleScratchFile(fs, sa.scratchUri, assembly)
+ )
+ val mainJar = if(isPython) {
+ resourceName
+ } else {
+ assemblyJar
+ }
// Extra JARs that are needed by the driver
val driverClassPathPrefix =
@@ -247,8 +256,13 @@ object Runner extends EitherLogging {
val sparkSubmitCommand =
Seq(Seq(sparkHome, "bin", "spark-submit").mkString(File.separator))
- val sparkSubmitJarsList = WorkflowUtils.thirdPartyJars ++ deployedJars ++
- Common.jarFilesForSpark(pioHome).map(_.toURI)
+ val sparkSubmitJarsList = if(isPython) {
+ WorkflowUtils.thirdPartyJars ++ deployedJars ++
+ Common.jarFilesForSpark(pioHome).map(_.toURI) ++ Seq(new URI(assemblyJar))
+ } else {
+ WorkflowUtils.thirdPartyJars ++ deployedJars ++
+ Common.jarFilesForSpark(pioHome).map(_.toURI)
+ }
val sparkSubmitJars = if (sparkSubmitJarsList.nonEmpty) {
Seq("--jars", sparkSubmitJarsList.map(_.toString).mkString(","))
} else {
@@ -275,12 +289,18 @@ object Runner extends EitherLogging {
Nil
}
+ val className = if(isPython) {
+ Nil
+ } else {
+ Seq("--class", resourceName)
+ }
+
val verboseArg = if (verbose) Seq("--verbose") else Nil
val pioLogDir = Option(System.getProperty("pio.log.dir")).getOrElse(s"$pioHome/log")
val sparkSubmitArgs = Seq(
sa.sparkPassThrough,
- Seq("--class", className),
+ className,
sparkSubmitJars,
sparkSubmitFiles,
sparkSubmitExtraClasspaths,
@@ -298,11 +318,18 @@ object Runner extends EitherLogging {
Seq("--env", pioEnvVars),
verboseArg).flatten.filter(_ != "")
info(s"Submission command: ${sparkSubmit.mkString(" ")}")
+ val extraEnv: Seq[(String, String)] = if(isPython) {
+ Seq("CLASSPATH" -> "",
+ "SPARK_YARN_USER_ENV" -> pioEnvVars,
+ "PYTHONPATH" -> s"$pioHome/python")
+ } else {
+ Seq("CLASSPATH" -> "",
+ "SPARK_YARN_USER_ENV" -> pioEnvVars)
+ }
val proc = Process(
sparkSubmit,
None,
- "CLASSPATH" -> "",
- "SPARK_YARN_USER_ENV" -> pioEnvVars).run()
+ extraEnv:_*).run()
Right((proc, () => cleanup(fs, sa.scratchUri)))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/df406bf9/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
index acd7598..04df82f 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
@@ -214,6 +214,9 @@ object Console extends Logging {
opt[String]("engine-params-key") action { (x, c) =>
c.copy(workflow = c.workflow.copy(engineParamsKey = Some(x)))
},
+ opt[String]("main-py-file") action { (x, c) =>
+ c.copy(workflow = c.workflow.copy(mainPyFile = Some(x)))
+ },
opt[String]("json-extractor") action { (x, c) =>
c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
} validate { x =>
[15/50] [abbrv] incubator-predictionio git commit: Update license
checker configuration
Posted by ch...@apache.org.
Update license checker configuration
Closes #415
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/9bbd1f51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/9bbd1f51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/9bbd1f51
Branch: refs/heads/master
Commit: 9bbd1f51a7da31a3dbc00d36942ecfe4315af169
Parents: dec9f84
Author: Naoki Takezoe <ta...@apache.org>
Authored: Wed Jul 26 18:10:11 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Wed Jul 26 18:10:11 2017 +0900
----------------------------------------------------------------------
build.sbt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/9bbd1f51/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 712d360..139bf39 100644
--- a/build.sbt
+++ b/build.sbt
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import com.typesafe.sbt.license.{LicenseInfo, DepModuleInfo}
import PIOBuild._
lazy val scalaSparkDepsVersion = Map(
@@ -99,6 +98,7 @@ val conf = file("conf")
val commonSettings = Seq(
autoAPIMappings := true,
licenseConfigurations := Set("compile"),
+ licenseReportTypes := Seq(Csv),
unmanagedClasspath in Test += conf,
unmanagedClasspath in Test += baseDirectory.value.getParentFile / s"storage/jdbc/target/scala-${scalaBinaryVersion.value}/classes")
[05/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/README.md b/examples/experimental/scala-stock/README.md
deleted file mode 100644
index d780820..0000000
--- a/examples/experimental/scala-stock/README.md
+++ /dev/null
@@ -1,388 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Using predictionIO to run predictions using Yahoo Finance
-
-This tutorial assumes you have gone through the quickstart guide for
-PredictionIO.
-
-## Setting up the environment
-
-### Step 1: Get your Pandas
-Where: PredictionIO-Python-SDK
-
-Run: sudo pip install pandas==0.13.1
-
-pip command not found? install python from curl -O
-
-https://bootstrap.pypa.io/get-pip.py
-
-python get-pip.py
-
-and then run sudo pip install pandas
-
-### Step 2: Edit import_yahoo.py
-Where: PredictionIO-Python-SDK/examples/import_yahoo.py
-
-At the end of file, find the following:
-```
-if __name__ == '__main__':
- #import_all(app_id=?)
- import_data_with_gaps(app_id=1)
- #import_one(app_id=1)
-```
-And, uncomment the first import, replacing app_id with your own id. Next, comment the second import statement (import_data_with_gaps).
-
-### Step 3: Import Yahoo Finance data.
-Where: PredictionIO-Python-SDK/examples
-
-Run: sudo python -m examples.import_yahoo
-
-
-### Step 4: Now make the distribution of PredictionIO
-Where: cloned PredictionIO directory (with source code, make sure code is updated, git pull)
-```
-./make-distribution.sh
-```
-### Step 5: Edit scala-stock
-go to examples/scala-stock/src/main/scala
-
-Edit YahooDataSource.scala
-
-Go to end of file to PredefinedDSP object
-
-Edit app_id to match the one from step 2
-
-### Step 6: Run scala-stock
-Go to PredictionIO/examples/scala-stock
-
-Now type:
-```
-../../bin/pio run --asm org.apache.predictionio.examples.stock.YahooDataSourceRun -- --master <Your spark master address found at http:local8080> --driver-memory <4-12G>
-```
-### Step 7: Open dashboard and view results
-In PredictionIO folder
-
-Type /bin/pio dashboard
-
-go to url: http://localhost:9000 to view output
-
-
----
-#PredictionIO Scala Stock Tutorial
-
-##Implementing New Indicators
-
-If you don't need to introduce any new indicators, skip this step. To introduce a new indicator, go to Indicators.scala and implement the `BaseIndicator` class. 'ShiftsIndicator' can serve as an example for how to do this:
-
-```
-abstract class BaseIndicator extends Serializable {
- def getTraining(logPrice: Series[DateTime, Double]): Series[DateTime, Double]
- def getOne(input: Series[DateTime, Double]): Double
- def minWindowSize(): Int
-}
-```
-####`getTraining`
-######Parameters:
-This function takes in a Series of `logPrices` created from the closing prices of the YahooFinanace Data imported in YahooFinance.scala. `logPrices` refers to the logarithm of each price in the series.
-######Functionality:
-Performs a transformation over every value in the series to return a training series that will be used in the regression.
-####`getOne()`
-######Parameters:
-This function takes in a smaller window of the price series representing the closing price of a single stock over `minWindowSize()`. The size of this series is defined by the return value of `minWindowSize()`
-######Functionality:
-This function performs the same function as `getTraining()` over a smaller window defined by the return value of `minWindowSize()`.
-####`minWindowSize()`
-######Functionality:
-Returns the minimum window sized required to do a single calculation of the Indicator function implemented by `getTraining()`.
-For example, an indicator that requires seeing the previous day to make a calculation would have a `minWindowSize` of 2 days.
-
-##Running New Indicators
-To change which indicators to include in your run, open YahooDataSource.scala. This file imports the Yahoo finance data, prepares it, and runs the predictive engine.
-
-Navigate to Workflow.run() in the `YahooDataSourceRun` object:
-```
- Workflow.run(
- dataSourceClassOpt = Some(classOf[YahooDataSource]),
- dataSourceParams = dsp,
- preparatorClassOpt = Some(IdentityPreparator(classOf[DataSource])),
- algorithmClassMapOpt = Some(Map(
- //"" -> classOf[MomentumStrategy]
- "" -> classOf[RegressionStrategy]
- )),
- //algorithmParamsList = Seq(("", momentumParams)),
- algorithmParamsList = Seq(("", RegressionStrategyParams(Seq[(String, BaseIndicator)](
- ("RSI1", new RSIIndicator(period=1)),
- ("RSI5", new RSIIndicator(period=5)),
- ("RSI22", new RSIIndicator(period=22))),
- 200))),
- servingClassOpt = Some(FirstServing(classOf[EmptyStrategy])),
- metricsClassOpt = Some(classOf[BacktestingMetrics]),
- metricsParams = metricsParams,
- params = WorkflowParams(
- verbose = 0,
- saveModel = false,
-```
-#### `algorithmParamsList ()`
-Edit these parameters to include any newly implemented indicator functions the predictive model should include. In this example you would change 'RSIIndicator' to your new indicator. If you would like to modify the strategy used, make sure to also change the constructor parameter, i.e. 'RegressionStrategyParams'.
-
-### Viewing Your Results
-To view the backtesting metrics, open up localhost:9000 in your browser and click on the 'HTML' button for the run you want to see the results of. On this page, the sharpe ratio is indicative of how effective your indicator implementations are in predictions.
-
-### Tips and Tricks
-To reduce run time run your code on a smaller dataset:
-1. Open `YahooDataSource.scala`
-2. Look for line `val dsp = PredefinedDSP.BigSP500` and comment it out
-3. Uncomment out the line: `val dsp = PredefinedDSP.SmallSP500`
-4. In the `PredefinedDSP` object, switch the `appId` for `BigSP500`with the `appId` for `SmallSP500`. *Note: It should match the appID in your runnable
-
-To view standard output:
-1. Open localhost:8080 (spark)
-2. Click on the most recent worker
-3. Click on stdout or stderr to view the data in 100kb increments
-
-
----
-
-
-# OLD DOCUMENTATION
-
-# (This doc is out-of-sync with the actual implementation)
-
-## How to implement a stock prediction algorithm
-
-### Fetch data from external source
-You only need to do it once. (Unless you want to update your data.)
-
-We provide a script which extracts historical data of SP500 stocks from Yahoo
-and store the data in your local data storage. Make sure you have setup storage
-according to [storage setup instructions](/README.md). Feel free to substitute
-with your favorite stock data source.
-
-Specify environment. (Temporary.)
-```
-$ cd $PIO_HOME/examples
-$ set -a
-$ source ../conf/pio-env.sh
-$ set +a
-```
-where `$PIO_HOME` is the root directory of PredictionIO's code tree.
-
-Run the fetch script.
-```
-$ ../sbt/sbt "runMain org.apache.predictionio.examples.stock.FetchMain"
-```
-As SP500 constituents change all the time, the hardcoded list may not reflect
-the current state and the script may fail to extract delisted tickers. Whilst
-the stock engine is designed to accomodate missing / incomplete data, you may as
-well update the ticker list in [stock engine
-settings](/engines/src/main/scala/stock/Stock.scala).
-
-### High Level Description
-A stock prediction algorithms employs a *rolling window* estimation method. For
-a given window with `testingWindowSize`, the algorithm trains a model with data
-in the window, and then the model is evaluated with another set of data.
-
-For example, it builds a model with data from Aug 31, 2012 to Aug 31, 2013, and
-evaluate the model with data from Sept 1, 2013 to Oct 1, 2013. And then, the
-training is rolled one month forward: model training with data from Sept 30,
-2012 to Sept 30, 2013, and evaluation with data from Oct 1, 2013 to Nov 1, 2013,
-and so on.
-
-Training Data | Testing Data
---------------|-------------
-2012-08-31 -> 2013-08-31 | 2013-09-01 -> 2013-10-01
-2012-09-30 -> 2013-09-30 | 2013-10-01 -> 2013-11-01
-2012-10-31 -> 2013-10-31 | 2013-11-01 -> 2013-12-01
-2012-11-30 -> 2013-11-30 | 2013-12-01 -> 2014-01-01
-2012-12-31 -> 2013-12-31 | 2014-01-01 -> 2014-02-01
-... | ...
-
-For an algorithm developer, the task is to create a model with the training
-data, and then make prediction based on testing data.
-
-### Key Data Structures
-
-#### Training Data
-[Training data](Data.scala) has 4 main fields. `tickers` is the list of tickers
-that is used for training. `mktTicker` is the ticker of market, for calculating
-beta related metrics. `timeIndex` is a list of dates representing the time
-window used for training. `price` is a map from ticker to array of stock price,
-the array is of the same length as `timeIndex`. Notice that only tickers that
-are active through out the whole time window is included, i.e. for example, if
-the `TrainingData`'s time window is from 2012-01-01 to 2013-01-01, Facebook (FB)
-will not be included in the training data.
-
-```scala
-class TrainingData(
- val tickers: Seq[String],
- val mktTicker: String,
- val timeIndex: Array[DateTime],
- val price: Array[(String, Array[Double])])
-```
-
-#### Query
-[Query](Data.scala) is the input for prediction. Most of the fields resemble
-`TrainingData`, with an additional field `tomorrow` indicating the date of the
-prediction output. Algorithm builders take this as input, together with the
-trained model created with `TrainingData` to make prediction.
-
-```scala
-class Query(
- val mktTicker: String,
- val tickerList: Seq[String],
- val timeIndex: Array[DateTime],
- val price: Array[(String, Array[Double])],
- val tomorrow: DateTime)
-```
-
-#### Target
-[Target](Data.scala) is the output for prediction. It is essentially a map from
-ticker to predicted return. Notice that the prediction need not to match
-`Query`, if an algorithm cannot make prediction some symbols, it can just leave
-them out.
-
-### The Algorithm
-Stock prediction algorithms should extends [StockAlgorithm](Stock.scala) class.
-
-```scala
-abstract class StockAlgorithm[P <: Params : ClassTag, M : ClassTag]
- extends LAlgorithm[P, TrainingData, M, Query, Target] {
- def train(trainingData: TrainingData): M
- def predict(model: M, query: Query): Target
-}
-```
-
-#### RegressionAlgorithm
-[RegressionAlgrotihm](RegressionAlgorithm.scala) creates a linear model for each
-stock using a vector comprised of the 1-day, 1-week, and 1-month return of the
-stock.
-
-#### RandomAlgorithm
-[RandomAlgorithm](RandomAlgorithm.scala) produces a gaussian random variable
-with scaling and drift.
-
-### Evaluation: Backtesting Metrics
-This is the most common method in quantitative equity research. We test the
-stock prediction algorithm against historical data. For each day in the
-evaluation period, we open or close positions according to the prediction of the
-algorithm. This allows us to simulate the daily P/L, volatility, and drawdown of
-the prediction algorithm.
-
-[BacktestingMetrics](BacktestingMetrics.scala) takes three parameters:
-`enterThreshold` the minimum predicted return to open a new position,
-`exitThreshold` the maximum predicted return to close an existing position, and
-`maxPositions` is the maximum number of open positions. Everyday, this metrics
-adjusts its portfolio based on the stock algorithm's prediction `Target`. For a
-current position, if its predicted return is lower than `exitThreshold`, then
-metrics will close this positions. On the other hand, metrics will look at all
-stocks that have predicted return higher than `enterThreshold`, and will
-repeatedly open new ones with maximum predicted value until it reaches
-`maxPositions`.
-
-##### First Evaluation: Demo1
-[stock.Demo1](Demo1.scala) shows a sample Runner program. To run this
-evaluation, you have to specify two sets of parameters:
-
-1. `DataSourceParams` governs the ticker and the time window you wish to
- evaluate. Here is the parameter we used in stock.Demo1.
- ```scala
- val dataSourceParams = new DataSourceParams(
- baseDate = new DateTime(2004, 1, 1, 0, 0),
- fromIdx = 400,
- untilIdx = 1000,
- trainingWindowSize = 300,
- evaluationInterval = 20,
- marketTicker = "SPY",
- tickerList = Seq("AAPL"))
- ```
- This means we start our evaluation at 400 market days after the first day of
- 2004 until 1200 days. Our `evalutionInterval` is 20 and `trainingWindowSize`
- is 300, meaning that we use day 100 until day 400 as *the first slice* of
- training data, and evaluate with data from day 400 until 420. Then, this
- process repeats, day 120 until 420 for training, and evaluation with data
- from day 420 until 440, until it reaches `untilIdx`. We only specify one
- ticker GOOGL for now, but our stock engine actually supports multiple
- tickers.
-
-2. `BacktestingParams` governs the backtesting evaluation.
- ```scala
- val backtestingParams = BacktestingParams(enterThreshold = 0.001,
- exitThreshold = 0.0)
- ```
- As explained above, the backtesting evaluator opens a new long position when
- the prediction of stock is higher than 0.001, and will exit such position
- when the prediction is lower than 0.0.
-
-You can run the evaluation with the following command.
-```
-$ cd $PIO_HOME/examples
-$ ../bin/pio-run org.apache.predictionio.examples.stock.Demo1
-```
-
-You should see that we are trading from April 2005 until Dec 2007, the NAV went
-from $1,000,000 to $1,433,449.24. YaY!!!
-
-(**Disclaimer**: I cherrypicked this parameter to make the demo looks nice. A
-buy-and-hold strategy of AAPL from April 2005 until 2007 performs even better.
-And, you will lose money if you let it run for longer: set `untilIdx = 1400`.)
-
-##### Second Evaluation
-[stock.Demo2](Demo2.scala) shows us how to run backtesting against a basket of
-stocks. You simply specify a list of tickers in the tickerList. `DataSource` wil
-l handle cases where the ticker was absent.
-
-In `BacktestingParams`, you may allow more stocks to be held concurrently. The
-backtesting class essentially divides the current NAV by the `maxPositions`. The
-demo is run the same way, by specifying the running main class.
-```
-$ cd $PIO_HOME/examples
-$ ../bin/pio-run org.apache.predictionio.examples.stock.Demo2
-```
-
-The result is not as great, of course.
-
-##### Third Evaluation
-Now, you may start wondering what actually contribute to the profit, and may
-want to dive deeper into the data. [DailyMetrics](DailyMetrics.scala) is a
-helper metrics which helps you to understand the performance of different
-parameter settings. It aggregates the prediction results for different
-`enterThreshold`, and output the average / stdev return of the prediction
-algorithm.
-
-All you need is to change the `metrics` variable to `DailyMetrics`.
-[Demo3](Demo3.scala) shows the actual code. Try it out with:
-```
-$ cd $PIO_HOME/examples
-$ ../bin/pio-run org.apache.predictionio.examples.stock.Demo3
-```
-
-### Last Words
-The current version is only a proof-of-concept for a stock engine using
-PredictionIO infrastructure. *A lot of* possible improvements can be done:
-- Use spark broadcast variable to deliver the whole dataset, instead of
- repeatedly copy to each Query and TrainingData. StockEngine should wrap around
- the data object and provide the correct slice of data and prevent look-ahead
- bias.
-- Better backtesting metrics report. Should also calculate various important
- metrics like daily / annual return and volatility, sharpe and other metrics,
- drawdown, etc.
-- Better backtesting method. Should be able to handle cost of capital, margin,
- transaction costs, variable position sizes, etc.
-- And a lot more.....
-
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/build.sbt b/examples/experimental/scala-stock/build.sbt
deleted file mode 100644
index d12157f..0000000
--- a/examples/experimental/scala-stock/build.sbt
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "example-scala-stock"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.predictionio" %% "engines" % "0.9.1" % "provided",
- "com.github.scopt" %% "scopt" % "3.2.0",
- "commons-io" % "commons-io" % "2.4",
- "org.apache.commons" % "commons-math3" % "3.3",
- "org.apache.mahout" % "mahout-core" % "0.9",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0"
- exclude("org.apache.spark", "spark-core_2.10")
- exclude("org.eclipse.jetty", "jetty-server"),
- "org.clapper" %% "grizzled-slf4j" % "1.0.2",
- "org.json4s" %% "json4s-native" % "3.2.10",
- "org.scala-saddle" %% "saddle-core" % "1.3.2"
- exclude("ch.qos.logback", "logback-classic"),
- "org.scalanlp" %% "breeze" % "0.9",
- "org.scalanlp" %% "breeze-natives" % "0.9",
- "org.scalanlp" %% "nak" % "1.3",
- "org.scalatest" %% "scalatest" % "2.2.0" % "test")
-
-
-mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
- {
- case PathList("scala", xs @ _*) => MergeStrategy.discard
- case PathList("org", "xmlpull", xs @ _*) => MergeStrategy.last
- case x => old(x)
- }
-}
-
-lazy val root = (project in file(".")).enablePlugins(SbtTwirl)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/project/assembly.sbt b/examples/experimental/scala-stock/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-stock/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/project/plugins.sbt b/examples/experimental/scala-stock/project/plugins.sbt
deleted file mode 100644
index 66ee665..0000000
--- a/examples/experimental/scala-stock/project/plugins.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.typesafe.sbt" % "sbt-twirl" % "1.0.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/Algorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/Algorithm.scala b/examples/experimental/scala-stock/src/main/scala/Algorithm.scala
deleted file mode 100644
index 7512da4..0000000
--- a/examples/experimental/scala-stock/src/main/scala/Algorithm.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-import org.apache.predictionio.controller.LAlgorithm
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.broadcast.Broadcast
-import org.apache.predictionio.controller.EmptyParams
-import org.saddle._
-
-import scala.reflect._
-import scala.reflect.runtime.universe._
-
-import scala.collection.immutable.HashMap
-
-abstract class StockStrategy[M: ClassTag]
- extends LAlgorithm[
- TrainingData,
- (TrainingData, M),
- QueryDate,
- Prediction] {
- def train(trainingData: TrainingData): (TrainingData, M) = {
- (trainingData, createModel(trainingData.view))
- }
-
- def createModel(dataView: DataView): M
-
- def predict(dataModel: (TrainingData, M), queryDate: QueryDate)
- : Prediction = {
- val (trainingData, model) = dataModel
-
- val rawData = trainingData.rawDataB.value
-
- val dataView: DataView =
- rawData.view(queryDate.idx, trainingData.maxWindowSize)
-
- val active = rawData._activeFrame
-
- val activeTickers = dataView
- .activeFrame()
- .rowAt(0)
- .filter(identity)
- .index.toVec.contents
-
-
- val query = Query(
- idx = queryDate.idx,
- dataView = dataView,
- tickers = activeTickers,
- mktTicker = rawData.mktTicker)
-
- val prediction: Prediction = onClose(model, query)
-
- return prediction
- }
-
- def onClose(model: M, query: Query): Prediction
-}
-
-class EmptyStrategy extends StockStrategy[AnyRef] {
- def createModel(dataView: DataView): AnyRef = None
-
- def onClose(model: AnyRef, query: Query): Prediction =
- Prediction(HashMap[String, Double]())
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/BackTestingMetrics.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/BackTestingMetrics.scala b/examples/experimental/scala-stock/src/main/scala/BackTestingMetrics.scala
deleted file mode 100644
index c2fbe4b..0000000
--- a/examples/experimental/scala-stock/src/main/scala/BackTestingMetrics.scala
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.Evaluator
-import org.apache.predictionio.controller.NiceRendering
-import com.github.nscala_time.time.Imports._
-import scala.collection.mutable.{ Map => MMap, ArrayBuffer }
-
-import org.json4s._
-import org.json4s.JsonDSL._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization
-//import org.json4s.native.Serialization.{read, write}
-
-import org.apache.predictionio.engines.util.{ EvaluatorVisualization => MV }
-
-import breeze.stats.{ mean, meanAndVariance, MeanAndVariance }
-
-case class BacktestingParams(
- val enterThreshold: Double,
- val exitThreshold: Double,
- val maxPositions: Int = 1,
- val optOutputPath: Option[String] = None
-) extends Params {}
-
-// prediction is Ticker -> ({1:Enter, -1:Exit}, ActualReturn)
-class DailyResult(
- //val date: DateTime,
- val dateIdx: Int,
- val toEnter: Seq[String],
- val toExit: Seq[String])
-extends Serializable {}
-
-case class DailyStat (
- time: Long,
- nav: Double,
- ret: Double,
- market: Double,
- positionCount: Int
-)
-
-case class OverallStat (
- ret: Double,
- vol: Double,
- sharpe: Double,
- days: Int
-)
-
-case class BacktestingResult(
- daily: Seq[DailyStat],
- overall: OverallStat
-) with NiceRendering {
- override def toString(): String = overall.toString
-
- def toHTML(): String = {
- implicit val formats = DefaultFormats
- html.backtesting().toString
- }
-
- def toJSON(): String = {
- implicit val formats = DefaultFormats
- Serialization.write(this)
- }
-}
-
-class BacktestingEvaluator(val params: BacktestingParams)
- extends Evaluator[
- DataParams, QueryDate, Prediction, AnyRef,
- DailyResult, Seq[DailyResult], BacktestingResult] {
-
- def evaluateUnit(queryDate: QueryDate, prediction: Prediction,
- unusedActual: AnyRef)
- : DailyResult = {
-
- val todayIdx = queryDate.idx
-
- // Decide enter / exit, also sort by pValue desc
- val data = prediction.data
- .map { case (ticker, pValue) => {
- val dir = pValue match {
- case p if p >= params.enterThreshold => 1
- case p if p <= params.exitThreshold => -1
- case _ => 0
- }
- (ticker, dir, pValue)
- }}
- .toArray
- .sortBy(-_._3)
-
- val toEnter = data.filter(_._2 == 1).map(_._1)
- val toExit = data.filter(_._2 == -1).map(_._1)
-
- new DailyResult(
- dateIdx = todayIdx,
- toEnter = toEnter,
- toExit = toExit)
- }
-
- def evaluateSet(dp: DataParams, input: Seq[DailyResult])
- : Seq[DailyResult] = input
-
- def evaluateAll(input: Seq[(DataParams, Seq[DailyResult])])
- : BacktestingResult = {
- var dailyResultsSeq = input
- .map(_._2)
- .flatten
- .toArray
- .sortBy(_.dateIdx)
-
- var rawData = input.head._1.rawDataB.value
- val retFrame = rawData._retFrame
- val priceFrame = rawData._priceFrame
- val mktTicker = rawData.mktTicker
-
- val dailyNavs = ArrayBuffer[Double]()
-
- val dailyStats = ArrayBuffer[DailyStat]()
-
- val initCash = 1000000.0
- var cash = initCash
- // Ticker to current size
- val positions = MMap[String, Double]()
- val maxPositions = params.maxPositions
-
- for (daily <- dailyResultsSeq) {
- val todayIdx = daily.dateIdx
- val today = rawData.timeIndex(todayIdx)
- val todayRet = retFrame.rowAt(todayIdx)
- val todayPrice = priceFrame.rowAt(todayIdx)
-
- // Update price change
- positions.keys.foreach { ticker => {
- positions(ticker) *= todayRet.first(ticker).get
- }}
-
- // Determine exit
- daily.toExit.foreach { ticker => {
- if (positions.contains(ticker)) {
- val money = positions.remove(ticker).get
- cash += money
- }
- }}
-
- // Determine enter
- val slack = maxPositions - positions.size
- val money = cash / slack
- daily.toEnter
- .filter(t => !positions.contains(t))
- .take(slack)
- .map{ ticker => {
- cash -= money
- positions += (ticker -> money)
- }}
-
- // Book keeping
- val nav = cash + positions.values.sum
-
- val ret = (if (dailyStats.isEmpty) 0 else {
- val yestStats = dailyStats.last
- val yestNav = yestStats.nav
- (nav - yestNav) / nav - 1
- })
-
- dailyStats.append(DailyStat(
- time = today.getMillis(),
- nav = nav,
- ret = ret,
- market = todayPrice.first(mktTicker).get,
- positionCount = positions.size
- ))
- }
- // FIXME. Force Close the last day.
-
- val lastStat = dailyStats.last
-
- //val dailyVariance = meanAndVariance(dailyStats.map(_.ret))._2
- //val dailyVariance = meanAndVariance(dailyStats.map(_.ret))._2
- val retStats: MeanAndVariance = meanAndVariance(dailyStats.map(_.ret))
- //val dailyVol = math.sqrt(dailyVariance)
- //val annualVol = dailyVariance * math.sqrt(252.0)
- val annualVol = retStats.stdDev * math.sqrt(252.0)
- val n = dailyStats.size
- val totalReturn = lastStat.nav / initCash
-
- val annualReturn = math.pow(totalReturn, 252.0 / n) - 1
- val sharpe = annualReturn / annualVol
-
- val overall = OverallStat(
- annualReturn,
- annualVol,
- sharpe,
- n)
-
- val result = BacktestingResult(
- daily = dailyStats,
- overall = overall
- )
-
- params.optOutputPath.map { path => MV.save(result, path) }
-
- result
- }
-}
-
-object RenderMain {
- def main(args: Array[String]) {
- MV.render(MV.load[BacktestingResult](args(0)), args(0))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/Data.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/Data.scala b/examples/experimental/scala-stock/src/main/scala/Data.scala
deleted file mode 100644
index 56b80c7..0000000
--- a/examples/experimental/scala-stock/src/main/scala/Data.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.broadcast.Broadcast
-import org.saddle._
-import org.saddle.index.IndexTime
-import com.github.nscala_time.time.Imports._
-import scala.collection.immutable.HashMap
-
-class RawData(
- val tickers: Array[String],
- val mktTicker: String,
- val timeIndex: Array[DateTime],
- private[stock] val _price: Array[(String, Array[Double])],
- private[stock] val _active: Array[(String, Array[Boolean])])
- extends Serializable {
-
- @transient lazy val _priceFrame: Frame[DateTime, String, Double] =
- SaddleWrapper.ToFrame(timeIndex, _price)
-
- // FIXME. Fill NA of result.
- @transient lazy val _retFrame: Frame[DateTime, String, Double] =
- _priceFrame.shift(1) / _priceFrame
-
- @transient lazy val _activeFrame: Frame[DateTime, String, Boolean] =
- SaddleWrapper.ToFrame(timeIndex, _active)
-
- def view(idx: Int, maxWindowSize: Int): DataView =
- DataView(this, idx, maxWindowSize)
-
- override def toString(): String = {
- val timeHead = timeIndex.head
- val timeLast = timeIndex.last
- s"RawData[$timeHead, $timeLast, $mktTicker, size=${tickers.size}]"
- }
-}
-
-// A data view of RawData from [idx - maxWindowSize + 1 : idx]
-// Notice that the last day is *inclusive*.
-// This clas takes the whole RawData reference, hence should *not* be serialized
-case class DataView(val rawData: RawData, val idx: Int, val maxWindowSize: Int) {
- def today(): DateTime = rawData.timeIndex(idx)
-
- val tickers = rawData.tickers
- val mktTicker = rawData.mktTicker
-
- def priceFrame(windowSize: Int = 1)
- : Frame[DateTime, String, Double] = {
- // Check windowSize <= maxWindowSize
- rawData._priceFrame.rowSlice(idx - windowSize + 1, idx + 1)
- }
-
- def retFrame(windowSize: Int = 1)
- : Frame[DateTime, String, Double] = {
- // Check windowSize <= maxWindowSize
- rawData._retFrame.rowSlice(idx - windowSize + 1, idx + 1)
- }
-
- def activeFrame(windowSize: Int = 1)
- : Frame[DateTime, String, Boolean] = {
- // Check windowSize <= maxWindowSize
- rawData._activeFrame.rowSlice(idx - windowSize + 1, idx + 1)
- }
-
- override def toString(): String = {
- priceFrame().toString
- }
-}
-
-
-// Training data visible to the user is [untilIdx - windowSize, untilIdx).
-case class TrainingData(
- untilIdx: Int,
- maxWindowSize: Int,
- rawDataB: Broadcast[RawData]) {
-
- def view(): DataView = DataView(rawDataB.value, untilIdx - 1, maxWindowSize)
-}
-
-case class DataParams(rawDataB: Broadcast[RawData])
-
-// Date
-case class QueryDate(idx: Int)
-
-case class Query(
- val idx: Int,
- val dataView: DataView,
- val tickers: Array[String],
- val mktTicker: String)
-
-// Prediction
-case class Prediction(data: HashMap[String, Double])
-
-object SaddleWrapper {
- def ToFrame[A](
- timeIndex: Array[DateTime],
- tickerPriceSeq: Array[(String, Array[A])]
- )(implicit st: ST[A])
- : Frame[DateTime, String, A] = {
- val index = IndexTime(timeIndex:_ *)
- val seriesList = tickerPriceSeq.map{ case(ticker, price) => {
- val series = Series(Vec(price), index)
- (ticker, series)
- }}
- Frame(seriesList:_*)
- }
-
- def FromFrame[A](data: Frame[DateTime, String, A]
- ): (Array[DateTime], Array[(String, Array[A])]) = {
- val timeIndex = data.rowIx.toVec.contents
- val tickers = data.colIx.toVec.contents
-
- val tickerDataSeq = tickers.map{ ticker => {
- (ticker, data.firstCol(ticker).toVec.contents)
- }}
-
- (timeIndex, tickerDataSeq)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/DataSource.scala b/examples/experimental/scala-stock/src/main/scala/DataSource.scala
deleted file mode 100644
index b8436e7..0000000
--- a/examples/experimental/scala-stock/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.LDataSource
-import org.apache.predictionio.controller.EmptyParams
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.broadcast.Broadcast
-
-import com.mongodb.casbah.Imports._
-import org.saddle._
-import org.saddle.index.IndexTime
-import com.github.nscala_time.time.Imports._
-
-
-/** Primary parameter for [[[DataSource]]].
- *
- * @param baseDate identify the beginning of our global time window, and
- * the rest are use index.
- * @param fromIdx the first date for testing
- * @param untilIdx the last date (exclusive) for testing
- * @param trainingWindowSize number of days used for training
- * @param testingWindowSize number of days for each testing data
- *
- * [[[DataSource]]] chops data into (overlapping) multiple
- * pieces. Within each piece, it is further splitted into training and testing
- * set. The testing sets is from <code>fromIdx</code> until
- * <code>untilIdx</code> with a step size of <code>testingWindowSize</code>.
- * A concrete example: (from, until, windowSize) = (100, 150, 20), it generates
- * three testing sets corresponding to time range: [100, 120), [120, 140), [140, 150).
- * For each testing sets, it also generates the training data set using
- * <code>maxTrainingWindowSize</code>. Suppose trainingWindowSize = 50 and testing set =
- * [100, 120), the training set draws data in time range [50, 100).
- */
-
-case class DataSourceParams(
- val appid: Int = 1008,
- val baseDate: DateTime,
- val fromIdx: Int,
- val untilIdx: Int,
- val trainingWindowSize: Int,
- val maxTestingWindowSize: Int,
- val marketTicker: String,
- val tickerList: Seq[String]) extends Params {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/Indicators.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/Indicators.scala b/examples/experimental/scala-stock/src/main/scala/Indicators.scala
deleted file mode 100644
index a8a0ac1..0000000
--- a/examples/experimental/scala-stock/src/main/scala/Indicators.scala
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-import org.saddle._
-import org.saddle.index.IndexTime
-
-import scala.collection.immutable.HashMap
-import breeze.linalg.DenseMatrix
-import breeze.linalg.DenseVector
-
-import com.github.nscala_time.time.Imports._
-
-import scala.math
-
-import math._
-
-import nak.regress.LinearRegression
-
-/**
- * Base class for an indicator.
- *
- * All indicators should be defined as classes that extend
- * this base class. See RSIIndicator as an example. These indicators can then
- * be instantiated and passed into a StockStrategy class. Refer to tutorial
- * for further explanation (found in the README.md file).
- */
-@SerialVersionUID(100L)
-abstract class BaseIndicator extends Serializable {
- /** Calculates training series for a particular stock.
- *
- * @param logPrice series of logarithm of all prices for a particular stock.
- * Logarithm values are recommended for more accurate results.
- * @return the training series of the stock
- */
- def getTraining(logPrice: Series[DateTime, Double]): Series[DateTime, Double]
-
- /** Applies indicator on a window size of the value returned by
- * getMinWindowSize() and returns the last value in the resulting series to
- * be used for prediction in RegressionStrategy.
- *
- * @param logPrice series of logarithm of all prices for a particular stock
- * @return the last value in the resulting series from the feature
- * calculation
- */
- def getOne(input: Series[DateTime, Double]): Double
-
- /** Returns window size to be used in getOne()
- *
- * @return the window size
- */
- def getMinWindowSize(): Int
-}
-
-/** Indicator that implements a relative strength index formula
- *
- * @constructor create an instance of an RSIIndicator
- * @param period number of days to use for each of the 14 periods
- * that are used in the RSI calculation
- */
-class RSIIndicator(rsiPeriod: Int = 14) extends BaseIndicator {
-
- private def getRet(dailyReturn: Series[DateTime, Double]) =
- (dailyReturn - dailyReturn.shift(1)).fillNA(_ => 0.0)
-
- def getMinWindowSize(): Int = rsiPeriod + 1
-
- private def calcRS(logPrice: Series[DateTime, Double])
- : Series[DateTime, Double] = {
- //Positive and Negative Vecs
- val posSeries = logPrice.mapValues[Double]((x: Double)
- => if (x > 0) x else 0)
- val negSeries = logPrice.mapValues[Double]((x: Double)
- => if (x < 0) x else 0)
-
- //Get the sum of positive/negative Frame
- val avgPosSeries =
- posSeries.rolling[Double] (rsiPeriod, (f: Series[DateTime,Double])
- => f.mean)
- val avgNegSeries =
- negSeries.rolling[Double] (rsiPeriod, (f: Series[DateTime,Double])
- => f.mean)
-
- val rsSeries = avgPosSeries / avgNegSeries
- rsSeries
- }
-
- // Computes RSI of price data over the defined training window time frame
- def getTraining(logPrice: Series[DateTime, Double])
- : Series[DateTime, Double] = {
- val rsSeries = calcRS(getRet(logPrice))
- val rsiSeries = rsSeries.mapValues[Double](
- (x:Double) => 100 - ( 100 / (1 + x)))
-
- // Fill in first 14 days offset with 50 to maintain results
- rsiSeries.reindex(logPrice.rowIx).fillNA(_ => 50.0)
- }
-
- // Computes the RSI for the most recent time frame, returns single double
- def getOne(logPrice: Series[DateTime, Double]): Double = {
- getTraining(logPrice).last
- }
-}
-
-/** Indicator that calcuate differences of closing prices
- *
- * @constructor create an instance of a ShiftsIndicator
- * @param period number of days between any 2 closing prices to consider for
- * calculating a return
- */
-class ShiftsIndicator(period: Int) extends BaseIndicator {
-
- private def getRet(logPrice: Series[DateTime, Double], frame: Int = period) =
- (logPrice - logPrice.shift(frame)).fillNA(_ => 0.0)
-
- def getMinWindowSize(): Int = period + 1
-
- def getTraining(logPrice: Series[DateTime, Double])
- : Series[DateTime, Double] = {
- getRet(logPrice)
- }
-
- def getOne(logPrice: Series[DateTime, Double]): Double = {
- getRet(logPrice).last
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/RegressionStrategy.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/RegressionStrategy.scala b/examples/experimental/scala-stock/src/main/scala/RegressionStrategy.scala
deleted file mode 100644
index 7647f6d..0000000
--- a/examples/experimental/scala-stock/src/main/scala/RegressionStrategy.scala
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-import org.apache.predictionio.controller.Params
-
-import org.saddle._
-import org.saddle.index.IndexTime
-
-import scala.collection.immutable.HashMap
-import breeze.linalg.DenseMatrix
-import breeze.linalg.DenseVector
-
-import com.github.nscala_time.time.Imports._
-
-import scala.math
-
-import nak.regress.LinearRegression
-
-/** Regression Strategy parameters case class
- *
- * @param indicators a sequence of tuples. The first element is a string that
- * can be helpful in acting as a label for the indicator,
- * such as for debugging. Pass in empty string if no label
- * desired.
- * @param maxTrainingWindowSize maximum window size of price frame desired
- * for training
- */
-case class RegressionStrategyParams (
- indicators: Seq[(String, BaseIndicator)],
- maxTrainingWindowSize: Int
-) extends Params
-
-class RegressionStrategy (params: RegressionStrategyParams)
- extends StockStrategy[Map[String, DenseVector[Double]]] {
-
- private def getRet(logPrice: Frame[DateTime, String, Double], d: Int) =
- (logPrice - logPrice.shift(d)).mapVec[Double](_.fillNA(_ => 0.0))
-
- // Regress on specific ticker
- private def regress(
- calculatedData: Seq[Series[DateTime, Double]],
- retF1d: Series[DateTime, Double]) = {
- val array = (
- calculatedData.map(_.toVec.contents).reduce(_++_) ++
- Array.fill(retF1d.length)(1.0)).toArray[Double]
- val target = DenseVector[Double](retF1d.toVec.contents)
- val m = DenseMatrix.create[Double](
- retF1d.length,
- calculatedData.length + 1,
- array
- )
- val result = LinearRegression.regress(m, target)
- result
- }
-
- // Compute each indicator value for training the model
- private def calcIndicator(logPrice: Series[DateTime, Double]):
- Seq[Series[DateTime, Double]] = {
- params.indicators.map {
- case(name, indicator) => indicator.getTraining(logPrice)
- }
- }
-
- // Get max period from series of indicators
- private def getMaxPeriod() : Int = {
- // create an array of periods
- val periods = params.indicators.map {
- case(name, indicator) => indicator.getMinWindowSize()
- }
- periods.max
- }
-
- // Apply regression algorithm on complete dataset to create a model
- def createModel(dataView: DataView): Map[String, DenseVector[Double]] = {
- // price: row is time, col is ticker, values are prices
- val price = dataView.priceFrame(params.maxTrainingWindowSize)
- val logPrice = price.mapValues(math.log)
- val active = dataView.activeFrame(params.maxTrainingWindowSize)
-
- // value used to query prediction results
- val retF1d = getRet(logPrice, -1)
-
- val timeIndex = price.rowIx
- val firstIdx = getMaxPeriod() + 3
- val lastIdx = timeIndex.length
-
- // Get array of ticker strings
- val tickers = price.colIx.toVec.contents
-
- // For each active ticker, pass in trained series into regress
- val tickerModelMap = tickers
- .filter(ticker => (active.firstCol(ticker).findOne(_ == false) == -1))
- .map(ticker => {
- val model = regress(
- calcIndicator(price.firstCol(ticker)).map(_.slice(firstIdx, lastIdx)),
- retF1d.firstCol(ticker).slice(firstIdx, lastIdx))
- (ticker, model)
- }).toMap
-
- // tickers mapped to model
- tickerModelMap
- }
-
- // returns a prediction for a specific ticker
- private def predictOne(
- coef: DenseVector[Double],
- ticker: String,
- dataView: DataView): Double = {
-
- val vecArray = params.indicators.map { case (name, indicator) => {
- val price = dataView.priceFrame(indicator.getMinWindowSize())
- val logPrice = price.mapValues(math.log)
- indicator.getOne(logPrice.firstCol(ticker))
- }}.toArray
-
- val densVecArray = vecArray ++ Array[Double](1)
- val vec = DenseVector[Double](densVecArray)
-
- val p = coef.dot(vec)
- return p
- }
-
- // Returns a mapping of tickers to predictions
- def onClose(model: Map[String, DenseVector[Double]], query: Query)
- : Prediction = {
- val dataView = query.dataView
-
- val prediction = query.tickers
- .filter(ticker => model.contains(ticker))
- .map { ticker => {
- val p = predictOne(
- model(ticker),
- ticker,
- dataView)
- (ticker, p)
- }}
-
- Prediction(HashMap[String, Double](prediction:_*))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/Run.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/Run.scala b/examples/experimental/scala-stock/src/main/scala/Run.scala
deleted file mode 100644
index 32c5d14..0000000
--- a/examples/experimental/scala-stock/src/main/scala/Run.scala
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-import org.apache.predictionio.controller.Workflow
-import org.apache.predictionio.controller.WorkflowParams
-import org.apache.predictionio.controller.PIdentityPreparator
-import org.apache.predictionio.controller.EmptyParams
-import org.apache.predictionio.controller.LFirstServing
-import org.apache.predictionio.controller.Params
-import com.github.nscala_time.time.Imports._
-import scala.collection.immutable.HashMap
-import java.io.File
-
-// Buy if l-days daily return is high than s-days daily-return
-case class MomentumStrategyParams(val l: Int, val s: Int) extends Params
-
-class MomentumStrategy(val p: MomentumStrategyParams)
- extends StockStrategy[AnyRef] {
-
- def createModel(dataView: DataView): AnyRef = None
-
- def onClose(model: AnyRef, query: Query): Prediction = {
- val dataView = query.dataView
-
- val priceFrame = dataView.priceFrame(p.l + 1)
- val todayLgPrice = priceFrame.rowAt(p.l).mapValues(math.log)
- val lLgPrice = priceFrame.rowAt(0).mapValues(math.log)
- val sLgPrice = priceFrame.rowAt(p.l - p.s).mapValues(math.log)
-
- val sLgRet = (todayLgPrice - sLgPrice) / p.s
- val lLgRet = (todayLgPrice - lLgPrice) / p.l
-
- val output = query.tickers
- .map { ticker => {
- val s = sLgRet.first(ticker)
- val l = lLgRet.first(ticker)
- val p = l - s
- (ticker, p)
- }}
-
- Prediction(data = HashMap(output:_*))
- }
-}
-
-
-object Run {
- val sp500List = Vector(
- "A", "AA", "AAPL", "ABBV", "ABC", "ABT", "ACE", "ACN", "ACT", "ADBE", "ADI",
- "ADM", "ADP", "ADS", "ADSK", "ADT", "AEE", "AEP", "AES", "AET", "AFL",
- "AGN", "AIG", "AIV", "AIZ", "AKAM", "ALL", "ALLE", "ALTR", "ALXN", "AMAT",
- "AME", "AMGN", "AMP", "AMT", "AMZN", "AN", "AON", "APA", "APC", "APD",
- "APH", "ARG", "ATI", "AVB", "AVP", "AVY", "AXP", "AZO", "BA", "BAC", "BAX",
- "BBBY", "BBT", "BBY", "BCR", "BDX", "BEAM", "BEN", "BF-B", "BHI", "BIIB",
- "BK", "BLK", "BLL", "BMS", "BMY", "BRCM", "BRK-B", "BSX", "BTU", "BWA",
- "BXP", "C", "CA", "CAG", "CAH", "CAM", "CAT", "CB", "CBG", "CBS", "CCE",
- "CCI", "CCL", "CELG", "CERN", "CF", "CFN", "CHK", "CHRW", "CI", "CINF",
- "CL", "CLX", "CMA", "CMCSA", "CME", "CMG", "CMI", "CMS", "CNP", "CNX",
- "COF", "COG", "COH", "COL", "COP", "COST", "COV", "CPB", "CRM", "CSC",
- "CSCO", "CSX", "CTAS", "CTL", "CTSH", "CTXS", "CVC", "CVS", "CVX", "D",
- "DAL", "DD", "DE", "DFS", "DG", "DGX", "DHI", "DHR", "DIS", "DISCA", "DLPH",
- "DLTR", "DNB", "DNR", "DO", "DOV", "DOW", "DPS", "DRI", "DTE", "DTV", "DUK",
- "DVA", "DVN", "EA", "EBAY", "ECL", "ED", "EFX", "EIX", "EL", "EMC", "EMN",
- "EMR", "EOG", "EQR", "EQT", "ESRX", "ESS", "ESV", "ETFC", "ETN", "ETR",
- "EW", "EXC", "EXPD", "EXPE", "F", "FAST", "FB", "FCX", "FDO", "FDX", "FE",
- "FFIV", "FIS", "FISV", "FITB", "FLIR", "FLR", "FLS", "FMC", "FOSL", "FOXA",
- "FRX", "FSLR", "FTI", "FTR", "GAS", "GCI", "GD", "GE", "GGP", "GHC", "GILD",
- "GIS", "GLW", "GM", "GMCR", "GME", "GNW", "GOOG", "GOOGL", "GPC", "GPS",
- "GRMN", "GS", "GT", "GWW", "HAL", "HAR", "HAS", "HBAN", "HCBK", "HCN",
- "HCP", "HD", "HES", "HIG", "HOG", "HON", "HOT", "HP", "HPQ", "HRB", "HRL",
- "HRS", "HSP", "HST", "HSY", "HUM", "IBM", "ICE", "IFF", "IGT", "INTC",
- "INTU", "IP", "IPG", "IR", "IRM", "ISRG", "ITW", "IVZ", "JBL", "JCI", "JEC",
- "JNJ", "JNPR", "JOY", "JPM", "JWN", "K", "KEY", "KIM", "KLAC", "KMB", "KMI",
- "KMX", "KO", "KORS", "KR", "KRFT", "KSS", "KSU", "L", "LB", "LEG", "LEN",
- "LH", "LLL", "LLTC", "LLY", "LM", "LMT", "LNC", "LO", "LOW", "LRCX", "LSI",
- "LUK", "LUV", "LYB", "M", "MA", "MAC", "MAR", "MAS", "MAT", "MCD", "MCHP",
- "MCK", "MCO", "MDLZ", "MDT", "MET", "MHFI", "MHK", "MJN", "MKC", "MMC",
- "MMM", "MNST", "MO", "MON", "MOS", "MPC", "MRK", "MRO", "MS", "MSFT", "MSI",
- "MTB", "MU", "MUR", "MWV", "MYL", "NBL", "NBR", "NDAQ", "NE", "NEE", "NEM",
- "NFLX", "NFX", "NI", "NKE", "NLSN", "NOC", "NOV", "NRG", "NSC", "NTAP",
- "NTRS", "NU", "NUE", "NVDA", "NWL", "NWSA", "OI", "OKE", "OMC", "ORCL",
- "ORLY", "OXY", "PAYX", "PBCT", "PBI", "PCAR", "PCG", "PCL", "PCLN", "PCP",
- "PDCO", "PEG", "PEP", "PETM", "PFE", "PFG", "PG", "PGR", "PH", "PHM", "PKI",
- "PLD", "PLL", "PM", "PNC", "PNR", "PNW", "POM", "PPG", "PPL", "PRGO", "PRU",
- "PSA", "PSX", "PVH", "PWR", "PX", "PXD", "QCOM", "QEP", "R", "RAI", "RDC",
- "REGN", "RF", "RHI", "RHT", "RIG", "RL", "ROK", "ROP", "ROST", "RRC", "RSG",
- "RTN", "SBUX", "SCG", "SCHW", "SE", "SEE", "SHW", "SIAL", "SJM", "SLB",
- "SLM", "SNA", "SNDK", "SNI", "SO", "SPG", "SPLS", "SRCL", "SRE", "STI",
- "STJ", "STT", "STX", "STZ", "SWK", "SWN", "SWY", "SYK", "SYMC", "SYY", "T",
- "TAP", "TDC", "TE", "TEG", "TEL", "TGT", "THC", "TIF", "TJX", "TMK", "TMO",
- "TRIP", "TROW", "TRV", "TSCO", "TSN", "TSO", "TSS", "TWC", "TWX", "TXN",
- "TXT", "TYC", "UNH", "UNM", "UNP", "UPS", "URBN", "USB", "UTX", "V", "VAR",
- "VFC", "VIAB", "VLO", "VMC", "VNO", "VRSN", "VRTX", "VTR", "VZ", "WAG",
- "WAT", "WDC", "WEC", "WFC", "WFM", "WHR", "WIN", "WLP", "WM", "WMB", "WMT",
- "WU", "WY", "WYN", "WYNN", "X", "XEL", "XL", "XLNX", "XOM", "XRAY", "XRX",
- "XYL", "YHOO", "YUM", "ZION", "ZMH", "ZTS")
-
- val tickerList = Seq(
- "GOOG", "GOOGL", "FB", "AAPL", "AMZN", "MSFT", "IBM", "HPQ", "INTC",
- "NTAP", "CSCO", "ORCL", "XRX", "YHOO", "AMAT", "QCOM", "TXN", "CRM",
- "INTU", "WDC", "SNDK")
-
- def main(args: Array[String]) {
- val dataSourceParams = (if (false) {
- new DataSourceParams(
- baseDate = new DateTime(2002, 1, 1, 0, 0),
- fromIdx = 300,
- untilIdx = 2000,
- trainingWindowSize = 200,
- maxTestingWindowSize = 20,
- marketTicker = "SPY",
- tickerList = tickerList)
- } else {
- // Need to pass "--driver-memory 8G" to pio-run since it requires a lot
- // of driver memory.
- new DataSourceParams(
- baseDate = new DateTime(2002, 1, 1, 0, 0),
- fromIdx = 300,
- untilIdx = 2000,
- trainingWindowSize = 200,
- maxTestingWindowSize = 20,
- marketTicker = "SPY",
- tickerList = sp500List)
- })
-
- val momentumParams = MomentumStrategyParams(20, 3)
-
- val evaluatorParams = BacktestingParams(
- enterThreshold = 0.01,
- exitThreshold = 0.0,
- maxPositions = 10,
- optOutputPath = Some(new File("metrics_results").getCanonicalPath)
- )
-
- Workflow.run(
- dataSourceClassOpt = Some(classOf[YahooDataSource]),
- dataSourceParams = dataSourceParams,
- preparatorClassOpt = Some(PIdentityPreparator(classOf[YahooDataSource])),
- algorithmClassMapOpt = Some(Map(
- //"" -> classOf[MomentumStrategy]
- "" -> classOf[RegressionStrategy]
- )),
- algorithmParamsList = Seq(("", momentumParams)),
- servingClassOpt = Some(LFirstServing(classOf[EmptyStrategy])),
- evaluatorClassOpt = Some(classOf[BacktestingEvaluator]),
- evaluatorParams = evaluatorParams,
- params = WorkflowParams(
- verbose = 0,
- batch = "Imagine: Stock II"))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/scala/YahooDataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/scala/YahooDataSource.scala b/examples/experimental/scala-stock/src/main/scala/YahooDataSource.scala
deleted file mode 100644
index d23d402..0000000
--- a/examples/experimental/scala-stock/src/main/scala/YahooDataSource.scala
+++ /dev/null
@@ -1,483 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.stock
-
-// YahooDataSource reads PredictionIO event store directly.
-
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-import org.apache.predictionio.data.view.LBatchView
-import org.apache.predictionio.data.storage.DataMap
-
-import org.joda.time.DateTime
-import org.joda.time.DateTimeZone
-import com.github.nscala_time.time.Imports._
-
-import scala.collection.mutable.{ Map => MMap }
-import scala.collection.GenMap
-
-import org.apache.predictionio.controller._
-import org.apache.predictionio.controller.{ Params => BaseParams }
-
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.broadcast.Broadcast
-
-import org.json4s._
-//import org.saddle._
-
-case class HistoricalData(
- ticker: String,
- timeIndex: Array[DateTime],
- close: Array[Double],
- adjClose: Array[Double],
- adjReturn: Array[Double],
- volume: Array[Double],
- active: Array[Boolean]) {
-
- override def toString(): String = {
- s"HistoricalData($ticker, ${timeIndex.head}, ${timeIndex.last}, " +
- s"${close.last})"
- }
-
- def toDetailedString(): String = {
- val adjCloseStr = adjClose.mkString("[", ", ", "]")
- val adjReturnStr = adjReturn.mkString("[", ", ", "]")
- val activeStr = active.mkString("[", ", ", "]")
-
- (s"HistoricalData($ticker, ${timeIndex.head}, ${timeIndex.last}, \n" +
- s" adjClose=$adjCloseStr\n" +
- s" adjReturn=$adjReturnStr\n" +
- s" active=$activeStr)")
- }
-}
-
-object HistoricalData {
- def apply(ticker: String, timeIndex: Array[DateTime]): HistoricalData = {
- val n = timeIndex.size
- HistoricalData(
- ticker,
- timeIndex,
- close = Array.fill(n)(0.0),
- adjClose = Array.fill(n)(0.0),
- adjReturn = Array.fill(n)(0.0),
- volume = Array.fill(n)(0.0),
- active = Array.fill(n)(false))
- }
-}
-
-
-class YahooDataSource(val params: YahooDataSource.Params)
- extends PDataSource[
- RDD[TrainingData],
- DataParams,
- QueryDate,
- AnyRef] {
- @transient lazy val batchView = new LBatchView(
- params.appId, params.startTime, params.untilTime)
-
- val timezone = DateTimeZone.forID("US/Eastern")
- val windowParams = params.windowParams
- val marketTicker = windowParams.marketTicker
-
- @transient lazy val market: HistoricalData = getTimeIndex()
- @transient lazy val timeIndex: Array[DateTime] = market.timeIndex
- @transient lazy val timeIndexSet: Set[DateTime] = timeIndex.toSet
-
- def merge(intermediate: YahooDataSource.Intermediate, e: Event,
- timeIndexSetOpt: Option[Set[DateTime]])
- : YahooDataSource.Intermediate = {
- val dm: DataMap = e.properties
-
- // TODO: Check ticker in intermediate
-
- val yahooData = dm.get[JObject]("yahoo")
-
- // used by json4s "extract" method.
- implicit val formats = DefaultFormats
-
- val closeList = (yahooData \ "close").extract[Array[Double]]
- val adjCloseList = (yahooData \ "adjclose").extract[Array[Double]]
- val volumeList = (yahooData \ "volume").extract[Array[Double]]
-
- val tList: Array[DateTime] = (yahooData \ "t").extract[Array[Long]]
- .map(t => new DateTime(t * 1000, timezone))
-
- // Add data either
- // 1. timeIndex exists and t is in timeIndex, or
- // 2. timeIndex is None.
- val newDailyMap: Map[DateTime, YahooDataSource.Daily] =
- tList.zipWithIndex.drop(1)
- .filter { case (t, idx) => timeIndexSetOpt.map(_(t)).getOrElse(true) }
- .map { case (t, idx) =>
- val adjReturn = (adjCloseList(idx) / adjCloseList(idx - 1)) - 1
-
- val daily = YahooDataSource.Daily(
- close = closeList(idx),
- adjClose = adjCloseList(idx),
- adjReturn = adjReturn,
- volume = volumeList(idx),
- active = true,
- prevDate = tList(idx - 1))
-
- (t -> daily)
- }
- .toMap
-
- YahooDataSource.Intermediate(
- ticker = e.entityId,
- dailyMap = intermediate.dailyMap ++ newDailyMap)
- }
-
- def mergeTimeIndex(intermediate: YahooDataSource.Intermediate, e: Event)
- : YahooDataSource.Intermediate = merge(intermediate, e, None)
-
- def mergeStock(intermediate: YahooDataSource.Intermediate, e: Event)
- : YahooDataSource.Intermediate = merge(intermediate, e, Some(timeIndexSet))
-
- def finalizeTimeIndex(intermediate: YahooDataSource.Intermediate)
- : HistoricalData = {
- val dailyMap = intermediate.dailyMap
- val ticker = intermediate.ticker
-
- // Construct the time index with windowParams
- val timeIndex: Array[DateTime] = dailyMap.keys.toArray
- .filter(_.isAfter(params.windowParams.baseDate))
- .sortBy(identity)
- .take(params.windowParams.untilIdx)
-
- // Check if the time is continuous
- (1 until timeIndex.size).foreach { idx => {
- require(dailyMap(timeIndex(idx)).prevDate == timeIndex(idx - 1),
- s"Time must be continuous. " +
- s"For ticker $ticker, there is a gap between " +
- s"${timeIndex(idx - 1)} and ${timeIndex(idx)}. " +
- s"Please import data to cover the gap or use a shorter range.")
- }}
-
- val adjReturn = timeIndex.map(t => dailyMap(t).adjReturn)
-
- HistoricalData(
- ticker = ticker,
- timeIndex = timeIndex,
- close = timeIndex.map(t => dailyMap(t).close),
- adjClose = return2Close(adjReturn),
- adjReturn = adjReturn,
- volume = timeIndex.map(t => dailyMap(t).volume),
- active = Array.fill(timeIndex.size)(true))
- }
-
- def return2Close(returns: Array[Double], base: Double = 100.0)
- : Array[Double] = {
- var v = base
- returns.map { ret =>
- v *= (1 + ret)
- v
- }
- }
-
- // Traverse the timeIndex to construct the actual time series using dailyMap
- // and extra fillNA logic.
- //
- // The time series is constructed in the same order as the global timeIndex
- // array. For a datetime t, if dailyMap contains the data, it calls valueFunc
- // to extract the value; otherwise, it calls fillNaFunc with the optional last
- // extracted value and get the default value.
- def activeFilter[A : Manifest](
- dailyMap: GenMap[DateTime, YahooDataSource.Daily],
- valueFunc: YahooDataSource.Daily => A,
- fillNAFunc: Option[A] => A) : Array[A] = {
-
- var lastOpt: Option[A] = None
-
- timeIndex
- .map { t =>
- if (dailyMap.contains(t)) {
- val v = valueFunc(dailyMap(t))
- lastOpt = Some(v)
- v
- } else {
- fillNAFunc(lastOpt)
- }
- }
- .toArray
- }
-
- def finalizeStock(intermediate: YahooDataSource.Intermediate)
- : HistoricalData = {
- val dailyMap = intermediate.dailyMap
-
- val adjReturn = activeFilter[Double](dailyMap, _.adjReturn, _ => 0.0)
-
- HistoricalData(
- ticker = intermediate.ticker,
- timeIndex = timeIndex,
- close = activeFilter[Double](dailyMap, _.close, _.getOrElse(0.0)),
- adjClose = return2Close(adjReturn),
- adjReturn = adjReturn,
- volume = activeFilter[Double](dailyMap, _.adjReturn, _ => 0.0),
- active = activeFilter[Boolean](dailyMap, _.active, _ => false))
- }
-
- def getTimeIndex(): HistoricalData = {
- // Only extracts market ticker as the main reference of market hours
- val predicate = (e: Event) =>
- (e.entityType == params.entityType && e.entityId == marketTicker)
-
- val tickerMap: Map[String, HistoricalData] = batchView
- .events
- .filter(predicate)
- .aggregateByEntityOrdered(
- //predicate,
- YahooDataSource.Intermediate(),
- mergeTimeIndex)
- .mapValues(finalizeTimeIndex)
-
- tickerMap(marketTicker)
- }
-
- def getHistoricalDataSet()
- : Map[String, HistoricalData] = {
- // Also extract market ticker again, just as a normal stock.
- val tickerSet = (windowParams.tickerList :+ windowParams.marketTicker).toSet
- val predicate = (e: Event) =>
- (e.entityType == params.entityType && tickerSet(e.entityId))
-
- val defaultTickerMap: Map[String, HistoricalData] =
- params.windowParams.tickerList.map {
- ticker => (ticker -> HistoricalData(ticker, timeIndex))
- }
- .toMap
-
- val tickerMap: Map[String, HistoricalData] = batchView
- .events
- .filter(predicate)
- .aggregateByEntityOrdered(
- //predicate,
- YahooDataSource.Intermediate(),
- mergeStock)
- .mapValues(finalizeStock)
-
- /*
- tickerMap.map { case (ticker, data) =>
- println(ticker)
- println(data.toDetailedString)
- }
- */
-
- defaultTickerMap ++ tickerMap
- }
-
- def getRawData(tickerMap: Map[String, HistoricalData]): RawData = {
- val allTickers = windowParams.tickerList :+ windowParams.marketTicker
-
- val price: Array[(String, Array[Double])] = allTickers
- .map { ticker => (ticker, tickerMap(ticker).adjClose) }
- .toArray
-
- val active: Array[(String, Array[Boolean])] = allTickers
- .map { ticker => (ticker, tickerMap(ticker).active) }
- .toArray
-
- new RawData(
- tickers = windowParams.tickerList.toArray,
- mktTicker = windowParams.marketTicker,
- timeIndex = timeIndex,
- _price = price,
- _active = active)
- }
-
- override
- def read(sc: SparkContext)
- : Seq[(RDD[TrainingData], DataParams, RDD[(QueryDate, AnyRef)])] = {
- val historicalSet = getHistoricalDataSet()
- //data.foreach { println }
- val rawData = getRawData(historicalSet)
- //println(rawData)
-
- // Broadcast it.
- val rawDataB = sc.broadcast(rawData)
-
- val dataParams = DataParams(rawDataB)
-
- val dsp: DataSourceParams = windowParams
-
- val dataSet: Seq[(TrainingData, Seq[(QueryDate, AnyRef)])] =
- Range(dsp.fromIdx, dsp.untilIdx, dsp.maxTestingWindowSize).map { idx => {
- val trainingData = TrainingData(
- untilIdx = idx,
- maxWindowSize = dsp.trainingWindowSize,
- rawDataB = rawDataB)
-
- // cannot evaluate the last item as data view only last until untilIdx.
- val testingUntilIdx = math.min(
- idx + dsp.maxTestingWindowSize,
- dsp.untilIdx - 1)
-
- val queries = (idx until testingUntilIdx)
- .map { idx => (QueryDate(idx), None) }
- (trainingData, queries)
- }}
-
- dataSet.map { case (trainingData, queries) =>
- /*
- (dataParams,
- sc.parallelize(Array(trainingData)),
- sc.parallelize(queries))
- */
- (
- sc.parallelize(Array(trainingData)),
- dataParams,
- sc.parallelize(queries))
- }
- }
-}
-
-object YahooDataSource {
- case class Params(
- windowParams: DataSourceParams,
- // Below filters with DataAPISpecific details
- appId: Int, // Ignore appId in DataSourceParams
- entityType: String,
- startTime: Option[DateTime] = None,
- untilTime: Option[DateTime] = None
- ) extends BaseParams
-
- case class Daily(
- close: Double,
- adjClose: Double,
- adjReturn: Double,
- volume: Double,
- active: Boolean,
- // prevDate is used to verify continuity
- prevDate: DateTime)
-
- /** Intermediate storage for constructing historical data
- * @param timeIndexSet Only datetime in this set is used to create historical
- * data.
- */
- case class Intermediate(
- ticker: String = "",
- dailyMap: Map[DateTime, Daily] = Map[DateTime, Daily]()
- ) {
- override def toString(): String =
- s"YDS.Intermediate($ticker, size=${dailyMap.size})"
- }
-
-
- /*
- def main(args: Array[String]) {
- val params = Params(
- appId = 1,
- untilTime = Some(new DateTime(2014, 5, 1, 0, 0)))
- //untilTime = None)
-
- val ds = new YahooDataSource(params)
- //ds.read
- }
- */
-}
-
-object PredefinedDSP {
- val BigSP500 = YahooDataSource.Params(
- appId = 2,
- entityType = "yahoo",
- untilTime = None,
- windowParams = DataSourceParams(
- baseDate = new DateTime(2000, 1, 1, 0, 0),
- fromIdx = 250,
- untilIdx = 3500,
- trainingWindowSize = 200,
- maxTestingWindowSize = 30,
- marketTicker = "SPY",
- tickerList = Run.sp500List))
-
- val SmallSP500 = YahooDataSource.Params(
- appId = 4,
- entityType = "yahoo",
- untilTime = None,
- windowParams = DataSourceParams(
- baseDate = new DateTime(2000, 1, 1, 0, 0),
- fromIdx = 250,
- untilIdx = 3500,
- trainingWindowSize = 200,
- maxTestingWindowSize = 30,
- marketTicker = "SPY",
- tickerList = Run.sp500List.take(25)))
-
- val Test = YahooDataSource.Params(
- appId = 4,
- entityType = "yahoo",
- untilTime = Some(new DateTime(2014, 5, 1, 0, 0)),
- windowParams = DataSourceParams(
- baseDate = new DateTime(2014, 1, 1, 0, 0),
- fromIdx = 20,
- untilIdx = 50,
- trainingWindowSize = 15,
- maxTestingWindowSize = 10,
- marketTicker = "SPY",
- tickerList = Seq("AAPL", "MSFT", "IBM", "FB", "AMZN", "IRONMAN")))
-}
-
-object YahooDataSourceRun {
-
- def main(args: Array[String]) {
- // Make sure you have a lot of memory.
- // --driver-memory 12G
-
- // val dsp = PredefinedDSP.BigSP500
- val dsp = PredefinedDSP.SmallSP500
- //val dsp = PredefinedDSP.Test
-
- val momentumParams = MomentumStrategyParams(20, 3)
-
- //val x = Series(Vec(1,2,3))
- //println(x)
-
- val metricsParams = BacktestingParams(
- enterThreshold = 0.01,
- exitThreshold = 0.0,
- maxPositions = 10//,
- //optOutputPath = Some(new File("metrics_results").getCanonicalPath)
- )
-
- Workflow.run(
- dataSourceClassOpt = Some(classOf[YahooDataSource]),
- dataSourceParams = dsp,
- preparatorClassOpt = Some(IdentityPreparator(classOf[YahooDataSource])),
- algorithmClassMapOpt = Some(Map(
- //"" -> classOf[MomentumStrategy]
- "" -> classOf[RegressionStrategy]
- )),
- //algorithmParamsList = Seq(("", momentumParams)),
- algorithmParamsList = Seq(("", RegressionStrategyParams(Seq[(String, BaseIndicator)](
- ("RSI1", new RSIIndicator(rsiPeriod=1)),
- ("RSI5", new RSIIndicator(rsiPeriod=5)),
- ("RSI22", new RSIIndicator(rsiPeriod=22))),
- 200))),
- servingClassOpt = Some(LFirstServing(classOf[EmptyStrategy])),
- evaluatorClassOpt = Some(classOf[BacktestingEvaluator]),
- evaluatorParams = metricsParams,
- params = WorkflowParams(
- verbose = 0,
- saveModel = false,
- batch = "Imagine: Stock III"))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-stock/src/main/twirl/io/prediction/examples/stock/backtesting.scala.html
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-stock/src/main/twirl/io/prediction/examples/stock/backtesting.scala.html b/examples/experimental/scala-stock/src/main/twirl/io/prediction/examples/stock/backtesting.scala.html
deleted file mode 100644
index ad0552f..0000000
--- a/examples/experimental/scala-stock/src/main/twirl/io/prediction/examples/stock/backtesting.scala.html
+++ /dev/null
@@ -1,125 +0,0 @@
-@import org.apache.predictionio.examples.stock.BacktestingResult
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
- <head>
- <script type='text/javascript' src='http://www.google.com/jsapi'></script>
- <script
- src="http://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
- <script type='text/javascript'>
- google.load('visualization', '1',
- {'packages':['annotatedtimeline', 'table', 'corechart']});
- </script>
- </head>
- <body>
- <h3>Backtesting Result</h3>
- <div id='overall' style='width:1024px;'></div>
-
- <div id='t_timeline' style='width: 1024px; height: 360px;'></div>
- <div id='b_timeline' style='width: 1024px; height: 200px;'></div>
- <!-- Event listener only works when served from http server -->
-
- <script>
- google.setOnLoadCallback(draw);
- var tChart;
- var bChart;
-
- var rawData;
-
- function draw() {
- var jsonData = $.ajax({
- url: 'evaluator_results.json',
- dataType: 'json',
- async: false,
- }).responseText;
-
- rawData = JSON.parse(jsonData);
-
- drawOverall();
- drawTimeline();
- }
-
- function drawOverall() {
- var data = new google.visualization.DataTable();
- data.addColumn('number', 'ret');
- data.addColumn('number', 'vol');
- data.addColumn('number', 'sharpe');
- data.addColumn('number', 'days');
-
- var overall = rawData.overall;
- data.addRow([overall.ret, overall.vol, overall.sharpe, overall.days]);
-
- var formatter = new google.visualization.NumberFormat(
- {fractionDigits: 4});
- formatter.format(data, 0);
- formatter.format(data, 1);
- formatter.format(data, 2);
-
- var overallDiv = document.getElementById('overall');
- var overallTable = new google.visualization.Table(overallDiv);
- overallTable.draw(data, {showRowNumber: false});
- }
-
- function drawTimeline() {
- var tData = new google.visualization.DataTable();
- tData.addColumn('date', 'Date');
- tData.addColumn('number', 'Nav');
- tData.addColumn('number', 'Market');
-
- var dailyStats = rawData.daily;
- for (i = 0; i < dailyStats.length; i++) {
- var stat = dailyStats[i];
- var row = [new Date(stat.time), stat.nav, stat.market];
- tData.addRow(row);
- }
-
- var tTimelineDiv = document.getElementById('t_timeline');
- tChart = new google.visualization.AnnotatedTimeLine(tTimelineDiv);
- tChart.draw(tData, {
- 'displayRangeSelector': false,
- 'displayZoomButtons': false,
- 'scaleType': 'allmaximized',
- 'scaleColumns': [0,1],
- });
-
- var bData = new google.visualization.DataTable();
- bData.addColumn('date', 'Date');
- bData.addColumn('number', 'Position Count');
-
- var dailyStats = rawData.daily;
- for (i = 0; i < dailyStats.length; i++) {
- var stat = dailyStats[i];
- var row = [new Date(stat.time), stat.positionCount];
- bData.addRow(row);
- }
-
- var bTimelineDiv = document.getElementById('b_timeline');
- bChart = new google.visualization.AnnotatedTimeLine(bTimelineDiv);
- bChart.draw(bData);
-
- google.visualization.events.addListener(
- bChart, 'rangechange', lineChartRangeChange);
- }
-
- function lineChartRangeChange() {
- var newRange = bChart.getVisibleChartRange();
- tChart.setVisibleChartRange(newRange.start, newRange.end);
- }
- </script>
-
- </body>
-</html>
[26/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by ch...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/66a562c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/66a562c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/66a562c8
Branch: refs/heads/master
Commit: 66a562c8a47522da4e900b4bf7a0aa5b748f81d1
Parents: 0d2e06d 8e96d1b
Author: Donald Szeto <do...@apache.org>
Authored: Fri Aug 4 10:44:29 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Fri Aug 4 10:44:29 2017 -0700
----------------------------------------------------------------------
docs/manual/data/nav/main.yml | 3 ++
.../source/community/contribute-code.html.md | 30 ++++++++++++----
docs/manual/source/resources/release.html.md | 38 ++++++++++++++++++++
3 files changed, 65 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/66a562c8/docs/manual/data/nav/main.yml
----------------------------------------------------------------------
[32/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by ch...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/bcc0afe6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/bcc0afe6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/bcc0afe6
Branch: refs/heads/master
Commit: bcc0afe61538e81c92d741c08a089720b68eb128
Parents: 08ebd6b 9c68d86
Author: Naoki Takezoe <ta...@apache.org>
Authored: Tue Aug 29 14:04:36 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Tue Aug 29 14:04:36 2017 +0900
----------------------------------------------------------------------
docs/manual/source/demo/textclassification.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[09/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/file_random.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/file_random.py b/examples/experimental/scala-local-friend-recommendation/file_random.py
deleted file mode 100644
index 95621e9..0000000
--- a/examples/experimental/scala-local-friend-recommendation/file_random.py
+++ /dev/null
@@ -1,207 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import sys
-import random
-
-read_file = open("data/user_profile.txt", 'r')
-write_file = open("data/mini_user_profile.txt", 'w')
-number_of_lines = int(sys.argv[1])
-number_of_items = int(sys.argv[2])
-
-#record number of lines
-count = 0
-random_num_list = []
-# loop through the file to get number of lines in the file
-for line in read_file:
- count += 1
-
-print "generating random numbers"
-# generating a list of random lines to read from
-for i in range(0, number_of_lines):
- random_num_list.append(random.randint(0, count))
-
-#get rid of any duplicates
-no_duplicate_list = list(set(random_num_list))
-
-#sort the list
-no_duplicate_list.sort()
-#print no_duplicate_list
-
-#go to file begining
-read_file.seek(0)
-count = 0
-index = 0
-user_id_list = []
-print "getting lines from user_profile"
-for line in read_file:
- if count == no_duplicate_list[index]:
- write_file.write(line)
- index += 1
- user_id_list.append(int(line.split()[0]))
- if index == len(no_duplicate_list):
- break
- count += 1
-
-#user_id_list is sorted
-
-user_id_list = map(str, user_id_list)
-user_id_list.sort()
-#print user_id_list
-print "user_id finished"
-
-print "getting lines from item"
-read_file = open("data/item.txt", 'r')
-write_file = open("data/mini_item.txt", 'w')
-count = 0
-random_num_list = []
-for line in read_file:
- count += 1
-
-for i in range(0, number_of_items):
- random_num_list.append(random.randint(0, count))
-
-#no duplicate
-random_num_list = list(set(random_num_list))
-
-random_num_list.sort()
-
-read_file.seek(0)
-count = 0
-index = 0
-item_id_list = []
-for line in read_file:
- if count == random_num_list[index]:
- write_file.write(line)
- index += 1
- item_id_list.append(int(line.split()[0]))
- if index == len(random_num_list):
- break
- count += 1
-print "item finished"
-
-print "getting mini user_key_word"
-read_file = open("data/user_key_word.txt", 'r')
-write_file = open("data/mini_user_key_word.txt", 'w')
-
-#record number of lines
-count = 0
-index = 0
-# loop through the file to get number of lines in the file
-for line in read_file:
- if line.split()[0] == user_id_list[index]:
- write_file.write(line)
- index += 1
- if index == len(user_id_list):
- #print "break"
- break
-print "user keyword finished"
-#go to file begining
-#getting the user_sns_small
-
-print "getting user sns"
-#print user_id_list
-read_file = open("data/user_sns.txt", 'r')
-
-#write_file = open("data/mini_user_sns_small.txt", 'w')
-user_sns_list = []
-index = 0
-met = False
-count = 0
-for line in read_file:
- count += 1
- #print count
- #Same user multiple following
- if met:
- if line.split()[0] != user_id_list[index]:
- index += 1
- met = False
- if index == len(user_id_list):
- break
- if line.split()[0] == user_id_list[index]:
- #print "here"
- user_sns_list.append(line)
- met = True
- # if the current line's user is greater than the user list, that means
- # the user doesn't follow or are following, then we move to next user
- if line.split()[0] > user_id_list[index]:
- index += 1
- if index == len(user_id_list):
- break
-
-#print user_sns_list
-write_file = open("data/mini_user_sns.txt",'w')
-for line in user_sns_list:
- for user_id in user_id_list:
- if line.split()[1] == user_id:
- write_file.write(line)
- break
-print "sns got"
-
-print "getting user action"
-#for line in write_file:
-read_file = open("data/user_action.txt", 'r')
-user_action_list = []
-index = 0
-met = False
-count = 0
-for line in read_file:
- count += 1
- #print count
- if met:
- if line.split()[0] != user_id_list[index]:
- index += 1
- met = False
- if index == len(user_id_list):
- break
- if line.split()[0] == user_id_list[index]:
- #print "here"
- user_action_list.append(line)
- met = True
- if line.split()[0] > user_id_list[index]:
- index += 1
- if index == len(user_id_list):
- break
-#print user_action_list
-write_file = open("data/mini_user_action.txt",'w')
-for line in user_action_list:
- for user_id in user_id_list:
- if line.split()[1] == user_id:
- write_file.write(line)
- break
-print "user action got"
-
-print "getting rec_log_train"
-user_set = set(user_id_list)
-item_set = set(item_id_list)
-read_file = open("data/rec_log_train.txt", 'r')
-write_file = open("data/mini_rec_log_train.txt",'w')
-count = 0
-#for item in item_set:
-# print type(item)
-#for user in user_set:
-# print type(user)
-for line in read_file:
- words = line.split()
-# if words[0] in user_set and (words[1] in user_set or words[1] in item_set):
- if words[0] in user_set and words[1] in item_set:
- write_file.write(line)
- print count
- count += 1
-
-print "Done"
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/keyword_similarity_engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/keyword_similarity_engine.json b/examples/experimental/scala-local-friend-recommendation/keyword_similarity_engine.json
deleted file mode 100644
index f80fe5d..0000000
--- a/examples/experimental/scala-local-friend-recommendation/keyword_similarity_engine.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "id": "org.apache.predictionio.examples.friendrecommendation.keywordsimilarity",
- "version": "0.1",
- "name": "Friend Recommendation Engine with Keyword Similarity Method",
- "engineFactory": "org.apache.predictionio.examples.friendrecommendation.KeywordSimilarityEngineFactory",
- "datasource": {
- "itemFilePath": "data/item.txt",
- "userKeywordFilePath": "data/user_key_word.txt",
- "userActionFilePath": "data/user_action.txt",
- "trainingRecordFilePath": "data/rec_log_train.txt"
- },
- "algorithms": [
- {
- "name": "KeywordSimilarityAlgorithm",
- "params": {
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/project/assembly.sbt b/examples/experimental/scala-local-friend-recommendation/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-local-friend-recommendation/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/random_engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/random_engine.json b/examples/experimental/scala-local-friend-recommendation/random_engine.json
deleted file mode 100644
index 5aba3c3..0000000
--- a/examples/experimental/scala-local-friend-recommendation/random_engine.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "id": "org.apache.predictionio.examples.friendrecommendation.random",
- "version": "0.1",
- "name": "Friend Recommendation Engine with Random Method",
- "engineFactory": "org.apache.predictionio.examples.friendrecommendation.RandomEngineFactory",
- "datasource": {
- "itemFilePath": "data/item.txt",
- "userKeywordFilePath": "data/user_key_word.txt",
- "userActionFilePath": "data/user_action.txt",
- "trainingRecordFilePath": "data/rec_log_train.txt"
- },
- "algorithms": [
- {
- "name": "RandomAlgorithm",
- "params": {
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationAlgoParams.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationAlgoParams.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationAlgoParams.scala
deleted file mode 100644
index 596a3fa..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationAlgoParams.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import org.apache.predictionio.controller._
-
-class FriendRecommendationAlgoParams (
-) extends Params
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSource.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSource.scala
deleted file mode 100644
index 399e31d..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSource.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import org.apache.predictionio.controller._
-import scala.io.Source
-import scala.collection.immutable.HashMap
-
-class FriendRecommendationDataSource (
- val dsp: FriendRecommendationDataSourceParams
-) extends LDataSource[FriendRecommendationTrainingData,
- EmptyEvaluationInfo, FriendRecommendationQuery, EmptyActualResult] {
-
- override
- def readTraining() : FriendRecommendationTrainingData = {
- val (itemIdMap, itemKeyword) = readItem(dsp.itemFilePath)
- val (userIdMap, userKeyword) = readUser(dsp.userKeywordFilePath)
- val adjArray = readRelationship(dsp.userActionFilePath,
- userKeyword.size, userIdMap)
- // Originally for the purpose of training an acceptance threshold
- // Commented out here due to the high time and space complexity of training
- // val trainingRecord = readTrainingRecord(dsp.trainingRecordFilePath,
- // userIdMap, itemIdMap)
- val trainingRecord = null
- new FriendRecommendationTrainingData(userIdMap,
- itemIdMap, userKeyword, itemKeyword, adjArray, trainingRecord)
- }
-
- def readItem(file: String) :
- (HashMap[Int, Int], Array[HashMap[Int, Double]]) = {
- val itemSize = Source.fromFile(file).getLines().size
- val lines = Source.fromFile(file).getLines()
- // An array on Map[keywordId -> weight] values with internal item id index
- val itemKeyword = new Array[HashMap[Int, Double]](itemSize)
- // A map from external id to internal id
- var itemIdMap = new HashMap[Int, Int]()
- var internalId = 0
- lines.foreach{
- line =>
- val data = line.split("\\s")
- itemIdMap += (data(0).toInt -> internalId)
- var keywordMap = new HashMap[Int, Double]()
- data(2).split(";").foreach{
- term =>
- keywordMap += (term.toInt -> 1.0)
- }
- itemKeyword(internalId) = keywordMap
- internalId += 1
- }
- (itemIdMap, itemKeyword)
- }
-
- def readUser(file: String) :
- (HashMap[Int, Int], Array[HashMap[Int, Double]]) = {
- val userSize = Source.fromFile(file).getLines().size
- val lines = Source.fromFile(file).getLines()
- // An array on Map[keywordId -> weight] values with internal item id index
- val userKeyword = new Array[HashMap[Int, Double]](userSize)
- // A map from external id to internal id
- var userIdMap = new HashMap[Int, Int]()
- var internalId = 0
- lines.foreach{
- line =>
- val data = line.split("\\s")
- userIdMap += (data(0).toInt -> internalId)
- var keywordMap = new HashMap[Int, Double]()
- data(1).split(";").foreach{
- termWeight =>
- val termWeightPair = termWeight.split(":")
- keywordMap += (termWeightPair(0).toInt -> termWeightPair(1).toDouble)
- }
- userKeyword(internalId) = keywordMap
- internalId += 1
- }
- (userIdMap, userKeyword)
- }
-
- def readRelationship(file: String,
- userSize: Int, userIdMap: HashMap[Int, Int]) :
- Array[List[(Int, Int)]] = {
- val adjArray = new Array[List[(Int, Int)]](userSize)
- val lines = Source.fromFile(file).getLines()
- lines.foreach{
- line =>
- val data = line.split("\\s").map(s => s.toInt)
- if (userIdMap.contains(data(0)) && userIdMap.contains(data(1))) {
- val srcInternalId = userIdMap(data(0))
- val destInternalId = userIdMap(data(1))
- if (adjArray(srcInternalId) == null) {
- adjArray(srcInternalId) = (destInternalId, data.slice(2,5).sum)::
- List()
- } else {
- adjArray(srcInternalId) = (destInternalId, data.slice(2,5).sum)::
- adjArray(srcInternalId)
- }
- }
- }
- adjArray
- }
-
- def readTrainingRecord(file: String,
- userIdMap: HashMap[Int, Int], itemIdMap: HashMap[Int, Int]) :
- Stream[(Int, Int, Boolean)] = {
- val lines = Source.fromFile(file).getLines()
- var trainingRecord: Stream[(Int, Int, Boolean)] = Stream()
- lines.foreach{
- line =>
- val data = line.split("\\s")
- val userId = userIdMap(data(0).toInt)
- val itemId = itemIdMap(data(1).toInt)
- val result = (data(2).toInt == 1)
- trainingRecord = (userId, itemId, result) #:: trainingRecord
- }
- trainingRecord
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSourceParams.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSourceParams.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSourceParams.scala
deleted file mode 100644
index bd8f6b0..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationDataSourceParams.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import org.apache.predictionio.controller._
-
-class FriendRecommendationDataSourceParams(
- val itemFilePath: String,
- val userKeywordFilePath: String,
- val userActionFilePath: String,
- val trainingRecordFilePath: String
-) extends Params
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationPrediction.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationPrediction.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationPrediction.scala
deleted file mode 100644
index c7e2b8b..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationPrediction.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-class FriendRecommendationPrediction (
- val confidence: Double,
- // returning boolean acceptance to align with KDD 2012 scenario
- val acceptance: Boolean
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationQuery.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationQuery.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationQuery.scala
deleted file mode 100644
index ca2f912..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationQuery.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-class FriendRecommendationQuery (
- // To align with the KDD 2012 scenario
- // Given a user and an item, predict acceptance
- val user: Int,
- val item: Int
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationTrainingData.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationTrainingData.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationTrainingData.scala
deleted file mode 100644
index 2a0596e..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/FriendRecommendationTrainingData.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import scala.collection.immutable.HashMap
-
-class FriendRecommendationTrainingData (
- // Designed to fit random, keyword similarity methods and simrank now
- // Will be updated to to fit more advanced algorithms when they are developed
- // External-internal id map
- val userIdMap: HashMap[Int, Int],
- val itemIdMap: HashMap[Int, Int],
- // Keyword array, internal id index, term-weight map item
- val userKeyword: Array[HashMap[Int, Double]],
- val itemKeyword: Array[HashMap[Int, Double]],
- // User relationship array,
- // src internal id index, dest-internal-id-weight list item
- val socialAction: Array[List[(Int, Int)]],
- // Training record for training purpose
- val trainingRecord: Stream[(Int, Int, Boolean)]
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityAlgorithm.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityAlgorithm.scala
deleted file mode 100644
index 115fa85..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityAlgorithm.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import org.apache.predictionio.controller._
-import scala.collection.immutable.HashMap
-import scala.math
-import scala.io.Source
-
-class KeywordSimilarityAlgorithm (val ap: FriendRecommendationAlgoParams)
- extends LAlgorithm[FriendRecommendationTrainingData,
- KeywordSimilarityModel, FriendRecommendationQuery,
- FriendRecommendationPrediction] {
-
- override
- def train(td: FriendRecommendationTrainingData): KeywordSimilarityModel = {
- var keywordSimWeight = 1.0
- var keywordSimThreshold = 1.0
- // Originally for the purpose of training an acceptance threshold
- // Commented out here due to the high time and space complexity of training
- /*
- td.trainingRecord.foreach{
- record =>
- val sim = findKeywordSimilarity(td.userKeyword(record._1),
- td.itemKeyword(record._2))
- val prediction = (keywordSimWeight * sim - keywordSimThreshold >= 0)
- if (prediction != record._3) {
- val y = if (record._3) 1 else -1
- keywordSimWeight += y * sim
- keywordSimThreshold += y * -1
- }
- }
- */
- new KeywordSimilarityModel(td.userIdMap,
- td.itemIdMap, td.userKeyword, td.itemKeyword,
- keywordSimWeight, keywordSimThreshold)
- }
-
- def findKeywordSimilarity(keywordMap1: HashMap[Int, Double],
- keywordMap2: HashMap[Int, Double]):
- Double = {
- var similarity = 0.0
- keywordMap1.foreach(kw =>
- similarity += kw._2 * keywordMap2.getOrElse(kw._1, 0.0))
- similarity
- }
-
- override
- def predict(model: KeywordSimilarityModel,
- query: FriendRecommendationQuery):
- FriendRecommendationPrediction = {
- // Currently use empty map for unseen users or items
- if (model.userIdMap.contains(query.user) &&
- model.itemIdMap.contains(query.item)) {
- val confidence = findKeywordSimilarity(
- model.userKeyword(model.userIdMap(query.user)),
- model.itemKeyword(model.itemIdMap(query.item)))
- val acceptance = ((confidence * model.keywordSimWeight)
- >= model.keywordSimThreshold)
- new FriendRecommendationPrediction(confidence, acceptance)
- } else {
- val confidence = 0
- val acceptance = ((confidence * model.keywordSimWeight)
- >= model.keywordSimThreshold)
- new FriendRecommendationPrediction(confidence, acceptance)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityEngineFactory.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityEngineFactory.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityEngineFactory.scala
deleted file mode 100644
index 51023a4..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityEngineFactory.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import org.apache.predictionio.controller._
-
-object KeywordSimilarityEngineFactory extends IEngineFactory {
- override
- def apply() = {
- new Engine(
- classOf[FriendRecommendationDataSource],
- classOf[LIdentityPreparator[FriendRecommendationTrainingData]],
- Map("KeywordSimilarityAlgorithm" -> classOf[KeywordSimilarityAlgorithm]),
- classOf[LFirstServing[FriendRecommendationQuery,
- FriendRecommendationPrediction]]
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityModel.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityModel.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityModel.scala
deleted file mode 100644
index 6de7ee3..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/KeywordSimilarityModel.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import scala.collection.immutable.HashMap
-
-class KeywordSimilarityModel (
- // External-internal id map
- val userIdMap: HashMap[Int, Int],
- val itemIdMap: HashMap[Int, Int],
- // Keyword array, internal id index, term-weight map item
- val userKeyword: Array[HashMap[Int, Double]],
- val itemKeyword: Array[HashMap[Int, Double]],
- // Weight and threshold trained
- val keywordSimWeight: Double,
- val keywordSimThreshold: Double
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomAlgorithm.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomAlgorithm.scala
deleted file mode 100644
index a71913e..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomAlgorithm.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import org.apache.predictionio.controller._
-
-// For random algorithm
-import scala.util.Random
-
-class RandomAlgorithm (val ap: FriendRecommendationAlgoParams)
- extends LAlgorithm[FriendRecommendationTrainingData,
- RandomModel, FriendRecommendationQuery, FriendRecommendationPrediction] {
-
- override
- def train(pd: FriendRecommendationTrainingData): RandomModel = {
- new RandomModel(0.5)
- }
-
- override
- def predict(model: RandomModel, query: FriendRecommendationQuery):
- FriendRecommendationPrediction = {
- val randomConfidence = Random.nextDouble
- val acceptance = randomConfidence >= model.randomThreshold
- new FriendRecommendationPrediction(randomConfidence, acceptance)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomEngineFactory.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomEngineFactory.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomEngineFactory.scala
deleted file mode 100644
index e650199..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomEngineFactory.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-import org.apache.predictionio.controller._
-
-object RandomEngineFactory extends IEngineFactory {
- override
- def apply() = {
- new Engine(
- classOf[FriendRecommendationDataSource],
- classOf[LIdentityPreparator[FriendRecommendationTrainingData]],
- Map("RandomAlgorithm" -> classOf[RandomAlgorithm]),
- classOf[LFirstServing[FriendRecommendationQuery,
- FriendRecommendationPrediction]]
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomModel.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomModel.scala b/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomModel.scala
deleted file mode 100644
index 91a8866..0000000
--- a/examples/experimental/scala-local-friend-recommendation/src/main/scala/RandomModel.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.friendrecommendation
-
-class RandomModel(
- val randomThreshold: Double
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-helloworld/HelloWorld.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-helloworld/HelloWorld.scala b/examples/experimental/scala-local-helloworld/HelloWorld.scala
deleted file mode 100644
index 6ec4f0a..0000000
--- a/examples/experimental/scala-local-helloworld/HelloWorld.scala
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.helloworld
-
-import org.apache.predictionio.controller._
-
-import scala.io.Source
-import scala.collection.immutable.HashMap
-
-// all data need to be serializable
-class MyTrainingData(
- // list of (day, temperature) tuples
- val temperatures: List[(String, Double)]
-) extends Serializable
-
-class MyQuery(
- val day: String
-) extends Serializable
-
-class MyModel(
- val temperatures: HashMap[String, Double]
-) extends Serializable {
- override def toString = temperatures.toString
-}
-
-class MyPredictedResult(
- val temperature: Double
-) extends Serializable
-
-case class MyDataSourceParams(val multiplier: Int
- ) extends Params
-
-class MyDataSource extends LDataSource[
- MyTrainingData,
- EmptyEvaluationInfo,
- MyQuery,
- EmptyActualResult] {
-
- /* override this to return Training Data only */
-
- override
- def readTraining(): MyTrainingData = {
- val lines = Source.fromFile("../data/helloworld/data.csv").getLines()
- .toList.map{ line =>
- val data = line.split(",")
- (data(0), data(1).toDouble)
- }
-
- new MyTrainingData(lines)
- }
-}
-
-class MyAlgorithm extends LAlgorithm[
- MyTrainingData,
- MyModel,
- MyQuery,
- MyPredictedResult] {
-
-
- override
- def train(pd: MyTrainingData): MyModel = {
- // calculate average value of each day
- val average = pd.temperatures
- .groupBy(_._1) // group by day
- .mapValues{ list =>
- val tempList = list.map(_._2) // get the temperature
- tempList.sum / tempList.size
- }
-
- // trait Map is not serializable, use concrete class HashMap
- new MyModel(HashMap[String, Double]() ++ average)
- }
-
- override
- def predict(model: MyModel, query: MyQuery): MyPredictedResult = {
- val temp = model.temperatures(query.day)
- new MyPredictedResult(temp)
- }
-}
-
-// factory
-object MyEngineFactory extends IEngineFactory {
- override
- def apply() = {
- /* SimpleEngine only requires one DataSouce and one Algorithm */
- new SimpleEngine(
- classOf[MyDataSource],
- classOf[MyAlgorithm]
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-helloworld/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-helloworld/README.md b/examples/experimental/scala-local-helloworld/README.md
deleted file mode 100644
index 4a8310b..0000000
--- a/examples/experimental/scala-local-helloworld/README.md
+++ /dev/null
@@ -1,92 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# My First "Hello World" Engine
-
-Prepare training data:
-```
-$ cd $PIO_HOME/examples/scala-local-helloworld
-$ cp ../data/helloworld/data1.csv ../data/helloworld/data.csv
-```
-
-Build engine:
-
-```
-$ ../../bin/pio build
-```
-
-Train:
-
-```
-$ ../../bin/pio train
-```
-
-Example output:
-
-```
-2014-08-05 17:06:02,638 INFO APIDebugWorkflow$ - Metrics is null. Stop here
-2014-08-05 17:06:02,769 INFO APIDebugWorkflow$ - Run information saved with ID: 201408050005
-```
-
-Deploy:
-
-```
-$ ../../bin/pio deploy
-```
-
-Retrieve prediction:
-
-```
-$ curl -H "Content-Type: application/json" -d '{ "day": "Mon" }' http://localhost:8000/queries.json
-```
-
-Output:
-
-```
-{"temperature":75.5}
-```
-
-Retrieve prediction:
-
-```
-$ curl -H "Content-Type: application/json" -d '{ "day": "Tue" }' http://localhost:8000/queries.json
-```
-
-Output:
-```
-{"temperature":80.5}
-```
-
-## 4. Re-training
-
-Re-train with new data:
-
-```
-$ cd $PIO_HOME/examples/scala-local-helloworld
-$ cp ../data/helloworld/data2.csv ../data/helloworld/data.csv
-```
-
-```
-$ ../../bin/pio train
-$ ../../bin/pio deploy
-```
-
-```
-$ curl -H "Content-Type: application/json" -d '{ "day": "Mon" }' http://localhost:8000/queries.json
-
-{"temperature":76.66666666666667}
-```
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-helloworld/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-helloworld/build.sbt b/examples/experimental/scala-local-helloworld/build.sbt
deleted file mode 100644
index e14c05b..0000000
--- a/examples/experimental/scala-local-helloworld/build.sbt
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "example-scala-local-helloworld"
-
-organization := "org.sample"
-
-libraryDependencies ++= Seq(
- "org.slf4j" % "slf4j-api" % "1.6.1",
- "org.apache.predictionio" %% "apache-predictionio-data" % "0.10.0-incubating",
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.10.0-incubating",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-helloworld/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-helloworld/engine.json b/examples/experimental/scala-local-helloworld/engine.json
deleted file mode 100644
index 6dd2f74..0000000
--- a/examples/experimental/scala-local-helloworld/engine.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "id": "default",
- "description": "My Hello World Engine",
- "engineFactory": "org.sample.helloworld.MyEngineFactory"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-helloworld/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-helloworld/project/assembly.sbt b/examples/experimental/scala-local-helloworld/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-local-helloworld/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-evaluation/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-evaluation/build.sbt b/examples/experimental/scala-local-movielens-evaluation/build.sbt
deleted file mode 100644
index ea04235..0000000
--- a/examples/experimental/scala-local-movielens-evaluation/build.sbt
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "scala-local-movielens-evaluation"
-
-organization := "myorg"
-
-version := "0.0.1-SNAPSHOT"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.predictionio" %% "engines" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-evaluation/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-evaluation/engine.json b/examples/experimental/scala-local-movielens-evaluation/engine.json
deleted file mode 100644
index e11e5de..0000000
--- a/examples/experimental/scala-local-movielens-evaluation/engine.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "id": "scala-local-movielens-evaluation",
- "description": "scala-local-movielens-evaluation",
- "engineFactory": "myorg.MyEngineFactory"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-evaluation/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-evaluation/project/assembly.sbt b/examples/experimental/scala-local-movielens-evaluation/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-local-movielens-evaluation/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-evaluation/src/main/scala/Evaluation.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-evaluation/src/main/scala/Evaluation.scala b/examples/experimental/scala-local-movielens-evaluation/src/main/scala/Evaluation.scala
deleted file mode 100644
index d972fbe..0000000
--- a/examples/experimental/scala-local-movielens-evaluation/src/main/scala/Evaluation.scala
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.mlc
-
-import org.apache.predictionio.engines.itemrank.PreparatorParams
-import org.apache.predictionio.engines.itemrank.EventsDataSourceParams
-import org.apache.predictionio.engines.itemrank.ItemRankEngine
-import org.apache.predictionio.engines.itemrank.ItemRankDetailedEvaluator
-import org.apache.predictionio.engines.itemrank.DetailedEvaluatorParams
-import org.apache.predictionio.engines.itemrank.MeasureType
-import org.apache.predictionio.engines.itemrank.mahout.ItemBasedAlgoParams
-import org.apache.predictionio.engines.base.AttributeNames
-import org.apache.predictionio.engines.base.EventsSlidingEvalParams
-import org.apache.predictionio.engines.base.BinaryRatingParams
-import org.apache.predictionio.controller.WorkflowParams
-import org.apache.predictionio.controller.Workflow
-import org.apache.predictionio.controller.EngineParams
-
-import com.github.nscala_time.time.Imports._
-
-object CommonParams {
- val DataSourceAttributeNames = AttributeNames(
- user = "pio_user",
- item = "pio_item",
- u2iActions = Set("rate"),
- itypes = "pio_itypes",
- starttime = "pio_starttime",
- endtime = "pio_endtime",
- inactive = "pio_inactive",
- rating = "pio_rating")
-
- val PreparatorParams = new PreparatorParams(
- actions = Map("rate" -> None),
- conflict = "latest")
-
- val MahoutAlgoParams0 = new ItemBasedAlgoParams(
- booleanData = true,
- itemSimilarity = "LogLikelihoodSimilarity",
- weighted = false,
- nearestN = 10,
- threshold = 4.9E-324,
- numSimilarItems = 50,
- numUserActions = 50,
- freshness = 0,
- freshnessTimeUnit = 86400,
- recommendationTime = Some(DateTime.now.millis))
-
- val CompleteDataSourceParams = EventsDataSourceParams(
- appId = 9,
- actions = Set("rate"),
- attributeNames = CommonParams.DataSourceAttributeNames,
- slidingEval = Some(new EventsSlidingEvalParams(
- firstTrainingUntilTime = new DateTime(1998, 2, 1, 0, 0),
- evalDuration = Duration.standardDays(7),
- evalCount = 12)))
-}
-
-object Evaluation1 {
- def main(args: Array[String]) {
- // Engine Settings
- val engine = ItemRankEngine()
-
- val dsp = EventsDataSourceParams(
- appId = 9,
- actions = Set("rate"),
- attributeNames = CommonParams.DataSourceAttributeNames,
- slidingEval = Some(new EventsSlidingEvalParams(
- firstTrainingUntilTime = new DateTime(1998, 2, 1, 0, 0),
- evalDuration = Duration.standardDays(7),
- evalCount = 3))
- )
-
- val engineParams = new EngineParams(
- dataSourceParams = dsp,
- preparatorParams = CommonParams.PreparatorParams,
- algorithmParamsList = Seq(
- ("mahoutItemBased", CommonParams.MahoutAlgoParams0))
- )
-
- // Evaluator Setting
- val evaluatorParams = new DetailedEvaluatorParams(
- ratingParams = new BinaryRatingParams(
- actionsMap = Map("rate" -> None),
- goodThreshold = 3),
- measureType = MeasureType.PrecisionAtK,
- measureK = 10
- )
-
- // Run
- Workflow.runEngine(
- params = WorkflowParams(batch = "MLC: Evaluation1"),
- engine = engine,
- engineParams = engineParams,
- evaluatorClassOpt = Some(classOf[ItemRankDetailedEvaluator]),
- evaluatorParams = evaluatorParams
- )
- }
-}
-
-object Evaluation2 {
- def main(args: Array[String]) {
- // Engine Settings
- val engine = ItemRankEngine()
-
- val engineParams = new EngineParams(
- dataSourceParams = CommonParams.CompleteDataSourceParams,
- preparatorParams = CommonParams.PreparatorParams,
- algorithmParamsList = Seq(
- ("mahoutItemBased", CommonParams.MahoutAlgoParams0))
- )
-
- // Evaluator Setting
- val evaluatorParams = new DetailedEvaluatorParams(
- ratingParams = new BinaryRatingParams(
- actionsMap = Map("rate" -> None),
- goodThreshold = 3),
- measureType = MeasureType.PrecisionAtK,
- measureK = 10
- )
-
- // Run
- Workflow.runEngine(
- params = WorkflowParams(batch = "MLC: Evaluation2"),
- engine = engine,
- engineParams = engineParams,
- evaluatorClassOpt = Some(classOf[ItemRankDetailedEvaluator]),
- evaluatorParams = evaluatorParams
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-evaluation/src/main/scala/ItemRecEvaluation.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-evaluation/src/main/scala/ItemRecEvaluation.scala b/examples/experimental/scala-local-movielens-evaluation/src/main/scala/ItemRecEvaluation.scala
deleted file mode 100644
index a26f2de..0000000
--- a/examples/experimental/scala-local-movielens-evaluation/src/main/scala/ItemRecEvaluation.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.mlc
-
-import org.apache.predictionio.engines.itemrec.ItemRecEngine
-import org.apache.predictionio.engines.itemrec.EventsDataSourceParams
-import org.apache.predictionio.engines.itemrec.PreparatorParams
-import org.apache.predictionio.engines.itemrec.NCItemBasedAlgorithmParams
-import org.apache.predictionio.engines.itemrec.EvalParams
-import org.apache.predictionio.engines.itemrec.ItemRecEvaluator
-import org.apache.predictionio.engines.itemrec.ItemRecEvaluatorParams
-import org.apache.predictionio.engines.itemrec.MeasureType
-import org.apache.predictionio.engines.base.EventsSlidingEvalParams
-import org.apache.predictionio.engines.base.BinaryRatingParams
-
-import org.apache.predictionio.controller.EngineParams
-import org.apache.predictionio.controller.Workflow
-import org.apache.predictionio.controller.WorkflowParams
-
-import com.github.nscala_time.time.Imports._
-
-// Recommend to run with "--driver-memory 2G"
-object ItemRecEvaluation1 {
- def main(args: Array[String]) {
- val engine = ItemRecEngine()
-
- val dsp = EventsDataSourceParams(
- appId = 9,
- actions = Set("rate"),
- attributeNames = CommonParams.DataSourceAttributeNames,
- slidingEval = Some(new EventsSlidingEvalParams(
- firstTrainingUntilTime = new DateTime(1998, 2, 1, 0, 0),
- evalDuration = Duration.standardDays(7),
- evalCount = 12)),
- //evalCount = 3)),
- evalParams = Some(new EvalParams(queryN = 10))
- )
-
- val pp = new PreparatorParams(
- actions = Map("rate" -> None),
- seenActions = Set("rate"),
- conflict = "latest")
-
- val ncMahoutAlgoParams = new NCItemBasedAlgorithmParams(
- booleanData = true,
- itemSimilarity = "LogLikelihoodSimilarity",
- weighted = false,
- threshold = 4.9E-324,
- nearestN = 10,
- unseenOnly = false,
- freshness = 0,
- freshnessTimeUnit = 86400)
-
- val engineParams = new EngineParams(
- dataSourceParams = dsp,
- preparatorParams = pp,
- algorithmParamsList = Seq(
- ("ncMahoutItemBased", ncMahoutAlgoParams)))
-
- val evaluatorParams = new ItemRecEvaluatorParams(
- ratingParams = new BinaryRatingParams(
- actionsMap = Map("rate" -> None),
- goodThreshold = 3),
- measureType = MeasureType.PrecisionAtK,
- measureK = 10
- )
-
- Workflow.runEngine(
- params = WorkflowParams(batch = "MLC: ItemRec Evaluation1", verbose = 0),
- engine = engine,
- engineParams = engineParams,
- evaluatorClassOpt = Some(classOf[ItemRecEvaluator]),
- evaluatorParams = evaluatorParams
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-filtering/blacklisted.txt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-filtering/blacklisted.txt b/examples/experimental/scala-local-movielens-filtering/blacklisted.txt
deleted file mode 100644
index 2d959ca..0000000
--- a/examples/experimental/scala-local-movielens-filtering/blacklisted.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-272
-123
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-filtering/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-filtering/build.sbt b/examples/experimental/scala-local-movielens-filtering/build.sbt
deleted file mode 100644
index 699c90c..0000000
--- a/examples/experimental/scala-local-movielens-filtering/build.sbt
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "scala-local-movielens-filtering"
-
-organization := "myorg"
-
-version := "0.0.1-SNAPSHOT"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.predictionio" %% "engines" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-filtering/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-filtering/engine.json b/examples/experimental/scala-local-movielens-filtering/engine.json
deleted file mode 100644
index 4938c15..0000000
--- a/examples/experimental/scala-local-movielens-filtering/engine.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "id": "scala-local-movielens-filtering",
- "description": "scala-local-movielens-filtering",
- "engineFactory": "myorg.TempFilterEngine",
- "datasource": {
- "params": {
- "appId": YOUR_APP_ID,
- "actions": [
- "view",
- "like",
- "dislike",
- "conversion",
- "rate"
- ],
- "attributeNames": {
- "user" : "pio_user",
- "item" : "pio_item",
- "u2iActions": [
- "view",
- "like",
- "dislike",
- "conversion",
- "rate"
- ],
- "itypes" : "pio_itypes",
- "starttime" : "pio_starttime",
- "endtime" : "pio_endtime",
- "inactive" : "pio_inactive",
- "rating" : "pio_rating"
- }
- }
- },
- "preparator": {
- "params": {
- "actions": {
- "view": 3,
- "like": 5,
- "dislike": 1,
- "conversion": 4,
- "rate": null
- },
- "conflict": "latest"
- }
- },
- "algorithms": [
- {
- "name": "ncMahoutItemBased",
- "params": {
- "booleanData": true,
- "itemSimilarity": "LogLikelihoodSimilarity",
- "weighted": false,
- "threshold": 4.9E-324,
- "nearestN": 10,
- "unseenOnly": false,
- "freshness" : 0,
- "freshnessTimeUnit" : 86400
- }
- }
- ],
- "serving": {
- "params": {
- "filepath": FULL_PATH_TO_BLACKLISTED_FILE
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-filtering/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-filtering/project/assembly.sbt b/examples/experimental/scala-local-movielens-filtering/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-local-movielens-filtering/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-filtering/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-filtering/src/main/scala/Engine.scala b/examples/experimental/scala-local-movielens-filtering/src/main/scala/Engine.scala
deleted file mode 100644
index 1710661..0000000
--- a/examples/experimental/scala-local-movielens-filtering/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package myorg
-
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.engines.itemrec.EventsDataSource
-import org.apache.predictionio.engines.itemrec.ItemRecPreparator
-import org.apache.predictionio.engines.itemrec.NCItemBasedAlgorithm
-
-object TempFilterEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[EventsDataSource],
- classOf[ItemRecPreparator],
- Map("ncMahoutItemBased" -> classOf[NCItemBasedAlgorithm]),
- classOf[TempFilter]
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-movielens-filtering/src/main/scala/Filtering.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-movielens-filtering/src/main/scala/Filtering.scala b/examples/experimental/scala-local-movielens-filtering/src/main/scala/Filtering.scala
deleted file mode 100644
index 83f5866..0000000
--- a/examples/experimental/scala-local-movielens-filtering/src/main/scala/Filtering.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package myorg
-
-import org.apache.predictionio.controller.LServing
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.engines.itemrec.Prediction
-import org.apache.predictionio.engines.itemrec.Query
-import scala.io.Source
-
-case class TempFilterParams(val filepath: String) extends Params
-
-class TempFilter(val params: TempFilterParams)
- extends LServing[TempFilterParams, Query, Prediction] {
- override def serve(query: Query, predictions: Seq[Prediction]): Prediction = {
- val disabledIids: Set[String] = Source.fromFile(params.filepath)
- .getLines()
- .toSet
-
- val prediction = predictions.head
- // prediction.items is a list of (item_id, score)-tuple
- prediction.copy(items = prediction.items.filter(e => !disabledIids(e._1)))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-regression/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-regression/README.md b/examples/experimental/scala-local-regression/README.md
deleted file mode 100644
index 5c0dafe..0000000
--- a/examples/experimental/scala-local-regression/README.md
+++ /dev/null
@@ -1,165 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-Linear Regression Engine
-========================
-
-This document describes a Scala-based single-machine linear regression engine.
-
-
-Prerequisite
-------------
-
-Make sure you have built PredictionIO and setup storage described
-[here](/README.md).
-
-
-High Level Description
-----------------------
-
-This engine demonstrates how one can simply wrap around the
-[Nak](https://github.com/scalanlp/nak) library to train a linear regression
-model and serve real-time predictions.
-
-All code definition can be found [here](Run.scala).
-
-
-### Data Source
-
-Training data is located at `/examples/data/lr_data.txt`. The first column are
-values of the dependent variable, and the rest are values of explanatory
-variables. In this example, they are represented by the `TrainingData` case
-class as a vector of double (all rows of the first column), and a vector of
-vector of double (all rows of the remaining columns) respectively.
-
-
-### Preparator
-
-The preparator in this example accepts two parameters: `n` and `k`. Each row of
-data is indexed by `index` starting from 0. When `n > 0`, rows matching `index
-mod n = k` will be dropped.
-
-
-### Algorithm
-
-This example engine contains one single algorithm that wraps around the Nak
-library's linear regression routine. The `train()` method simply massage the
-`TrainingData` into a form that can be used by Nak.
-
-
-### Serving
-
-This example engine uses `FirstServing`, which serves only predictions from the
-first algorithm. Since there is only one algorithm in this engine, predictions
-from the linear regression algorithm will be served.
-
-
-Training a Model
-----------------
-
-This example provides a set of ready-to-use parameters for each component
-mentioned in the previous section. They are located inside the `params`
-subdirectory.
-
-Before training, you must let PredictionIO know about the engine. Run the
-following command to build and register the engine.
-```
-$ cd $PIO_HOME/examples/scala-local-regression
-$ ../../bin/pio build
-```
-where `$PIO_HOME` is the root directory of the PredictionIO code tree.
-
-To start training, use the following command.
-```
-$ cd $PIO_HOME/examples/scala-local-regression
-$ ../../bin/pio train
-```
-This will train a model and save it in PredictionIO's metadata storage. Notice
-that when the run is completed, it will display a run ID, like below.
-```
-2014-08-08 17:18:09,399 INFO SparkContext - Job finished: collect at DebugWorkflow.scala:571, took 0.046796 s
-2014-08-08 17:18:09,399 INFO APIDebugWorkflow$ - Metrics is null. Stop here
-2014-08-08 17:18:09,498 INFO APIDebugWorkflow$ - Saved engine instance with ID: CHURP-cvQta5VKxorx_9Aw
-```
-
-
-Running Evaluation Metrics
---------------------------
-
-To run evaluation metrics, use the following command.
-```
-$ cd $PIO_HOME/examples/scala-local-regression
-$ ../../bin/pio eval --metrics-class org.apache.predictionio.controller.MeanSquareError
-```
-Notice the extra required argument `--metrics-class
-org.apache.predictionio.controller.MeanSquareError` for the `eval` command. This instructs
-PredictionIO to run the specified metrics during evaluation. When you look at
-the console output again, you should be able to see a mean square error
-computed, like the following.
-```
-2014-08-08 17:21:01,042 INFO APIDebugWorkflow$ - Set: The One Size: 1000 MSE: 0.092519
-2014-08-08 17:21:01,042 INFO APIDebugWorkflow$ - APIDebugWorkflow.run completed.
-2014-08-08 17:21:01,140 INFO APIDebugWorkflow$ - Saved engine instance with ID: icfEp9njR76NQOrvowC-dQ
-```
-
-
-Deploying a Real-time Prediction Server
----------------------------------------
-
-Following from instructions above, you should have trained a model. Use the
-following command to start a server.
-```
-$ cd $PIO_HOME/examples/scala-local-regression
-$ ../../bin/pio deploy
-```
-This will create a server that by default binds to http://localhost:8000. You
-can visit that page in your web browser to check its status.
-
-To perform real-time predictions, try the following.
-```
-$ curl -H "Content-Type: application/json" -d '[2.1419053154730548, 1.919407948982788, 0.0501333631091041, -0.10699028639933772, 1.2809776380727795, 1.6846227956326554, 0.18277859260127316, -0.39664340267804343, 0.8090554869291249, 2.48621339239065]' http://localhost:8000/queries.json
-$ curl -H "Content-Type: application/json" -d '[-0.8600615539670898, -1.0084357652346345, -1.3088407119560064, -1.9340485539299312, -0.6246990990796732, -2.325746651211032, -0.28429904752434976, -0.1272785164794058, -1.3787859877532718, -0.24374419289538318]' http://localhost:8000/queries.json
-```
-Congratulations! You have just trained a linear regression model and is able to
-perform real time prediction.
-
-
-Production Prediction Server Deployment
----------------------------------------
-
-Prediction servers support reloading models on the fly with the latest completed
-run.
-
-1. Assuming you already have a running prediction server from the previous
- section, go to http://localhost:8000 to check its status. Take note of the
- **Run ID** at the top.
-
-2. Run training and deploy again.
-
- ```
- $ cd $PIO_HOME/examples/scala-local-regression
- $ ../../bin/pio train
- $ ../../bin/pio deploy
- ```
-
-3. Refresh the page at http://localhost:8000, you should see the prediction
- server status page with a new **Run ID** at the top.
-
-Congratulations! You have just experienced a production-ready setup that can
-reload itself automatically after every training! Simply add the training or
-evaluation command to your *crontab*, and your setup will be able to re-deploy
-itself automatically in a regular interval.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-regression/Run.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-regression/Run.scala b/examples/experimental/scala-local-regression/Run.scala
deleted file mode 100644
index d9d3380..0000000
--- a/examples/experimental/scala-local-regression/Run.scala
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.regression.local
-
-import org.apache.predictionio.controller.EmptyParams
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.EngineParams
-import org.apache.predictionio.controller.LFirstServing
-import org.apache.predictionio.controller.LAlgorithm
-import org.apache.predictionio.controller.LDataSource
-import org.apache.predictionio.controller.LPreparator
-import org.apache.predictionio.controller.MeanSquareError
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.Utils
-import org.apache.predictionio.controller.Workflow
-import org.apache.predictionio.controller.WorkflowParams
-
-import breeze.linalg.DenseMatrix
-import breeze.linalg.DenseVector
-import breeze.linalg.inv
-import nak.regress.LinearRegression
-import org.json4s._
-
-import scala.io.Source
-import java.io.File
-
-case class DataSourceParams(val filepath: String, val seed: Int = 9527)
- extends Params
-
-case class TrainingData(x: Vector[Vector[Double]], y: Vector[Double]) {
- val r = x.length
- val c = x.head.length
-}
-
-case class LocalDataSource(val dsp: DataSourceParams)
- extends LDataSource[
- DataSourceParams, String, TrainingData, Vector[Double], Double] {
- override
- def read(): Seq[(String, TrainingData, Seq[(Vector[Double], Double)])] = {
- val lines = Source.fromFile(dsp.filepath).getLines
- .toSeq.map(_.split(" ", 2))
-
- // FIXME: Use different training / testing data.
- val x = lines.map{ _(1).split(' ').map{_.toDouble} }.map{ e => Vector(e:_*)}
- val y = lines.map{ _(0).toDouble }
-
- val td = TrainingData(Vector(x:_*), Vector(y:_*))
-
- val oneData = ("The One", td, x.zip(y))
- return Seq(oneData)
- }
-}
-
-// When n = 0, don't drop data
-// When n > 0, drop data when index mod n == k
-case class PreparatorParams(n: Int = 0, k: Int = 0) extends Params
-
-case class LocalPreparator(val pp: PreparatorParams = PreparatorParams())
- extends LPreparator[PreparatorParams, TrainingData, TrainingData] {
- def prepare(td: TrainingData): TrainingData = {
- val xyi: Vector[(Vector[Double], Double)] = td.x.zip(td.y)
- .zipWithIndex
- .filter{ e => (e._2 % pp.n) != pp.k}
- .map{ e => (e._1._1, e._1._2) }
- TrainingData(xyi.map(_._1), xyi.map(_._2))
- }
-}
-
-case class LocalAlgorithm()
- extends LAlgorithm[
- EmptyParams, TrainingData, Array[Double], Vector[Double], Double] {
-
- def train(td: TrainingData): Array[Double] = {
- val xArray: Array[Double] = td.x.foldLeft(Vector[Double]())(_ ++ _).toArray
- // DenseMatrix.create fills first column, then second.
- val m = DenseMatrix.create[Double](td.c, td.r, xArray).t
- val y = DenseVector[Double](td.y.toArray)
- val result = LinearRegression.regress(m, y)
- return result.data.toArray
- }
-
- def predict(model: Array[Double], query: Vector[Double]) = {
- model.zip(query).map(e => e._1 * e._2).sum
- }
-
- @transient override lazy val querySerializer =
- Utils.json4sDefaultFormats + new VectorSerializer
-}
-
-class VectorSerializer extends CustomSerializer[Vector[Double]](format => (
- {
- case JArray(s) =>
- s.map {
- case JDouble(x) => x
- case _ => 0
- }.toVector
- },
- {
- case x: Vector[Double] =>
- JArray(x.toList.map(y => JDouble(y)))
- }
-))
-
-object RegressionEngineFactory extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[LocalDataSource],
- classOf[LocalPreparator],
- Map("" -> classOf[LocalAlgorithm]),
- classOf[LFirstServing[Vector[Double], Double]])
- }
-}
-
-object Run {
- val workflowParams = WorkflowParams(
- batch = "Imagine: Local Regression",
- verbose = 3,
- saveModel = true)
-
- def runComponents() {
- val filepath = new File("../data/lr_data.txt").getCanonicalPath
- val dataSourceParams = new DataSourceParams(filepath)
- val preparatorParams = new PreparatorParams(n = 2, k = 0)
-
- Workflow.run(
- params = workflowParams,
- dataSourceClassOpt = Some(classOf[LocalDataSource]),
- dataSourceParams = dataSourceParams,
- preparatorClassOpt = Some(classOf[LocalPreparator]),
- preparatorParams = preparatorParams,
- algorithmClassMapOpt = Some(Map("" -> classOf[LocalAlgorithm])),
- algorithmParamsList = Seq(
- ("", EmptyParams())),
- servingClassOpt = Some(classOf[LFirstServing[Vector[Double], Double]]),
- evaluatorClassOpt = Some(classOf[MeanSquareError]))
- }
-
- def runEngine() {
- val filepath = new File("../data/lr_data.txt").getCanonicalPath
- val engine = RegressionEngineFactory()
- val engineParams = new EngineParams(
- dataSourceParams = DataSourceParams(filepath),
- preparatorParams = PreparatorParams(n = 2, k = 0),
- algorithmParamsList = Seq(("", EmptyParams())))
-
- Workflow.runEngine(
- params = workflowParams,
- engine = engine,
- engineParams = engineParams,
- evaluatorClassOpt = Some(classOf[MeanSquareError]))
- }
-
- def main(args: Array[String]) {
- runEngine()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-regression/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-regression/build.sbt b/examples/experimental/scala-local-regression/build.sbt
deleted file mode 100644
index 4db53c6..0000000
--- a/examples/experimental/scala-local-regression/build.sbt
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "example-scala-local-regression"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.json4s" %% "json4s-native" % "3.2.10",
- "org.scalanlp" %% "nak" % "1.3")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-regression/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-regression/engine.json b/examples/experimental/scala-local-regression/engine.json
deleted file mode 100644
index eaa8f14..0000000
--- a/examples/experimental/scala-local-regression/engine.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.apache.predictionio.examples.regression.local.RegressionEngineFactory",
- "datasource": {
- "params": {
- "filepath": "../data/lr_data.txt",
- "seed": 1337
- }
- },
- "preparator": {
- "params": {
- "n": 2,
- "k": 0
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-regression/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-regression/project/assembly.sbt b/examples/experimental/scala-local-regression/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-local-regression/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
[25/50] [abbrv] incubator-predictionio git commit: [PIO-111]
Documentation for `pio batchpredict`
Posted by ch...@apache.org.
[PIO-111] Documentation for `pio batchpredict`
Closes #418
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/0d2e06d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/0d2e06d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/0d2e06d5
Branch: refs/heads/master
Commit: 0d2e06d54524946ecc1ebfed085f58f36c1100e7
Parents: f752690
Author: Mars Hall <ma...@heroku.com>
Authored: Thu Aug 3 11:49:27 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Aug 3 11:49:27 2017 -0700
----------------------------------------------------------------------
docs/manual/Gemfile.lock | 8 +-
docs/manual/data/nav/main.yml | 3 +
docs/manual/source/batchpredict/index.html.md | 148 +++++++++++++++++++++
docs/manual/source/cli/index.html.md | 7 +-
4 files changed, 161 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/0d2e06d5/docs/manual/Gemfile.lock
----------------------------------------------------------------------
diff --git a/docs/manual/Gemfile.lock b/docs/manual/Gemfile.lock
index 947840f..ecf6194 100644
--- a/docs/manual/Gemfile.lock
+++ b/docs/manual/Gemfile.lock
@@ -91,7 +91,7 @@ GEM
kramdown (1.6.0)
launchy (2.4.3)
addressable (~> 2.3)
- libv8 (3.16.14.13)
+ libv8 (3.16.14.19)
listen (2.10.0)
celluloid (~> 0.16.0)
rb-fsevent (>= 0.9.3)
@@ -200,8 +200,8 @@ GEM
sprockets (~> 2.0)
tilt (~> 1.1)
temple (0.7.6)
- therubyracer (0.12.2)
- libv8 (~> 3.16.14.0)
+ therubyracer (0.12.3)
+ libv8 (~> 3.16.14.15)
ref
thor (0.19.1)
thread_safe (0.3.5)
@@ -253,4 +253,4 @@ DEPENDENCIES
wdm (~> 0.1.0)
BUNDLED WITH
- 1.11.2
+ 1.15.3
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/0d2e06d5/docs/manual/data/nav/main.yml
----------------------------------------------------------------------
diff --git a/docs/manual/data/nav/main.yml b/docs/manual/data/nav/main.yml
index 245fbaf..d80d550 100644
--- a/docs/manual/data/nav/main.yml
+++ b/docs/manual/data/nav/main.yml
@@ -62,6 +62,9 @@ root:
body: 'Engine Command-line Interface'
url: '/cli/#engine-commands'
-
+ body: 'Batch Predictions'
+ url: '/batchpredict/'
+ -
body: 'Monitoring Engine'
url: '/deploy/monitoring/'
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/0d2e06d5/docs/manual/source/batchpredict/index.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/batchpredict/index.html.md b/docs/manual/source/batchpredict/index.html.md
new file mode 100644
index 0000000..38ddb3b
--- /dev/null
+++ b/docs/manual/source/batchpredict/index.html.md
@@ -0,0 +1,148 @@
+---
+title: Batch Predictions
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+##Overview
+Process predictions for many queries using efficient parallelization
+through Spark. Useful for mass auditing of predictions and for
+generating predictions to push into other systems.
+
+Batch predict reads and writes multi-object JSON files similar to the
+[batch import](/datacollection/batchimport/) format. JSON objects are separated
+by newlines and cannot themselves contain unencoded newlines.
+
+##Compatibility
+`pio batchpredict` loads the engine and processes queries exactly like
+`pio deploy`. There is only one additional requirement for engines
+to utilize batch predict:
+
+WARNING: All algorithm classes used in the engine must be
+[serializable](https://www.scala-lang.org/api/2.11.8/index.html#scala.Serializable).
+**This is already true for PredictionIO's base algorithm classes**, but may be broken
+by including non-serializable fields in their constructor. Using the
+[`@transient` annotation](http://fdahms.com/2015/10/14/scala-and-the-transient-lazy-val-pattern/)
+may help in these cases.
+
+This requirement is due to processing the input queries as a
+[Spark RDD](https://spark.apache.org/docs/latest/rdd-programming-guide.html#resilient-distributed-datasets-rdds)
+which enables high-performance parallelization, even on a single machine.
+
+##Usage
+
+### `pio batchpredict`
+
+Command to process bulk predictions. Takes the same options as `pio deploy` plus:
+
+### `--input <value>`
+
+Path to file containing queries; a multi-object JSON file with one
+query object per line. Accepts any valid Hadoop file URL.
+
+Default: `batchpredict-input.json`
+
+### `--output <value>`
+
+Path to file to receive results; a multi-object JSON file with one
+object per line, the prediction + original query. Accepts any
+valid Hadoop file URL. Actual output will be written as Hadoop
+partition files in a directory with the output name.
+
+Default: `batchpredict-output.json`
+
+### `--query-partitions <value>`
+
+Configure the concurrency of predictions by setting the number of partitions
+used internally for the RDD of queries. This will directly effect the
+number of resulting `part-*` output files. While setting to `1` may seem
+appealing to get a single output file, this will remove parallelization
+for the batch process, reducing performance and possibly exhausting memory.
+
+Default: number created by Spark context's `textFile` (probably the number
+of cores available on the local machine)
+
+### `--engine-instance-id <value>`
+
+Identifier for the trained instance to use for batch predict.
+
+Default: the latest trained instance.
+
+##Example
+
+###Input
+
+A multi-object JSON file of queries as they would be sent to the engine's
+HTTP Queries API.
+
+NOTE: Read via
+[SparkContext's `textFile`](https://spark.apache.org/docs/latest/rdd-programming-guide.html#external-datasets)
+and so may be a single file or any supported Hadoop format.
+
+File: `batchpredict-input.json`
+
+```json
+{"user":"1"}
+{"user":"2"}
+{"user":"3"}
+{"user":"4"}
+{"user":"5"}
+```
+
+###Execute
+
+```bash
+pio batchpredict \
+ --input batchpredict-input.json \
+ --output batchpredict-output.json
+```
+
+This command will run to completion, aborting if any errors are encountered.
+
+###Output
+
+A multi-object JSON file of predictions + original queries. The predictions
+are JSON objects as they would be returned from the engine's HTTP Queries API.
+
+NOTE: Results are written via Spark RDD's `saveAsTextFile` so each partition
+will be written to its own `part-*` file.
+See [post-processing results](#post-processing-results).
+
+File 1: `batchpredict-output.json/part-00000`
+
+```json
+{"query":{"user":"1"},"prediction":{"itemScores":[{"item":"1","score":33},{"item":"2","score":32}]}}
+{"query":{"user":"3"},"prediction":{"itemScores":[{"item":"2","score":16},{"item":"3","score":12}]}}
+{"query":{"user":"4"},"prediction":{"itemScores":[{"item":"3","score":19},{"item":"1","score":18}]}}
+```
+
+File 2: `batchpredict-output.json/part-00001`
+
+```json
+{"query":{"user":"2"},"prediction":{"itemScores":[{"item":"5","score":55},{"item":"3","score":28}]}}
+{"query":{"user":"5"},"prediction":{"itemScores":[{"item":"1","score":24},{"item":"4","score":14}]}}
+```
+
+###Post-processing Results
+
+After the process exits successfully, the parts may be concatenated into a
+single output file using a command like:
+
+```bash
+cat batchpredict-output.json/part-* > batchpredict-output-all.json
+```
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/0d2e06d5/docs/manual/source/cli/index.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/cli/index.html.md b/docs/manual/source/cli/index.html.md
index e4927fd..3735348 100644
--- a/docs/manual/source/cli/index.html.md
+++ b/docs/manual/source/cli/index.html.md
@@ -67,4 +67,9 @@ third-party informational messages.
```pio train``` Kick off a training using an engine.
-```pio deploy``` Deploy an engine as an engine server. If no instance ID is specified, it will deploy the latest instance.
+```pio deploy``` Deploy an engine as an engine server.
+
+```pio batchpredict``` Process bulk predictions using an engine.
+
+For ```deploy``` & ```batchpredict```, if ```--engine-instance-id``` is not
+specified, it will use the latest trained instance.
[08/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/README.md b/examples/experimental/scala-parallel-friend-recommendation/README.md
deleted file mode 100644
index 9c94bb2..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/README.md
+++ /dev/null
@@ -1,65 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-Parallel SimRank Algorithm
-========================================================================
-Uses the Delta-Simrank Algorithm (http://dprg.cs.uiuc.edu/docs/deltasimrank/simrank.pdf).
-
-Takes advantage of Spark's GraphX framework and in-memory/iterative computation advantage over other MapReduce frameworks such as YARN by performing each SimRank iteration as a triple of Map and two Reduce tasks.
-
-Map1 : Emit key value pairs where each key is a pair of vertex ids adjacent to the vertices with updated scores in the previous iterations, and the value is the marginal increase in SimRank (refer to the delta-simrank algorithm for exact calculation method for deltas).
-
-Reduce1 : Aggregate delta values by key.
-
-Reduce2 : Add the delta to the previous iteration SimRank score to get the current iteration's SimRank score.
-
-Prerequisite: GraphX package.
-
-Parameter Explained
--------------------
-datasource - graphEdgelistPath : The edge-list passed to GraphX's graph loader. For efficient memory storage of intermediate SimRank score calculations, the vertex ids should be in a contiguous range from 0 to (#Vertex-1). There is a utility function for re-mapping the vertex Id values : org.apache.predictionio.examples.pfriendrecommendation.DeltaSimRankRDD.normalizeGraph.
-
-The provided DataSource class uses the GraphLoader provided by GraphX. Graphs can be specified by a tab-separated edge list, where each line specifies one edge.
-The the user can refer to the provided example edge list at `$EXAMPLE_HOME/data/edge_list_small.txt` for a graph specification with 10 vertices and 20 edges.
-
-algorithms - numIterations : Number of iterations for calculating SimRank. Typical recommended is 6-8 in various papers (e.g. http://www-cs-students.stanford.edu/~glenj/simrank.pdf)
-
-algorithms - decay : Decay constant used in calculating incremental changes to SimRank
-
-### Using SimRank to Recommend Similar Items
-SimRank's intuition says two items are similar if they have similar neighbor items. In a Facebook friend recommendation scenario, neighbors to a person can be that person's friends. Two people in the same circle of friends will have higher SimRank compared to two people in different friend circles.
-For example, SimRank(Obama, George Bush) is high since they have many mutual president friends and belong to shared friend networks, whereas SimRank(Obama, Usher) is low since they have few mutual friends or friend networks.
-
-### Configurable Datasources for Sampling
-Three data sources can be configured from the engine factory :
-
-**DataSource** generates a GraphX graph using the entire dataset.
-
-**NodeSamplingDataSource** generates a GraphX graph after performing node sampling
-with induced edges between the sampled nodes. This data source takes an
-additional parameter, sampleFraction, which is the fraction of graph nodes to
-sample.
-
-**ForestFireSamplingDataSource** generates a graph after performing forest fire
-sampling. This sampling method also uses the sampleFraction parameter and takes
-an additional parameter, geoParam, which is the parameter for a geometric
-distribution that is used within forest fire sampling.
-
-### Example query
-curl -H "Content-Type: application/json" -d '{"item1":10, "item2":9}' http://localhost:8000/queries.json
-
-This queries the SimRank score between nodes with ids 10 and 9.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/build.sbt b/examples/experimental/scala-parallel-friend-recommendation/build.sbt
deleted file mode 100644
index 57156cd..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-recommendation-custom-preparator"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-graphx" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/data/circle.txt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/data/circle.txt b/examples/experimental/scala-parallel-friend-recommendation/data/circle.txt
deleted file mode 100644
index d4f4206..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/data/circle.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-1 2
-2 3
-3 1
-4 1
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/data/edge_list_small.txt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/data/edge_list_small.txt b/examples/experimental/scala-parallel-friend-recommendation/data/edge_list_small.txt
deleted file mode 100644
index 1850ec9..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/data/edge_list_small.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-0 1
-2 3
-1 3
-3 0
-6 4
-6 5
-4 1
-5 5
-7 1
-8 5
-9 5
-9 4
-10 9
-10 8
-10 5
-10 3
-10 4
-10 2
-10 1
-10 0
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/data/simple.txt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/data/simple.txt b/examples/experimental/scala-parallel-friend-recommendation/data/simple.txt
deleted file mode 100644
index 53ed851..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/data/simple.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-1 2
-1 3
-3 1
-2 3
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/engine-forest.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/engine-forest.json b/examples/experimental/scala-parallel-friend-recommendation/engine-forest.json
deleted file mode 100644
index 2820578..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/engine-forest.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "id": "SimRankParallelEngine",
- "description": "Engine for graph vertex similarity using SimRank",
- "engineFactory": "org.apache.predictionio.examples.pfriendrecommendation.PSimRankEngineFactory",
-
- "datasource": {
- "name" : "forest",
- "params": {
- "graphEdgelistPath":"data/edge_list_small.txt",
- "sampleFraction":1.0,
- "geoParam":0.7
- }
- },
- "algorithms": [
- {
- "name": "simrank",
- "params": {
- "numIterations": 6,
- "decay":0.7
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/engine.json b/examples/experimental/scala-parallel-friend-recommendation/engine.json
deleted file mode 100644
index 413928f..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/engine.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "id": "SimRankParallelEngine",
- "description": "Engine for graph vertex similarity using SimRank",
- "engineFactory": "org.apache.predictionio.examples.pfriendrecommendation.PSimRankEngineFactory",
-
- "datasource": {
- "name" : "default",
- "params": {
- "graphEdgelistPath":"data/simple.txt"
- }
- },
- "algorithms": [
- {
- "name": "simrank",
- "params": {
- "numIterations": 4,
- "decay":0.7
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/project/assembly.sbt b/examples/experimental/scala-parallel-friend-recommendation/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DataSource.scala
deleted file mode 100644
index 7474180..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.pfriendrecommendation
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.graphx._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-/*
- Data Source Params : path to data file
- */
-case class DataSourceParams(
- val graphEdgelistPath: String
-) extends Params
-
-case class TrainingData(
- val g:Graph[Int,Int],
- val identityMatrix:RDD[(VertexId,Double)]
-)
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData, EmptyEvaluationInfo, Query, Double] {
-
- override
- def readTraining(sc:SparkContext) : TrainingData = {
- val g = GraphLoader.edgeListFile(sc, dsp.graphEdgelistPath)
- // In the interest of space (since we calculate at most n*n simrank scores),
- // each of the n vertices should have vertexID in the range 0 to n-1
- // val g2 = DeltaSimRankRDD.normalizeGraph(g)
- val identity = DeltaSimRankRDD.identityMatrix(sc, g.vertices.count())
- new TrainingData(g, identity)
- }
-}
-
-case class NodeSamplingDSParams(
- val graphEdgelistPath: String,
- val sampleFraction: Double
-) extends Params
-
-class NodeSamplingDataSource(val dsp: NodeSamplingDSParams)
- extends PDataSource[TrainingData, EmptyEvaluationInfo, Query, Double] {
-
- override
- def readTraining(sc:SparkContext) : TrainingData = {
- val g = GraphLoader.edgeListFile(sc, dsp.graphEdgelistPath)
- val sampled = Sampling.nodeSampling(sc, g, dsp.sampleFraction)
- val identity = DeltaSimRankRDD.identityMatrix(sc, g.vertices.count())
- new TrainingData(sampled, identity)
- }
-}
-
-case class FFSamplingDSParams(
- val graphEdgelistPath: String,
- val sampleFraction: Double,
- val geoParam: Double
-) extends Params
-
-class ForestFireSamplingDataSource(val dsp: FFSamplingDSParams)
- extends PDataSource[TrainingData, EmptyEvaluationInfo, Query, Double] {
-
- override
- def readTraining(sc:SparkContext) : TrainingData = {
- val g = GraphLoader.edgeListFile(sc, dsp.graphEdgelistPath)
- val sampled = Sampling.forestFireSamplingInduced(
- sc,
- g,
- dsp.sampleFraction,
- dsp.geoParam)
-
- val identity = DeltaSimRankRDD.identityMatrix(sc, g.vertices.count())
- new TrainingData(sampled, identity)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DeltaSimRankRDD.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DeltaSimRankRDD.scala b/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DeltaSimRankRDD.scala
deleted file mode 100644
index 2bdde35..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/DeltaSimRankRDD.scala
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.pfriendrecommendation
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.SparkContext._
-import org.apache.spark.graphx._
-import org.apache.spark.rdd.RDD
-
-import scala.collection.mutable.ListBuffer
-import scala.collection.mutable.Map
-
-
-object DeltaSimRankRDD {
- var decay:Double = 0.8
- var numNodes:Int = 0
-
- def calculateNthIter(
- numNodes:Int,
- g:Graph[Int, Int],
- prevDelta:RDD[((VertexId,VertexId),Double)],
- outDegreeMap:scala.collection.Map[VertexId,Long])
- : RDD[((VertexId,VertexId), Double)] =
- {
- // No changes in last iteration -> no changes this iteration.
- if (prevDelta.count() == 0)
- return prevDelta
-
- val pairList = prevDelta.toArray
- val kvPairs = pairList.map(pair => {
- val a = pair._1._1.toInt
- val b = pair._1._2.toInt
- val delta = pair._2
- val b_adj = g.edges.filter(e => e.dstId == b).map(x=>x.srcId)
- val a_adj = g.edges.filter(e => e.dstId == a).map(x=>x.srcId)
-
- val scorePairs = a_adj.cartesian(b_adj)
- scorePairs.filter(pair=> pair._1 != pair._2).map(pair => (pair, delta))
- })
-
- var union = kvPairs(0)
- var index = 0
- for (index <- 1 to kvPairs.size-1)
- union = union ++ kvPairs(index)
-
- val newDelta = union.reduceByKey(_ + _)
- .map(k =>
- (k._1, k._2*decay/(outDegreeMap(k._1._1) * outDegreeMap(k._1._2)))
- )
- newDelta
- }
-
- def identityMatrix(sc:SparkContext, numCols:Long) : RDD[(Long, Double)] =
- {
- val numElements = numCols * numCols
- val arr = Array[Long]((0L to numElements).toList:_*)
- // (Score, Index), where (x,y) = (Index/numCols, Index%numCols)
- val pairs = arr.map(x => {
- if (x/numCols == x % numCols)
- (x, 1.0)
- else
- (x, 0.0)
- })
- sc.parallelize(pairs)
- }
-
- def matrixToIndices(x:Int, y:Int, numCols:Int) = {
- x + y * numCols
- }
-
- def joinDelta(
- prevIter:RDD[(Long, Double)],
- numCols:Int,
- delta:RDD[((VertexId,VertexId), Double)]) : RDD[(Long,Double)] =
- {
- val deltaToIndex:RDD[(Long,Double)] = delta.map(x => {
- val index = (x._1._1-1)*numCols + (x._1._2-1)
- (index, x._2)
- })
- println("detaToIndex")
- deltaToIndex.foreach(println(_))
- val newIter = prevIter.leftOuterJoin(deltaToIndex)
- val newScores = newIter.map(x => {
- val index = x._1
- if (x._2._2.isDefined) {
- (index, x._2._1 + x._2._2.get)
- } else {
- (index, x._2._1)
- }
- })
- newScores
- }
-
- def getOutdegreeMap(g:Graph[Int,Int]) : scala.collection.Map[VertexId, Long] =
- {
- g.edges.map(edge => (edge.srcId,1L))
- .reduceByKey(_ + _)
- .collectAsMap()
- }
-
- def compute(
- g:Graph[Int,Int],
- numIterations:Int,
- identityMatrix:RDD[(VertexId,Double)],
- newDecay:Double) : RDD[(VertexId,Double)] =
- {
- numNodes = g.vertices.count().toInt
- decay = newDecay
-
- // Build the identity matrix representing 0-th iteration of SimRank
- val s0 = identityMatrix
- val outDegreeMap:scala.collection.Map[VertexId,Long] = getOutdegreeMap(g)
- val s0Delta = g.vertices.map(vertex => ((vertex._1, vertex._1), 1.0))
-
- var prevSimrank = s0
- var prevDelta = s0Delta
-
- println(s"initial")
- s0.foreach(println(_))
- prevDelta.foreach(println(_))
-
- for (i <- 0 to numIterations) {
- val nextIterDelta = calculateNthIter(numNodes, g, prevDelta, outDegreeMap)
- val nextIterSimrank = joinDelta(prevSimrank, numNodes, nextIterDelta)
- println(s"iteration: ${i}")
- nextIterDelta.foreach(println(_))
- nextIterSimrank.foreach(println(_))
- prevSimrank = nextIterSimrank
- prevDelta = nextIterDelta
- }
-
- prevSimrank
- }
-
- // Make all vertexId in one contiguous number range
- def normalizeGraph(g:Graph[Int,Int]) = {
- var counter = 0.toLong
- val hash = Map[VertexId, Long]()
-
- val v = g.vertices.map( pair => {
- hash(pair._1) = counter
- counter += 1
- (counter - 1, pair._2)
- })
-
- val e = g.edges.map( (e:Edge[Int]) => {
- if (hash.contains(e.srcId)) {
- e.srcId = hash(e.srcId)
- } else {
- hash += (e.srcId -> counter)
- counter += 1
- e.srcId = counter - 1
- }
-
- if (hash.contains(e.dstId)) {
- e.dstId = hash(e.dstId)
- } else {
- hash += (e.dstId -> counter)
- counter += 1
- e.dstId = counter - 1
- }
- e
- })
-
- val g2 = Graph(v,e)
- g2
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Engine.scala
deleted file mode 100644
index 5e0b8b9..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.pfriendrecommendation
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-
-case class Query(
- val item1: Long,
- val item2: Long
-)
-
-case class PredictedResult(
- val productScores: Array[ProductScore]
-)
-
-case class ProductScore(
- product: Int,
- score: Double
-)
-
-object PSimRankEngineFactory extends IEngineFactory {
- def apply() = {
- Engine(
- Map(
- "default" -> classOf[DataSource],
- "node" -> classOf[NodeSamplingDataSource],
- "forest" -> classOf[ForestFireSamplingDataSource]),
- classOf[IdentityPreparator],
- Map("simrank" -> classOf[SimRankAlgorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Preparator.scala
deleted file mode 100644
index 60b2ed5..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.pfriendrecommendation
-
-import org.apache.predictionio.controller.EmptyParams
-import org.apache.predictionio.controller.PPreparator
-import org.apache.predictionio.controller.EmptyPreparatorParams
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import scala.io.Source // ADDED
-import org.apache.predictionio.controller.Params // ADDED
-
- // ADDED CustomPreparatorParams case class
-case class CustomPreparatorParams(
- val filepath: String
-) extends Params
-
-class IdentityPreparator(pp: EmptyPreparatorParams)
- extends PPreparator[TrainingData, TrainingData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): TrainingData = {
- trainingData
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Sampling.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Sampling.scala b/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Sampling.scala
deleted file mode 100644
index 808780c..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Sampling.scala
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.pfriendrecommendation
-
-import org.apache.spark.SparkContext._
-import org.apache.spark.graphx._
-import org.apache.spark.rdd.RDD
-import scala.collection.mutable.{ListBuffer, ArraySeq, HashSet}
-import scala.util.Random
-
-import scala.collection.mutable.HashSet
-import scala.collection.mutable.Queue
-import org.apache.spark.SparkContext
-import scala.collection.mutable.Map
-
-object Sampling {
-
- def geometricSample(param: Double) : Int = {
- var num = 1
- while(Random.nextDouble <= param) {
- num += 1
- }
- num
- }
-
- def sortBySrc(a:Array[Edge[Int]]): Array[Edge[Int]] = {
- if (a.length < 2) {
- a
- } else {
- val pivot = a(a.length / 2).srcId
- // 'L'ess, 'E'qual, 'G'reater
- val partitions = a.groupBy( (e:Edge[Int]) => {
- if (e.srcId < pivot)
- 'L'
- else if (e.srcId > pivot)
- 'G'
- else
- 'E'
- })
-
- var sortedAccumulator: Array[Edge[Int]] = Array()
- List('L', 'E', 'G').foreach((c:Char) => {
- if (partitions.contains(c)) {
- sortedAccumulator = sortedAccumulator ++ partitions(c)
- }
- })
- sortedAccumulator
- }
- }
-
- // Samples vertices by forest fire random process and induces edges.
- // Fraction denotes fraction of total graph vertices to sample and geoParam
- // denotes the parameter for geometric distribution, which is used to
- // determine branching factor at each iteration of forest fire process.
- def forestFireSamplingInduced (
- sc: SparkContext,
- graph: Graph[Int,Int],
- fraction: Double,
- geoParam: Double = 0.7) =
- {
- var g = graph
- var e = sortBySrc(g.edges.toArray)
- val targetVertexCount = (graph.vertices.count() * fraction).toInt
- var seedVertices = graph.vertices
- .sample(false, fraction, targetVertexCount)
- .toArray.iterator
- var sampledVertices: HashSet[VertexId] = HashSet()
- var burnQueue: Queue[VertexId] = Queue()
-
- while (sampledVertices.size < targetVertexCount) {
- val seedVertex = seedVertices.next
- sampledVertices += seedVertex._1
- burnQueue += seedVertex._1
- while (burnQueue.size > 0 ){
- val vertexId = burnQueue.dequeue()
- val numToSample = geometricSample(geoParam)
- val edgeCandidates = accumulateEdges(e, vertexId)
- val burnCandidate = sc.parallelize(edgeCandidates)
- .filter((e:Edge[Int]) => {
- !sampledVertices.contains(e.dstId)
- })
- val burnFraction = numToSample.toDouble / burnCandidate.count.toDouble
- val burnEdges = burnCandidate.sample(
- false,
- burnFraction,
- Random.nextLong)
- val neighborVertexIds = burnEdges.map((e:Edge[Int]) => e.dstId)
- sampledVertices = sampledVertices ++ neighborVertexIds.toArray
- burnQueue = burnQueue ++ neighborVertexIds.toArray
-
- if (sampledVertices.size > targetVertexCount) {
- burnQueue.dequeueAll((v:VertexId) => true)
- }
- }
- }
- val vertex: Seq[(VertexId, Int)] = sampledVertices.map(v => (v,1))
- .toSeq
- val edges = graph.edges.filter(e =>
- sampledVertices.contains(e.srcId) && sampledVertices.contains(e.dstId)
- )
- Graph(sc.parallelize(vertex), edges)
- }
-
- // Samples vertices uniformly and induces edges.
- def nodeSampling(sc:SparkContext, graph:Graph[Int,Int], fraction:Double) = {
- val vertices = graph.vertices.sample(false, fraction, Random.nextLong)
- val vertexMap = vertices.collectAsMap()
- val edges = graph.edges.filter(e =>
- vertexMap.contains(e.srcId) && vertexMap.contains(e.dstId)
- )
- val graph2 = Graph(vertices, edges)
- graph2
- }
-
- // Get all edges with source vertexId of target
- def accumulateEdges(
- e:Array[Edge[Int]],
- target:VertexId) : ListBuffer[Edge[Int]] =
- {
- val idx = binarySearchE(e, target)(0, e.size-1)
- var outEdges: ListBuffer[Edge[Int]] = ListBuffer()
- if (idx == -1)
- return outEdges
- outEdges.append(e(idx))
- var tIdx = idx+1
- var edge:Edge[Int] = null
- // get upper edges
- while (tIdx < e.size) {
- edge = e(tIdx)
- if (edge.srcId == target) {
- outEdges.append(edge)
- tIdx += 1
- } else {
- tIdx = e.size
- }
- }
- // get lower edges
- tIdx = idx-1
- while (tIdx > -1){
- edge = e(tIdx)
- if (edge.srcId == target) {
- outEdges.append(edge)
- tIdx -= 1
- } else {
- tIdx = -1
- }
- }
- outEdges
- }
-
-
- // Binary search to find an edge with target vertexId
- def binarySearchE(list: Array[Edge[Int]], target: VertexId)
- (start: Int=0, end: Int=list.length-1): Int =
- {
- if (start>end)
- return -1
- val mid = start + (end-start+1)/2
- if (list(mid).srcId == target)
- return mid
- else if (list(mid).srcId > target)
- return binarySearchE(list, target)(start, mid-1)
- else
- return binarySearchE(list, target)(mid+1, end)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Serving.scala
deleted file mode 100644
index eca5d2d..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.pfriendrecommendation
-
-import org.apache.predictionio.controller.LServing
-import org.apache.predictionio.controller.EmptyServingParams
-
-class Serving
- extends LServing[Query, Double] {
-
- override
- def serve(query: Query, predictedResults: Seq[Double]): Double = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/SimRankAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/SimRankAlgorithm.scala b/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/SimRankAlgorithm.scala
deleted file mode 100644
index cd620ba..0000000
--- a/examples/experimental/scala-parallel-friend-recommendation/src/main/scala/SimRankAlgorithm.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.pfriendrecommendation
-import org.apache.predictionio.controller.PAlgorithm
-import org.apache.predictionio.controller.Params
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.graphx._
-import org.apache.spark.rdd.RDD
-
-case class SimRankParams(
- val numIterations: Int,
- val decay: Double) extends Params
-
-class SimRankAlgorithm(val ap: SimRankParams)
- extends PAlgorithm[TrainingData, RDD[(Long,Double)], Query, Double] {
-
- def train(td: TrainingData): RDD[(Long,Double)] = {
- td.g.edges.count()
- val scores = DeltaSimRankRDD.compute(
- td.g,
- ap.numIterations,
- td.identityMatrix,
- ap.decay)
- scores
- }
-
- /*
- def batchPredict(
- model: RDD[(VertexId,Double)],
- feature: RDD[(Long, (Int, Int))]): RDD[(Long, Double)] = {
- feature.map(x => (x._1, predict(model, (x._2._1, x._2._1))))
- }
- */
-
- def predict(
- model: RDD[(Long,Double)], query:Query): Double = {
- // Will never encounter rounding errors because model is an n*n "matrix".
- val numElems = math.sqrt(model.count()).toInt
- val index = query.item1 * numElems + query.item2
- val seq = model.lookup(index)
- seq.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/.gitignore b/examples/experimental/scala-parallel-recommendation-cat/.gitignore
deleted file mode 100644
index 64fa18b..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-manifest.json
-target/
-pio.log
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/README.md b/examples/experimental/scala-parallel-recommendation-cat/README.md
deleted file mode 100644
index 9394dbf..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/README.md
+++ /dev/null
@@ -1,94 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Recommendation Template support categories filtering
-
-## Documentation
-
-The data requirement is similar to "Similar Product Template". Please see.
-http://predictionio.incubator.apache.org/templates/similarproduct/quickstart/
-
-By default, view events and MLlib ALS trainImplicit is used.
-
-The main difference is:
-
-Query by user Id instead of list of item Ids.
-
-## Development Notes
-
-### import sample data
-
-```
-$ python data/import_eventserver.py --access_key <your_access_key>
-```
-
-### query
-
-normal:
-
-```
-$ curl -H "Content-Type: application/json" \
--d '{
- "user" : "u1",
- "num" : 10 }' \
-http://localhost:8000/queries.json \
--w %{time_connect}:%{time_starttransfer}:%{time_total}
-```
-
-```
-$ curl -H "Content-Type: application/json" \
--d '{
- "user" : "u1",
- "num": 10,
- "categories" : ["c4", "c3"]
-}' \
-http://localhost:8000/queries.json \
--w %{time_connect}:%{time_starttransfer}:%{time_total}
-```
-
-```
-curl -H "Content-Type: application/json" \
--d '{
- "user" : "u1",
- "num": 10,
- "whiteList": ["i21", "i26", "i40"]
-}' \
-http://localhost:8000/queries.json \
--w %{time_connect}:%{time_starttransfer}:%{time_total}
-```
-
-```
-curl -H "Content-Type: application/json" \
--d '{
- "user" : "u1",
- "num": 10,
- "blackList": ["i21", "i26", "i40"]
-}' \
-http://localhost:8000/queries.json \
--w %{time_connect}:%{time_starttransfer}:%{time_total}
-```
-
-unknown user:
-
-```
-curl -H "Content-Type: application/json" \
--d '{
- "user" : "unk1",
- "num": 10}' \
-http://localhost:8000/queries.json \
--w %{time_connect}:%{time_starttransfer}:%{time_total}
-```
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/build.sbt b/examples/experimental/scala-parallel-recommendation-cat/build.sbt
deleted file mode 100644
index 3ff8d27..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-recommendation-cat"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.8.6" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/data/import_eventserver.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/data/import_eventserver.py b/examples/experimental/scala-parallel-recommendation-cat/data/import_eventserver.py
deleted file mode 100644
index d47d60f..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/data/import_eventserver.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Import sample data for E-Commerce Recommendation Engine Template
-"""
-
-import predictionio
-import argparse
-import random
-
-SEED = 3
-
-def import_events(client):
- random.seed(SEED)
- count = 0
- print client.get_status()
- print "Importing data..."
-
- # generate 10 users, with user ids u1,u2,....,u10
- user_ids = ["u%s" % i for i in range(1, 11)]
- for user_id in user_ids:
- print "Set user", user_id
- client.create_event(
- event="$set",
- entity_type="user",
- entity_id=user_id
- )
- count += 1
-
- # generate 50 items, with item ids i1,i2,....,i50
- # random assign 1 to 4 categories among c1-c6 to items
- categories = ["c%s" % i for i in range(1, 7)]
- item_ids = ["i%s" % i for i in range(1, 51)]
- for item_id in item_ids:
- print "Set item", item_id
- client.create_event(
- event="$set",
- entity_type="item",
- entity_id=item_id,
- properties={
- "categories" : random.sample(categories, random.randint(1, 4))
- }
- )
- count += 1
-
- # each user randomly viewed 10 items
- for user_id in user_ids:
- for viewed_item in random.sample(item_ids, 10):
- print "User", user_id ,"views item", viewed_item
- client.create_event(
- event="view",
- entity_type="user",
- entity_id=user_id,
- target_entity_type="item",
- target_entity_id=viewed_item
- )
- count += 1
- # randomly buy some of the viewed items
- if random.choice([True, False]):
- print "User", user_id ,"buys item", viewed_item
- client.create_event(
- event="buy",
- entity_type="user",
- entity_id=user_id,
- target_entity_type="item",
- target_entity_id=viewed_item
- )
- count += 1
-
- print "%s events are imported." % count
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description="Import sample data for e-commerce recommendation engine")
- parser.add_argument('--access_key', default='invald_access_key')
- parser.add_argument('--url', default="http://localhost:7070")
-
- args = parser.parse_args()
- print args
-
- client = predictionio.EventClient(
- access_key=args.access_key,
- url=args.url,
- threads=5,
- qsize=500)
- import_events(client)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/data/send_query.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/data/send_query.py b/examples/experimental/scala-parallel-recommendation-cat/data/send_query.py
deleted file mode 100644
index 21e9146..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/data/send_query.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Send sample query to prediction engine
-"""
-
-import predictionio
-engine_client = predictionio.EngineClient(url="http://localhost:8000")
-print engine_client.send_query({"user": "u1", "num": 4})
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/engine.json b/examples/experimental/scala-parallel-recommendation-cat/engine.json
deleted file mode 100644
index d3406af..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/engine.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.template.recommendation.RecommendationEngine",
- "datasource": {
- "params" : {
- "appId": 17
- }
- },
- "algorithms": [
- {
- "name": "als",
- "params": {
- "rank": 10,
- "numIterations" : 20,
- "lambda": 0.01,
- "seed": 3
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/project/assembly.sbt b/examples/experimental/scala-parallel-recommendation-cat/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/ALSAlgorithm.scala b/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/ALSAlgorithm.scala
deleted file mode 100644
index 84ed3a4..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/ALSAlgorithm.scala
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.P2LAlgorithm
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.BiMap
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.mllib.recommendation.ALS
-import org.apache.spark.mllib.recommendation.{Rating => MLlibRating}
-
-import grizzled.slf4j.Logger
-
-import scala.collection.mutable.PriorityQueue
-import scala.concurrent.duration.Duration
-import scala.concurrent.ExecutionContext.Implicits.global
-
-case class ALSAlgorithmParams(
- rank: Int,
- numIterations: Int,
- lambda: Double,
- seed: Option[Long]
-) extends Params
-
-class ALSModel(
- val rank: Int,
- val userFeatures: Map[Int, Array[Double]],
- val productFeatures: Map[Int, (Item, Option[Array[Double]])],
- val userStringIntMap: BiMap[String, Int],
- val itemStringIntMap: BiMap[String, Int]
-) extends Serializable {
-
- @transient lazy val itemIntStringMap = itemStringIntMap.inverse
-
- override def toString = {
- s" rank: ${rank}" +
- s" userFeatures: [${userFeatures.size}]" +
- s"(${userFeatures.take(2).toList}...)" +
- s" productFeatures: [${productFeatures.size}]" +
- s"(${productFeatures.take(2).toList}...)" +
- s" userStringIntMap: [${userStringIntMap.size}]" +
- s"(${userStringIntMap.take(2).toString}...)]" +
- s" itemStringIntMap: [${itemStringIntMap.size}]" +
- s"(${itemStringIntMap.take(2).toString}...)]"
- }
-}
-
-/**
- * Use ALS to build item x feature matrix
- */
-class ALSAlgorithm(val ap: ALSAlgorithmParams)
- extends P2LAlgorithm[PreparedData, ALSModel, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: PreparedData): ALSModel = {
- require(!data.viewEvents.take(1).isEmpty,
- s"viewEvents in PreparedData cannot be empty." +
- " Please check if DataSource generates TrainingData" +
- " and Preprator generates PreparedData correctly.")
- require(!data.users.take(1).isEmpty,
- s"users in PreparedData cannot be empty." +
- " Please check if DataSource generates TrainingData" +
- " and Preprator generates PreparedData correctly.")
- require(!data.items.take(1).isEmpty,
- s"items in PreparedData cannot be empty." +
- " Please check if DataSource generates TrainingData" +
- " and Preprator generates PreparedData correctly.")
- // create User and item's String ID to integer index BiMap
- val userStringIntMap = BiMap.stringInt(data.users.keys)
- val itemStringIntMap = BiMap.stringInt(data.items.keys)
-
- val mllibRatings = data.viewEvents
- .map { r =>
- // Convert user and item String IDs to Int index for MLlib
- val uindex = userStringIntMap.getOrElse(r.user, -1)
- val iindex = itemStringIntMap.getOrElse(r.item, -1)
-
- if (uindex == -1)
- logger.info(s"Couldn't convert nonexistent user ID ${r.user}"
- + " to Int index.")
-
- if (iindex == -1)
- logger.info(s"Couldn't convert nonexistent item ID ${r.item}"
- + " to Int index.")
-
- ((uindex, iindex), 1)
- }.filter { case ((u, i), v) =>
- // keep events with valid user and item index
- (u != -1) && (i != -1)
- }
- .reduceByKey(_ + _) // aggregate all view events of same user-item pair
- .map { case ((u, i), v) =>
- // MLlibRating requires integer index for user and item
- MLlibRating(u, i, v)
- }.cache()
-
- // MLLib ALS cannot handle empty training data.
- require(!mllibRatings.take(1).isEmpty,
- s"mllibRatings cannot be empty." +
- " Please check if your events contain valid user and item ID.")
-
- // seed for MLlib ALS
- val seed = ap.seed.getOrElse(System.nanoTime)
-
- val m = ALS.trainImplicit(
- ratings = mllibRatings,
- rank = ap.rank,
- iterations = ap.numIterations,
- lambda = ap.lambda,
- blocks = -1,
- alpha = 1.0,
- seed = seed)
-
- val userFeatures = m.userFeatures.collectAsMap.toMap
-
- // convert ID to Int index
- val items = data.items.map { case (id, item) =>
- (itemStringIntMap(id), item)
- }
-
- // join item with the trained productFeatures
- val productFeatures = items.leftOuterJoin(m.productFeatures)
- .collectAsMap.toMap
-
- new ALSModel(
- rank = m.rank,
- userFeatures = userFeatures,
- productFeatures = productFeatures,
- userStringIntMap = userStringIntMap,
- itemStringIntMap = itemStringIntMap
- )
- }
-
- def predict(model: ALSModel, query: Query): PredictedResult = {
-
- val userFeatures = model.userFeatures
- val productFeatures = model.productFeatures
-
- // convert whiteList's string ID to integer index
- val whiteList: Option[Set[Int]] = query.whiteList.map( set =>
- set.map(model.itemStringIntMap.get(_)).flatten
- )
-
- val blackList: Set[String] = query.blackList.getOrElse(Set[String]())
-
- // combine query's blackList
- // into final blackList.
- // convert seen Items list from String ID to interger Index
- val finalBlackList: Set[Int] = blackList.map( x =>
- model.itemStringIntMap.get(x)).flatten
-
- val userFeature =
- model.userStringIntMap.get(query.user).map { userIndex =>
- userFeatures.get(userIndex)
- }
- // flatten Option[Option[Array[Double]]] to Option[Array[Double]]
- .flatten
-
- val topScores = if (userFeature.isDefined) {
- // the user has feature vector
- val uf = userFeature.get
- val indexScores: Map[Int, Double] =
- productFeatures.par // convert to parallel collection
- .filter { case (i, (item, feature)) =>
- feature.isDefined &&
- isCandidateItem(
- i = i,
- item = item,
- categories = query.categories,
- whiteList = whiteList,
- blackList = finalBlackList
- )
- }
- .map { case (i, (item, feature)) =>
- // NOTE: feature must be defined, so can call .get
- val s = dotProduct(uf, feature.get)
- // Can adjust score here
- (i, s)
- }
- .filter(_._2 > 0) // only keep items with score > 0
- .seq // convert back to sequential collection
-
- val ord = Ordering.by[(Int, Double), Double](_._2).reverse
- val topScores = getTopN(indexScores, query.num)(ord).toArray
-
- topScores
-
- } else {
- // the user doesn't have feature vector.
- // For example, new user is created after model is trained.
- logger.info(s"No userFeature found for user ${query.user}.")
- Array[(Int, Double)]()
- }
-
- val itemScores = topScores.map { case (i, s) =>
- new ItemScore(
- // convert item int index back to string ID
- item = model.itemIntStringMap(i),
- score = s
- )
- }
-
- new PredictedResult(itemScores)
- }
-
- private
- def getTopN[T](s: Iterable[T], n: Int)(implicit ord: Ordering[T]): Seq[T] = {
-
- val q = PriorityQueue()
-
- for (x <- s) {
- if (q.size < n)
- q.enqueue(x)
- else {
- // q is full
- if (ord.compare(x, q.head) < 0) {
- q.dequeue()
- q.enqueue(x)
- }
- }
- }
-
- q.dequeueAll.toSeq.reverse
- }
-
- private
- def dotProduct(v1: Array[Double], v2: Array[Double]): Double = {
- val size = v1.size
- var i = 0
- var d: Double = 0
- while (i < size) {
- d += v1(i) * v2(i)
- i += 1
- }
- d
- }
-
- private
- def isCandidateItem(
- i: Int,
- item: Item,
- categories: Option[Set[String]],
- whiteList: Option[Set[Int]],
- blackList: Set[Int]
- ): Boolean = {
- // can add other custom filtering here
- whiteList.map(_.contains(i)).getOrElse(true) &&
- !blackList.contains(i) &&
- // filter categories
- categories.map { cat =>
- item.categories.map { itemCat =>
- // keep this item if has ovelap categories with the query
- !(itemCat.toSet.intersect(cat).isEmpty)
- }.getOrElse(false) // discard this item if it has no categories
- }.getOrElse(true)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/DataSource.scala
deleted file mode 100644
index a7e8f5b..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-case class DataSourceParams(appId: Int) extends Params
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- val eventsDb = Storage.getPEvents()
-
- // create a RDD of (entityID, User)
- val usersRDD: RDD[(String, User)] = eventsDb.aggregateProperties(
- appId = dsp.appId,
- entityType = "user"
- )(sc).map { case (entityId, properties) =>
- val user = try {
- User()
- } catch {
- case e: Exception => {
- logger.error(s"Failed to get properties ${properties} of" +
- s" user ${entityId}. Exception: ${e}.")
- throw e
- }
- }
- (entityId, user)
- }.cache()
-
- // create a RDD of (entityID, Item)
- val itemsRDD: RDD[(String, Item)] = eventsDb.aggregateProperties(
- appId = dsp.appId,
- entityType = "item"
- )(sc).map { case (entityId, properties) =>
- val item = try {
- // Assume categories is optional property of item.
- Item(categories = properties.getOpt[List[String]]("categories"))
- } catch {
- case e: Exception => {
- logger.error(s"Failed to get properties ${properties} of" +
- s" item ${entityId}. Exception: ${e}.")
- throw e
- }
- }
- (entityId, item)
- }.cache()
-
- // get all "user" "view" "item" events
- val viewEventsRDD: RDD[ViewEvent] = eventsDb.find(
- appId = dsp.appId,
- entityType = Some("user"),
- eventNames = Some(List("view")),
- // targetEntityType is optional field of an event.
- targetEntityType = Some(Some("item")))(sc)
- // eventsDb.find() returns RDD[Event]
- .map { event =>
- val viewEvent = try {
- event.event match {
- case "view" => ViewEvent(
- user = event.entityId,
- item = event.targetEntityId.get,
- t = event.eventTime.getMillis)
- case _ => throw new Exception(s"Unexpected event ${event} is read.")
- }
- } catch {
- case e: Exception => {
- logger.error(s"Cannot convert ${event} to ViewEvent." +
- s" Exception: ${e}.")
- throw e
- }
- }
- viewEvent
- }.cache()
-
- new TrainingData(
- users = usersRDD,
- items = itemsRDD,
- viewEvents = viewEventsRDD
- )
- }
-}
-
-case class User()
-
-case class Item(categories: Option[List[String]])
-
-case class ViewEvent(user: String, item: String, t: Long)
-
-class TrainingData(
- val users: RDD[(String, User)],
- val items: RDD[(String, Item)],
- val viewEvents: RDD[ViewEvent]
-) extends Serializable {
- override def toString = {
- s"users: [${users.count()} (${users.take(2).toList}...)]" +
- s"items: [${items.count()} (${items.take(2).toList}...)]" +
- s"viewEvents: [${viewEvents.count()}] (${viewEvents.take(2).toList}...)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Engine.scala
deleted file mode 100644
index 32e2760..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-
-case class Query(
- user: String,
- num: Int,
- categories: Option[Set[String]],
- whiteList: Option[Set[String]],
- blackList: Option[Set[String]]
-)
-
-case class PredictedResult(
- itemScores: Array[ItemScore]
-)
-
-case class ItemScore(
- item: String,
- score: Double
-)
-
-object RecommendationEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- classOf[Preparator],
- Map("als" -> classOf[ALSAlgorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Preparator.scala
deleted file mode 100644
index 3193317..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PPreparator
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(
- users = trainingData.users,
- items = trainingData.items,
- viewEvents = trainingData.viewEvents)
- }
-}
-
-class PreparedData(
- val users: RDD[(String, User)],
- val items: RDD[(String, Item)],
- val viewEvents: RDD[ViewEvent]
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Serving.scala
deleted file mode 100644
index 48fda35..0000000
--- a/examples/experimental/scala-parallel-recommendation-cat/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/.gitignore b/examples/experimental/scala-parallel-recommendation-custom-datasource/.gitignore
deleted file mode 100644
index ea4e89d..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-data/sample_movielens_data.txt
-manifest.json
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/build.sbt b/examples/experimental/scala-parallel-recommendation-custom-datasource/build.sbt
deleted file mode 100644
index 91e94f2..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-recommendation"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/data/send_query.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/data/send_query.py b/examples/experimental/scala-parallel-recommendation-custom-datasource/data/send_query.py
deleted file mode 100644
index ca19dc5..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/data/send_query.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Send sample query to prediction engine
-"""
-
-import predictionio
-engine_client = predictionio.EngineClient(url="http://localhost:8000")
-print engine_client.send_query({"user": "1", "num": 4})
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/engine.json b/examples/experimental/scala-parallel-recommendation-custom-datasource/engine.json
deleted file mode 100644
index 4f24334..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/engine.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.template.recommendation.RecommendationEngine",
- "datasource": {
- "params": {
- "filepath": "./data/sample_movielens_data.txt"
- }
- },
- "algorithms": [
- {
- "name": "als",
- "params": {
- "rank": 10,
- "numIterations": 20,
- "lambda": 0.01
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/project/assembly.sbt b/examples/experimental/scala-parallel-recommendation-custom-datasource/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSAlgorithm.scala b/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSAlgorithm.scala
deleted file mode 100644
index fd93407..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSAlgorithm.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PAlgorithm
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.BiMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.mllib.recommendation.ALS
-import org.apache.spark.mllib.recommendation.{Rating => MLlibRating}
-import org.apache.spark.mllib.recommendation.ALSModel
-
-import grizzled.slf4j.Logger
-
-case class ALSAlgorithmParams(
- rank: Int,
- numIterations: Int,
- lambda: Double) extends Params
-
-class ALSAlgorithm(val ap: ALSAlgorithmParams)
- extends PAlgorithm[PreparedData, ALSModel, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: PreparedData): ALSModel = {
- // Convert user and item String IDs to Int index for MLlib
- val userStringIntMap = BiMap.stringInt(data.ratings.map(_.user))
- val itemStringIntMap = BiMap.stringInt(data.ratings.map(_.item))
- val mllibRatings = data.ratings.map( r =>
- // MLlibRating requires integer index for user and item
- MLlibRating(userStringIntMap(r.user), itemStringIntMap(r.item), r.rating)
- )
- val m = ALS.train(mllibRatings, ap.rank, ap.numIterations, ap.lambda)
- new ALSModel(
- rank = m.rank,
- userFeatures = m.userFeatures,
- productFeatures = m.productFeatures,
- userStringIntMap = userStringIntMap,
- itemStringIntMap = itemStringIntMap)
- }
-
- def predict(model: ALSModel, query: Query): PredictedResult = {
- // Convert String ID to Int index for Mllib
- model.userStringIntMap.get(query.user).map { userInt =>
- // create inverse view of itemStringIntMap
- val itemIntStringMap = model.itemStringIntMap.inverse
- // recommendProducts() returns Array[MLlibRating], which uses item Int
- // index. Convert it to String ID for returning PredictedResult
- val itemScores = model.recommendProducts(userInt, query.num)
- .map (r => ItemScore(itemIntStringMap(r.product), r.rating))
- new PredictedResult(itemScores)
- }.getOrElse{
- logger.info(s"No prediction for unknown user ${query.user}.")
- new PredictedResult(Array.empty)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSModel.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSModel.scala b/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSModel.scala
deleted file mode 100644
index 4697732..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/ALSModel.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.mllib.recommendation
-// This must be the same package as Spark's MatrixFactorizationModel because
-// MatrixFactorizationModel's constructor is private and we are using
-// its constructor in order to save and load the model
-
-import org.template.recommendation.ALSAlgorithmParams
-
-import org.apache.predictionio.controller.IPersistentModel
-import org.apache.predictionio.controller.IPersistentModelLoader
-import org.apache.predictionio.data.storage.BiMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class ALSModel(
- override val rank: Int,
- override val userFeatures: RDD[(Int, Array[Double])],
- override val productFeatures: RDD[(Int, Array[Double])],
- val userStringIntMap: BiMap[String, Int],
- val itemStringIntMap: BiMap[String, Int])
- extends MatrixFactorizationModel(rank, userFeatures, productFeatures)
- with IPersistentModel[ALSAlgorithmParams] {
-
- def save(id: String, params: ALSAlgorithmParams,
- sc: SparkContext): Boolean = {
-
- sc.parallelize(Seq(rank)).saveAsObjectFile(s"/tmp/${id}/rank")
- userFeatures.saveAsObjectFile(s"/tmp/${id}/userFeatures")
- productFeatures.saveAsObjectFile(s"/tmp/${id}/productFeatures")
- sc.parallelize(Seq(userStringIntMap))
- .saveAsObjectFile(s"/tmp/${id}/userStringIntMap")
- sc.parallelize(Seq(itemStringIntMap))
- .saveAsObjectFile(s"/tmp/${id}/itemStringIntMap")
- true
- }
-
- override def toString = {
- s"userFeatures: [${userFeatures.count()}]" +
- s"(${userFeatures.take(2).toList}...)" +
- s" productFeatures: [${productFeatures.count()}]" +
- s"(${productFeatures.take(2).toList}...)" +
- s" userStringIntMap: [${userStringIntMap.size}]" +
- s"(${userStringIntMap.take(2)}...)" +
- s" itemStringIntMap: [${itemStringIntMap.size}]" +
- s"(${itemStringIntMap.take(2)}...)"
- }
-}
-
-object ALSModel
- extends IPersistentModelLoader[ALSAlgorithmParams, ALSModel] {
- def apply(id: String, params: ALSAlgorithmParams,
- sc: Option[SparkContext]) = {
- new ALSModel(
- rank = sc.get.objectFile[Int](s"/tmp/${id}/rank").first,
- userFeatures = sc.get.objectFile(s"/tmp/${id}/userFeatures"),
- productFeatures = sc.get.objectFile(s"/tmp/${id}/productFeatures"),
- userStringIntMap = sc.get
- .objectFile[BiMap[String, Int]](s"/tmp/${id}/userStringIntMap").first,
- itemStringIntMap = sc.get
- .objectFile[BiMap[String, Int]](s"/tmp/${id}/itemStringIntMap").first)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/DataSource.scala
deleted file mode 100644
index fed37a9..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-case class DataSourceParams(filepath: String) extends Params // CHANGED
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- // CHANGED
- val data = sc.textFile(dsp.filepath)
- val ratings: RDD[Rating] = data.map(_.split("::") match {
- case Array(user, item, rate) =>
- Rating(user, item, rate.toDouble)
- })
- new TrainingData(ratings)
- }
-}
-
-case class Rating(
- user: String,
- item: String,
- rating: Double
-)
-
-class TrainingData(
- val ratings: RDD[Rating]
-) extends Serializable {
- override def toString = {
- s"ratings: [${ratings.count()}] (${ratings.take(2).toList}...)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Engine.scala
deleted file mode 100644
index 1446ca4..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-
-case class Query(
- user: String,
- num: Int
-)
-
-case class PredictedResult(
- itemScores: Array[ItemScore]
-)
-
-case class ItemScore(
- item: String,
- score: Double
-)
-
-object RecommendationEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- classOf[Preparator],
- Map("als" -> classOf[ALSAlgorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Preparator.scala
deleted file mode 100644
index 0bab35b..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PPreparator
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(ratings = trainingData.ratings)
- }
-}
-
-class PreparedData(
- val ratings: RDD[Rating]
-) extends Serializable
[20/50] [abbrv] incubator-predictionio git commit: [PIO-104] Make
proper implementation of plugins
Posted by ch...@apache.org.
[PIO-104] Make proper implementation of plugins
- Remove start() method of EventServerPlugin and EngineServerPlugin
- Send prediction result to outputSniffers in the engine server
Closes #407
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/965c73f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/965c73f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/965c73f0
Branch: refs/heads/master
Commit: 965c73f0f5dae726b930f4de12cf613fbe23bf58
Parents: 5813b80
Author: Naoki Takezoe <ta...@apache.org>
Authored: Fri Jul 28 13:27:49 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Fri Jul 28 13:27:49 2017 -0700
----------------------------------------------------------------------
.../scala/org/apache/predictionio/workflow/CreateServer.scala | 4 ++++
.../org/apache/predictionio/workflow/EngineServerPlugin.scala | 2 --
.../org/apache/predictionio/data/api/EventServerPlugin.scala | 2 --
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/965c73f0/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
index 8ef36e2..8f0aed7 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
@@ -579,6 +579,7 @@ class ServerActor[Q, P](
pluginContext.outputBlockers.values.foldLeft(result) { case (r, p) =>
p.process(engineInstance, queryJValue, r, pluginContext)
}
+ pluginsActorRef ! (engineInstance, queryJValue, result)
// Bookkeeping
val servingEndTime = DateTime.now
@@ -679,6 +680,9 @@ class ServerActor[Q, P](
val pluginType = segments(0)
val pluginName = segments(1)
pluginType match {
+ case EngineServerPlugin.outputBlocker =>
+ pluginContext.outputBlockers(pluginName).handleREST(
+ pluginArgs)
case EngineServerPlugin.outputSniffer =>
pluginsActorRef ? PluginsActor.HandleREST(
pluginName = pluginName,
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/965c73f0/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPlugin.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPlugin.scala b/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPlugin.scala
index 545fdb3..0857405 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPlugin.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPlugin.scala
@@ -26,8 +26,6 @@ trait EngineServerPlugin {
val pluginDescription: String
val pluginType: String
- def start(context: EngineServerPluginContext): Unit
-
def process(
engineInstance: EngineInstance,
query: JValue,
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/965c73f0/data/src/main/scala/org/apache/predictionio/data/api/EventServerPlugin.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/EventServerPlugin.scala b/data/src/main/scala/org/apache/predictionio/data/api/EventServerPlugin.scala
index cfb5ee2..5fe46a3 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/EventServerPlugin.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/EventServerPlugin.scala
@@ -23,8 +23,6 @@ trait EventServerPlugin {
val pluginDescription: String
val pluginType: String
- def start(context: EventServerPluginContext): Unit
-
def process(eventInfo: EventInfo, context: EventServerPluginContext)
def handleREST(appId: Int, channelId: Option[Int], arguments: Seq[String]): String
[27/50] [abbrv] incubator-predictionio git commit: Update
eventmodel.html.md.erb
Posted by ch...@apache.org.
Update eventmodel.html.md.erb
closes apache/incubator-predictionio#423
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/3856e5c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/3856e5c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/3856e5c0
Branch: refs/heads/master
Commit: 3856e5c0083dbdc66287ed8bbac6a3674fca67d6
Parents: 66a562c
Author: Nilmax Moura <ni...@gmail.com>
Authored: Fri Aug 18 15:56:43 2017 +0900
Committer: Takahiro Hagino <ha...@apache.org>
Committed: Fri Aug 18 15:56:43 2017 +0900
----------------------------------------------------------------------
docs/manual/source/datacollection/eventmodel.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3856e5c0/docs/manual/source/datacollection/eventmodel.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/datacollection/eventmodel.html.md.erb b/docs/manual/source/datacollection/eventmodel.html.md.erb
index f40880c..3f678f3 100644
--- a/docs/manual/source/datacollection/eventmodel.html.md.erb
+++ b/docs/manual/source/datacollection/eventmodel.html.md.erb
@@ -144,7 +144,7 @@ Date: Tue, 02 Jun 2015 23:13:58 GMT
Content-Type: application/json; charset=UTF-8
Content-Length: 57
-{"eventId":"PVjOIP6AJ5PgsiGQW6pgswAAAUhc7EwZpCfSj5bS5yg"}a
+{"eventId":"PVjOIP6AJ5PgsiGQW6pgswAAAUhc7EwZpCfSj5bS5yg"}
```
After this eventTime, user-2 is created and has properties of a = 3 and b = 4.
[12/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/data/movielens.txt
----------------------------------------------------------------------
diff --git a/examples/experimental/data/movielens.txt b/examples/experimental/data/movielens.txt
deleted file mode 100644
index f0eee19..0000000
--- a/examples/experimental/data/movielens.txt
+++ /dev/null
@@ -1,1501 +0,0 @@
-0::2::3
-0::3::1
-0::5::2
-0::9::4
-0::11::1
-0::12::2
-0::15::1
-0::17::1
-0::19::1
-0::21::1
-0::23::1
-0::26::3
-0::27::1
-0::28::1
-0::29::1
-0::30::1
-0::31::1
-0::34::1
-0::37::1
-0::41::2
-0::44::1
-0::45::2
-0::46::1
-0::47::1
-0::48::1
-0::50::1
-0::51::1
-0::54::1
-0::55::1
-0::59::2
-0::61::2
-0::64::1
-0::67::1
-0::68::1
-0::69::1
-0::71::1
-0::72::1
-0::77::2
-0::79::1
-0::83::1
-0::87::1
-0::89::2
-0::91::3
-0::92::4
-0::94::1
-0::95::2
-0::96::1
-0::98::1
-0::99::1
-1::2::2
-1::3::1
-1::4::2
-1::6::1
-1::9::3
-1::12::1
-1::13::1
-1::14::1
-1::16::1
-1::19::1
-1::21::3
-1::27::1
-1::28::3
-1::33::1
-1::36::2
-1::37::1
-1::40::1
-1::41::2
-1::43::1
-1::44::1
-1::47::1
-1::50::1
-1::54::1
-1::56::2
-1::57::1
-1::58::1
-1::60::1
-1::62::4
-1::63::1
-1::67::1
-1::68::4
-1::70::2
-1::72::1
-1::73::1
-1::74::2
-1::76::1
-1::77::3
-1::78::1
-1::81::1
-1::82::1
-1::85::3
-1::86::2
-1::88::2
-1::91::1
-1::92::2
-1::93::1
-1::94::2
-1::96::1
-1::97::1
-2::4::3
-2::6::1
-2::8::5
-2::9::1
-2::10::1
-2::12::3
-2::13::1
-2::15::2
-2::18::2
-2::19::4
-2::22::1
-2::26::1
-2::28::1
-2::34::4
-2::35::1
-2::37::5
-2::38::1
-2::39::5
-2::40::4
-2::47::1
-2::50::1
-2::52::2
-2::54::1
-2::55::1
-2::57::2
-2::58::2
-2::59::1
-2::61::1
-2::62::1
-2::64::1
-2::65::1
-2::66::3
-2::68::1
-2::71::3
-2::76::1
-2::77::1
-2::78::1
-2::80::1
-2::83::5
-2::85::1
-2::87::2
-2::88::1
-2::89::4
-2::90::1
-2::92::4
-2::93::5
-3::0::1
-3::1::1
-3::2::1
-3::7::3
-3::8::3
-3::9::1
-3::14::1
-3::15::1
-3::16::1
-3::18::4
-3::19::1
-3::24::3
-3::26::1
-3::29::3
-3::33::1
-3::34::3
-3::35::1
-3::36::3
-3::37::1
-3::38::2
-3::43::1
-3::44::1
-3::46::1
-3::47::1
-3::51::5
-3::52::3
-3::56::1
-3::58::1
-3::60::3
-3::62::1
-3::65::2
-3::66::1
-3::67::1
-3::68::2
-3::70::1
-3::72::2
-3::76::3
-3::79::3
-3::80::4
-3::81::1
-3::83::1
-3::84::1
-3::86::1
-3::87::2
-3::88::4
-3::89::1
-3::91::1
-3::94::3
-4::1::1
-4::6::1
-4::8::1
-4::9::1
-4::10::1
-4::11::1
-4::12::1
-4::13::1
-4::14::2
-4::15::1
-4::17::1
-4::20::1
-4::22::1
-4::23::1
-4::24::1
-4::29::4
-4::30::1
-4::31::1
-4::34::1
-4::35::1
-4::36::1
-4::39::2
-4::40::3
-4::41::4
-4::43::2
-4::44::1
-4::45::1
-4::46::1
-4::47::1
-4::49::2
-4::50::1
-4::51::1
-4::52::4
-4::54::1
-4::55::1
-4::60::3
-4::61::1
-4::62::4
-4::63::3
-4::65::1
-4::67::2
-4::69::1
-4::70::4
-4::71::1
-4::73::1
-4::78::1
-4::84::1
-4::85::1
-4::87::3
-4::88::3
-4::89::2
-4::96::1
-4::97::1
-4::98::1
-4::99::1
-5::0::1
-5::1::1
-5::4::1
-5::5::1
-5::8::1
-5::9::3
-5::10::2
-5::13::3
-5::15::1
-5::19::1
-5::20::3
-5::21::2
-5::23::3
-5::27::1
-5::28::1
-5::29::1
-5::31::1
-5::36::3
-5::38::2
-5::39::1
-5::42::1
-5::48::3
-5::49::4
-5::50::3
-5::51::1
-5::52::1
-5::54::1
-5::55::5
-5::56::3
-5::58::1
-5::60::1
-5::61::1
-5::64::3
-5::65::2
-5::68::4
-5::70::1
-5::71::1
-5::72::1
-5::74::1
-5::79::1
-5::81::2
-5::84::1
-5::85::1
-5::86::1
-5::88::1
-5::90::4
-5::91::2
-5::95::2
-5::99::1
-6::0::1
-6::1::1
-6::2::3
-6::5::1
-6::6::1
-6::9::1
-6::10::1
-6::15::2
-6::16::2
-6::17::1
-6::18::1
-6::20::1
-6::21::1
-6::22::1
-6::24::1
-6::25::5
-6::26::1
-6::28::1
-6::30::1
-6::33::1
-6::38::1
-6::39::1
-6::43::4
-6::44::1
-6::45::1
-6::48::1
-6::49::1
-6::50::1
-6::53::1
-6::54::1
-6::55::1
-6::56::1
-6::58::4
-6::59::1
-6::60::1
-6::61::3
-6::63::3
-6::66::1
-6::67::3
-6::68::1
-6::69::1
-6::71::2
-6::73::1
-6::75::1
-6::77::1
-6::79::1
-6::81::1
-6::84::1
-6::85::3
-6::86::1
-6::87::1
-6::88::1
-6::89::1
-6::91::2
-6::94::1
-6::95::2
-6::96::1
-7::1::1
-7::2::2
-7::3::1
-7::4::1
-7::7::1
-7::10::1
-7::11::2
-7::14::2
-7::15::1
-7::16::1
-7::18::1
-7::21::1
-7::22::1
-7::23::1
-7::25::5
-7::26::1
-7::29::4
-7::30::1
-7::31::3
-7::32::1
-7::33::1
-7::35::1
-7::37::2
-7::39::3
-7::40::2
-7::42::2
-7::44::1
-7::45::2
-7::47::4
-7::48::1
-7::49::1
-7::53::1
-7::54::1
-7::55::1
-7::56::1
-7::59::1
-7::61::2
-7::62::3
-7::63::2
-7::66::1
-7::67::3
-7::74::1
-7::75::1
-7::76::3
-7::77::1
-7::81::1
-7::82::1
-7::84::2
-7::85::4
-7::86::1
-7::92::2
-7::96::1
-7::97::1
-7::98::1
-8::0::1
-8::2::4
-8::3::2
-8::4::2
-8::5::1
-8::7::1
-8::9::1
-8::11::1
-8::15::1
-8::18::1
-8::19::1
-8::21::1
-8::29::5
-8::31::3
-8::33::1
-8::35::1
-8::36::1
-8::40::2
-8::44::1
-8::45::1
-8::50::1
-8::51::1
-8::52::5
-8::53::5
-8::54::1
-8::55::1
-8::56::1
-8::58::4
-8::60::3
-8::62::4
-8::64::1
-8::67::3
-8::69::1
-8::71::1
-8::72::3
-8::77::3
-8::78::1
-8::79::1
-8::83::1
-8::85::5
-8::86::1
-8::88::1
-8::90::1
-8::92::2
-8::95::4
-8::96::3
-8::97::1
-8::98::1
-8::99::1
-9::2::3
-9::3::1
-9::4::1
-9::5::1
-9::6::1
-9::7::5
-9::9::1
-9::12::1
-9::14::3
-9::15::1
-9::19::1
-9::21::1
-9::22::1
-9::24::1
-9::25::1
-9::26::1
-9::30::3
-9::32::4
-9::35::2
-9::36::2
-9::37::2
-9::38::1
-9::39::1
-9::43::3
-9::49::5
-9::50::3
-9::53::1
-9::54::1
-9::58::1
-9::59::1
-9::60::1
-9::61::1
-9::63::3
-9::64::3
-9::68::1
-9::69::1
-9::70::3
-9::71::1
-9::73::2
-9::75::1
-9::77::2
-9::81::2
-9::82::1
-9::83::1
-9::84::1
-9::86::1
-9::87::4
-9::88::1
-9::90::3
-9::94::2
-9::95::3
-9::97::2
-9::98::1
-10::0::3
-10::2::4
-10::4::3
-10::7::1
-10::8::1
-10::10::1
-10::13::2
-10::14::1
-10::16::2
-10::17::1
-10::18::1
-10::21::1
-10::22::1
-10::24::1
-10::25::3
-10::28::1
-10::35::1
-10::36::1
-10::37::1
-10::38::1
-10::39::1
-10::40::4
-10::41::2
-10::42::3
-10::43::1
-10::49::3
-10::50::1
-10::51::1
-10::52::1
-10::55::2
-10::56::1
-10::58::1
-10::63::1
-10::66::1
-10::67::2
-10::68::1
-10::75::1
-10::77::1
-10::79::1
-10::86::1
-10::89::3
-10::90::1
-10::97::1
-10::98::1
-11::0::1
-11::6::2
-11::9::1
-11::10::1
-11::11::1
-11::12::1
-11::13::4
-11::16::1
-11::18::5
-11::19::4
-11::20::1
-11::21::1
-11::22::1
-11::23::5
-11::25::1
-11::27::5
-11::30::5
-11::32::5
-11::35::3
-11::36::2
-11::37::2
-11::38::4
-11::39::1
-11::40::1
-11::41::1
-11::43::2
-11::45::1
-11::47::1
-11::48::5
-11::50::4
-11::51::3
-11::59::1
-11::61::1
-11::62::1
-11::64::1
-11::66::4
-11::67::1
-11::69::5
-11::70::1
-11::71::3
-11::72::3
-11::75::3
-11::76::1
-11::77::1
-11::78::1
-11::79::5
-11::80::3
-11::81::4
-11::82::1
-11::86::1
-11::88::1
-11::89::1
-11::90::4
-11::94::2
-11::97::3
-11::99::1
-12::2::1
-12::4::1
-12::6::1
-12::7::3
-12::8::1
-12::14::1
-12::15::2
-12::16::4
-12::17::5
-12::18::2
-12::21::1
-12::22::2
-12::23::3
-12::24::1
-12::25::1
-12::27::5
-12::30::2
-12::31::4
-12::35::5
-12::38::1
-12::41::1
-12::44::2
-12::45::1
-12::50::4
-12::51::1
-12::52::1
-12::53::1
-12::54::1
-12::56::2
-12::57::1
-12::60::1
-12::63::1
-12::64::5
-12::66::3
-12::67::1
-12::70::1
-12::72::1
-12::74::1
-12::75::1
-12::77::1
-12::78::1
-12::79::3
-12::82::2
-12::83::1
-12::84::1
-12::85::1
-12::86::1
-12::87::1
-12::88::1
-12::91::3
-12::92::1
-12::94::4
-12::95::2
-12::96::1
-12::98::2
-13::0::1
-13::3::1
-13::4::2
-13::5::1
-13::6::1
-13::12::1
-13::14::2
-13::15::1
-13::17::1
-13::18::3
-13::20::1
-13::21::1
-13::22::1
-13::26::1
-13::27::1
-13::29::3
-13::31::1
-13::33::1
-13::40::2
-13::43::2
-13::44::1
-13::45::1
-13::49::1
-13::51::1
-13::52::2
-13::53::3
-13::54::1
-13::62::1
-13::63::2
-13::64::1
-13::68::1
-13::71::1
-13::72::3
-13::73::1
-13::74::3
-13::77::2
-13::78::1
-13::79::2
-13::83::3
-13::85::1
-13::86::1
-13::87::2
-13::88::2
-13::90::1
-13::93::4
-13::94::1
-13::98::1
-13::99::1
-14::1::1
-14::3::3
-14::4::1
-14::5::1
-14::6::1
-14::7::1
-14::9::1
-14::10::1
-14::11::1
-14::12::1
-14::13::1
-14::14::3
-14::15::1
-14::16::1
-14::17::1
-14::20::1
-14::21::1
-14::24::1
-14::25::2
-14::27::1
-14::28::1
-14::29::5
-14::31::3
-14::34::1
-14::36::1
-14::37::2
-14::39::2
-14::40::1
-14::44::1
-14::45::1
-14::47::3
-14::48::1
-14::49::1
-14::51::1
-14::52::5
-14::53::3
-14::54::1
-14::55::1
-14::56::1
-14::62::4
-14::63::5
-14::67::3
-14::68::1
-14::69::3
-14::71::1
-14::72::4
-14::73::1
-14::76::5
-14::79::1
-14::82::1
-14::83::1
-14::88::1
-14::93::3
-14::94::1
-14::95::2
-14::96::4
-14::98::1
-15::0::1
-15::1::4
-15::2::1
-15::5::2
-15::6::1
-15::7::1
-15::13::1
-15::14::1
-15::15::1
-15::17::2
-15::19::2
-15::22::2
-15::23::2
-15::25::1
-15::26::3
-15::27::1
-15::28::2
-15::29::1
-15::32::1
-15::33::2
-15::34::1
-15::35::2
-15::36::1
-15::37::1
-15::39::1
-15::42::1
-15::46::5
-15::48::2
-15::50::2
-15::51::1
-15::52::1
-15::58::1
-15::62::1
-15::64::3
-15::65::2
-15::72::1
-15::73::1
-15::74::1
-15::79::1
-15::80::1
-15::81::1
-15::82::2
-15::85::1
-15::87::1
-15::91::2
-15::96::1
-15::97::1
-15::98::3
-16::2::1
-16::5::3
-16::6::2
-16::7::1
-16::9::1
-16::12::1
-16::14::1
-16::15::1
-16::19::1
-16::21::2
-16::29::4
-16::30::2
-16::32::1
-16::34::1
-16::36::1
-16::38::1
-16::46::1
-16::47::3
-16::48::1
-16::49::1
-16::50::1
-16::51::5
-16::54::5
-16::55::1
-16::56::2
-16::57::1
-16::60::1
-16::63::2
-16::65::1
-16::67::1
-16::72::1
-16::74::1
-16::80::1
-16::81::1
-16::82::1
-16::85::5
-16::86::1
-16::90::5
-16::91::1
-16::93::1
-16::94::3
-16::95::2
-16::96::3
-16::98::3
-16::99::1
-17::2::1
-17::3::1
-17::6::1
-17::10::4
-17::11::1
-17::13::2
-17::17::5
-17::19::1
-17::20::5
-17::22::4
-17::28::1
-17::29::1
-17::33::1
-17::34::1
-17::35::2
-17::37::1
-17::38::1
-17::45::1
-17::46::5
-17::47::1
-17::49::3
-17::51::1
-17::55::5
-17::56::3
-17::57::1
-17::58::1
-17::59::1
-17::60::1
-17::63::1
-17::66::1
-17::68::4
-17::69::1
-17::70::1
-17::72::1
-17::73::3
-17::78::1
-17::79::1
-17::82::2
-17::84::1
-17::90::5
-17::91::3
-17::92::1
-17::93::1
-17::94::4
-17::95::2
-17::97::1
-18::1::1
-18::4::3
-18::5::2
-18::6::1
-18::7::1
-18::10::1
-18::11::4
-18::12::2
-18::13::1
-18::15::1
-18::18::1
-18::20::1
-18::21::2
-18::22::1
-18::23::2
-18::25::1
-18::26::1
-18::27::1
-18::28::5
-18::29::1
-18::31::1
-18::32::1
-18::36::1
-18::38::5
-18::39::5
-18::40::1
-18::42::1
-18::43::1
-18::44::4
-18::46::1
-18::47::1
-18::48::1
-18::51::2
-18::55::1
-18::56::1
-18::57::1
-18::62::1
-18::63::1
-18::66::3
-18::67::1
-18::70::1
-18::75::1
-18::76::3
-18::77::1
-18::80::3
-18::81::3
-18::82::1
-18::83::5
-18::84::1
-18::97::1
-18::98::1
-18::99::2
-19::0::1
-19::1::1
-19::2::1
-19::4::1
-19::6::2
-19::11::1
-19::12::1
-19::14::1
-19::23::1
-19::26::1
-19::31::1
-19::32::4
-19::33::1
-19::34::1
-19::37::1
-19::38::1
-19::41::1
-19::43::1
-19::45::1
-19::48::1
-19::49::1
-19::50::2
-19::53::2
-19::54::3
-19::55::1
-19::56::2
-19::58::1
-19::61::1
-19::62::1
-19::63::1
-19::64::1
-19::65::1
-19::69::2
-19::72::1
-19::74::3
-19::76::1
-19::78::1
-19::79::1
-19::81::1
-19::82::1
-19::84::1
-19::86::1
-19::87::2
-19::90::4
-19::93::1
-19::94::4
-19::95::2
-19::96::1
-19::98::4
-20::0::1
-20::1::1
-20::2::2
-20::4::2
-20::6::1
-20::8::1
-20::12::1
-20::21::2
-20::22::5
-20::24::2
-20::25::1
-20::26::1
-20::29::2
-20::30::2
-20::32::2
-20::39::1
-20::40::1
-20::41::2
-20::45::2
-20::48::1
-20::50::1
-20::51::3
-20::53::3
-20::55::1
-20::57::2
-20::60::1
-20::61::1
-20::64::1
-20::66::1
-20::70::2
-20::72::1
-20::73::2
-20::75::4
-20::76::1
-20::77::4
-20::78::1
-20::79::1
-20::84::2
-20::85::2
-20::88::3
-20::89::1
-20::90::3
-20::91::1
-20::92::2
-20::93::1
-20::94::4
-20::97::1
-21::0::1
-21::2::4
-21::3::1
-21::7::2
-21::11::1
-21::12::1
-21::13::1
-21::14::3
-21::17::1
-21::19::1
-21::20::1
-21::21::1
-21::22::1
-21::23::1
-21::24::1
-21::27::1
-21::29::5
-21::30::2
-21::38::1
-21::40::2
-21::43::3
-21::44::1
-21::45::1
-21::46::1
-21::48::1
-21::51::1
-21::53::5
-21::54::1
-21::55::1
-21::56::1
-21::58::3
-21::59::3
-21::64::1
-21::66::1
-21::68::1
-21::71::1
-21::73::1
-21::74::4
-21::80::1
-21::81::1
-21::83::1
-21::84::1
-21::85::3
-21::87::4
-21::89::2
-21::92::2
-21::96::3
-21::99::1
-22::0::1
-22::3::2
-22::5::2
-22::6::2
-22::9::1
-22::10::1
-22::11::1
-22::13::1
-22::14::1
-22::16::1
-22::18::3
-22::19::1
-22::22::5
-22::25::1
-22::26::1
-22::29::3
-22::30::5
-22::32::4
-22::33::1
-22::35::1
-22::36::3
-22::37::1
-22::40::1
-22::41::3
-22::44::1
-22::45::2
-22::48::1
-22::51::5
-22::55::1
-22::56::2
-22::60::3
-22::61::1
-22::62::4
-22::63::1
-22::65::1
-22::66::1
-22::68::4
-22::69::4
-22::70::3
-22::71::1
-22::74::5
-22::75::5
-22::78::1
-22::80::3
-22::81::1
-22::82::1
-22::84::1
-22::86::1
-22::87::3
-22::88::5
-22::90::2
-22::92::3
-22::95::2
-22::96::2
-22::98::4
-22::99::1
-23::0::1
-23::2::1
-23::4::1
-23::6::2
-23::10::4
-23::12::1
-23::13::4
-23::14::1
-23::15::1
-23::18::4
-23::22::2
-23::23::4
-23::24::1
-23::25::1
-23::26::1
-23::27::5
-23::28::1
-23::29::1
-23::30::4
-23::32::5
-23::33::2
-23::36::3
-23::37::1
-23::38::1
-23::39::1
-23::43::1
-23::48::5
-23::49::5
-23::50::4
-23::53::1
-23::55::5
-23::57::1
-23::59::1
-23::60::1
-23::61::1
-23::64::4
-23::65::5
-23::66::2
-23::67::1
-23::68::3
-23::69::1
-23::72::1
-23::73::3
-23::77::1
-23::82::2
-23::83::1
-23::84::1
-23::85::1
-23::87::3
-23::88::1
-23::95::2
-23::97::1
-24::4::1
-24::6::3
-24::7::1
-24::10::2
-24::12::1
-24::15::1
-24::19::1
-24::24::1
-24::27::3
-24::30::5
-24::31::1
-24::32::3
-24::33::1
-24::37::1
-24::39::1
-24::40::1
-24::42::1
-24::43::3
-24::45::2
-24::46::1
-24::47::1
-24::48::1
-24::49::1
-24::50::1
-24::52::5
-24::57::1
-24::59::4
-24::63::4
-24::65::1
-24::66::1
-24::67::1
-24::68::3
-24::69::5
-24::71::1
-24::72::4
-24::77::4
-24::78::1
-24::80::1
-24::82::1
-24::84::1
-24::86::1
-24::87::1
-24::88::2
-24::89::1
-24::90::5
-24::91::1
-24::92::1
-24::94::2
-24::95::1
-24::96::5
-24::98::1
-24::99::1
-25::1::3
-25::2::1
-25::7::1
-25::9::1
-25::12::3
-25::16::3
-25::17::1
-25::18::1
-25::20::1
-25::22::1
-25::23::1
-25::26::2
-25::29::1
-25::30::1
-25::31::2
-25::33::4
-25::34::3
-25::35::2
-25::36::1
-25::37::1
-25::40::1
-25::41::1
-25::43::1
-25::47::4
-25::50::1
-25::51::1
-25::53::1
-25::56::1
-25::58::2
-25::64::2
-25::67::2
-25::68::1
-25::70::1
-25::71::4
-25::73::1
-25::74::1
-25::76::1
-25::79::1
-25::82::1
-25::84::2
-25::85::1
-25::91::3
-25::92::1
-25::94::1
-25::95::1
-25::97::2
-26::0::1
-26::1::1
-26::2::1
-26::3::1
-26::4::4
-26::5::2
-26::6::3
-26::7::5
-26::13::3
-26::14::1
-26::16::1
-26::18::3
-26::20::1
-26::21::3
-26::22::5
-26::23::5
-26::24::5
-26::27::1
-26::31::1
-26::35::1
-26::36::4
-26::40::1
-26::44::1
-26::45::2
-26::47::1
-26::48::1
-26::49::3
-26::50::2
-26::52::1
-26::54::4
-26::55::1
-26::57::3
-26::58::1
-26::61::1
-26::62::2
-26::66::1
-26::68::4
-26::71::1
-26::73::4
-26::76::1
-26::81::3
-26::85::1
-26::86::3
-26::88::5
-26::91::1
-26::94::5
-26::95::1
-26::96::1
-26::97::1
-27::0::1
-27::9::1
-27::10::1
-27::18::4
-27::19::3
-27::20::1
-27::22::2
-27::24::2
-27::25::1
-27::27::3
-27::28::1
-27::29::1
-27::31::1
-27::33::3
-27::40::1
-27::42::1
-27::43::1
-27::44::3
-27::45::1
-27::51::3
-27::52::1
-27::55::3
-27::57::1
-27::59::1
-27::60::1
-27::61::1
-27::64::1
-27::66::3
-27::68::1
-27::70::1
-27::71::2
-27::72::1
-27::75::3
-27::78::1
-27::80::3
-27::82::1
-27::83::3
-27::86::1
-27::87::2
-27::90::1
-27::91::1
-27::92::1
-27::93::1
-27::94::2
-27::95::1
-27::98::1
-28::0::3
-28::1::1
-28::2::4
-28::3::1
-28::6::1
-28::7::1
-28::12::5
-28::13::2
-28::14::1
-28::15::1
-28::17::1
-28::19::3
-28::20::1
-28::23::3
-28::24::3
-28::27::1
-28::29::1
-28::33::1
-28::34::1
-28::36::1
-28::38::2
-28::39::2
-28::44::1
-28::45::1
-28::49::4
-28::50::1
-28::52::1
-28::54::1
-28::56::1
-28::57::3
-28::58::1
-28::59::1
-28::60::1
-28::62::3
-28::63::1
-28::65::1
-28::75::1
-28::78::1
-28::81::5
-28::82::4
-28::83::1
-28::85::1
-28::88::2
-28::89::4
-28::90::1
-28::92::5
-28::94::1
-28::95::2
-28::98::1
-28::99::1
-29::3::1
-29::4::1
-29::5::1
-29::7::2
-29::9::1
-29::10::3
-29::11::1
-29::13::3
-29::14::1
-29::15::1
-29::17::3
-29::19::3
-29::22::3
-29::23::4
-29::25::1
-29::29::1
-29::31::1
-29::32::4
-29::33::2
-29::36::2
-29::38::3
-29::39::1
-29::42::1
-29::46::5
-29::49::3
-29::51::2
-29::59::1
-29::61::1
-29::62::1
-29::67::1
-29::68::3
-29::69::1
-29::70::1
-29::74::1
-29::75::1
-29::79::2
-29::80::1
-29::81::2
-29::83::1
-29::85::1
-29::86::1
-29::90::4
-29::93::1
-29::94::4
-29::97::1
-29::99::1
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/MyAlgorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/MyAlgorithm.java b/examples/experimental/java-local-helloworld/MyAlgorithm.java
deleted file mode 100644
index 08084aa..0000000
--- a/examples/experimental/java-local-helloworld/MyAlgorithm.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.java.helloworld;
-
-import org.apache.predictionio.controller.java.*;
-
-import java.util.Map;
-import java.util.HashMap;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MyAlgorithm extends LJavaAlgorithm<
- EmptyAlgorithmParams, MyTrainingData, MyModel, MyQuery, MyPredictedResult> {
-
- final static Logger logger = LoggerFactory.getLogger(MyAlgorithm.class);
-
- @Override
- public MyModel train(MyTrainingData data) {
- Map<String, Double> sumMap = new HashMap<String, Double>();
- Map<String, Integer> countMap = new HashMap<String, Integer>();
-
- // calculate sum and count for each day
- for (MyTrainingData.DayTemperature temp : data.temperatures) {
- Double sum = sumMap.get(temp.day);
- Integer count = countMap.get(temp.day);
- if (sum == null) {
- sumMap.put(temp.day, temp.temperature);
- countMap.put(temp.day, 1);
- } else {
- sumMap.put(temp.day, sum + temp.temperature);
- countMap.put(temp.day, count + 1);
- }
- }
-
- // calculate the average
- Map<String, Double> averageMap = new HashMap<String, Double>();
- for (Map.Entry<String, Double> entry : sumMap.entrySet()) {
- String day = entry.getKey();
- Double average = entry.getValue() / countMap.get(day);
- averageMap.put(day, average);
- }
-
- return new MyModel(averageMap);
- }
-
- @Override
- public MyPredictedResult predict(MyModel model, MyQuery query) {
- Double temp = model.temperatures.get(query.day);
- return new MyPredictedResult(temp);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/MyDataSource.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/MyDataSource.java b/examples/experimental/java-local-helloworld/MyDataSource.java
deleted file mode 100644
index e942de0..0000000
--- a/examples/experimental/java-local-helloworld/MyDataSource.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.java.helloworld;
-
-import org.apache.predictionio.controller.java.*;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.io.FileReader;
-import java.io.BufferedReader;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MyDataSource extends LJavaDataSource<
- EmptyDataSourceParams, EmptyDataParams, MyTrainingData, MyQuery, EmptyActualResult> {
-
- final static Logger logger = LoggerFactory.getLogger(MyDataSource.class);
-
- @Override
- public MyTrainingData readTraining() {
- List<MyTrainingData.DayTemperature> temperatures =
- new ArrayList<MyTrainingData.DayTemperature>();
-
- try {
- BufferedReader reader = new BufferedReader(new FileReader("../data/helloworld/data.csv"));
- String line;
- while ((line = reader.readLine()) != null) {
- String[] tokens = line.split(",");
- temperatures.add(
- new MyTrainingData.DayTemperature(tokens[0], Double.parseDouble(tokens[1])));
- }
- reader.close();
- } catch (Exception e) {
- logger.error("Caught Exception: " + e.getMessage());
- System.exit(1);
- }
-
- return new MyTrainingData(temperatures);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/MyEngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/MyEngineFactory.java b/examples/experimental/java-local-helloworld/MyEngineFactory.java
deleted file mode 100644
index 6c28cb6..0000000
--- a/examples/experimental/java-local-helloworld/MyEngineFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.java.helloworld;
-
-import org.apache.predictionio.controller.java.*;
-
-public class MyEngineFactory implements IJavaEngineFactory {
- public JavaSimpleEngine<MyTrainingData, EmptyDataParams, MyQuery,
- MyPredictedResult, EmptyActualResult> apply() {
-
- return new JavaSimpleEngineBuilder<MyTrainingData, EmptyDataParams,
- MyQuery, MyPredictedResult, EmptyActualResult> ()
- .dataSourceClass(MyDataSource.class)
- .preparatorClass() // Use default Preparator
- .addAlgorithmClass("", MyAlgorithm.class)
- .servingClass() // Use default Serving
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/MyModel.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/MyModel.java b/examples/experimental/java-local-helloworld/MyModel.java
deleted file mode 100644
index 07ccd34..0000000
--- a/examples/experimental/java-local-helloworld/MyModel.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.java.helloworld;
-
-import java.io.Serializable;
-import java.util.Map;
-
-public class MyModel implements Serializable {
- Map<String, Double> temperatures;
-
- public MyModel(Map<String, Double> temperatures) {
- this.temperatures = temperatures;
- }
-
- @Override
- public String toString() {
- return temperatures.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/MyPredictedResult.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/MyPredictedResult.java b/examples/experimental/java-local-helloworld/MyPredictedResult.java
deleted file mode 100644
index 129a5f9..0000000
--- a/examples/experimental/java-local-helloworld/MyPredictedResult.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.java.helloworld;
-
-import java.io.Serializable;
-
-public class MyPredictedResult implements Serializable {
- Double temperature;
-
- public MyPredictedResult(Double temperature) {
- this.temperature = temperature;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/MyQuery.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/MyQuery.java b/examples/experimental/java-local-helloworld/MyQuery.java
deleted file mode 100644
index d98e9dc..0000000
--- a/examples/experimental/java-local-helloworld/MyQuery.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.java.helloworld;
-
-import java.io.Serializable;
-
-public class MyQuery implements Serializable {
- String day;
-
- public MyQuery(String day) {
- this.day = day;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/MyTrainingData.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/MyTrainingData.java b/examples/experimental/java-local-helloworld/MyTrainingData.java
deleted file mode 100644
index adca58f..0000000
--- a/examples/experimental/java-local-helloworld/MyTrainingData.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sample.java.helloworld;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class MyTrainingData implements Serializable {
- List<DayTemperature> temperatures;
-
- public MyTrainingData(List<DayTemperature> temperatures) {
- this.temperatures = temperatures;
- }
-
- public static class DayTemperature implements Serializable {
- String day;
- Double temperature;
-
- public DayTemperature(String day, Double temperature) {
- this.day = day;
- this.temperature = temperature;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/README.md b/examples/experimental/java-local-helloworld/README.md
deleted file mode 100644
index 245ba35..0000000
--- a/examples/experimental/java-local-helloworld/README.md
+++ /dev/null
@@ -1,93 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# My First "Hello World" Engine
-
-```
-$ cd $PIO_HOME/examples/java-local-helloworld
-```
-
-Prepare training data:
-```
-$ cp ../data/helloworld/data1.csv ../data/helloworld/data.csv
-```
-
-Register engine:
-
-```
-$ ../../bin/pio build
-
-```
-
-Train:
-
-```
-$ ../../bin/pio train
-```
-
-Example output:
-
-```
-2014-08-11 14:29:35,877 INFO APIDebugWorkflow$ - Metrics is null. Stop here
-2014-08-11 14:29:36,099 INFO APIDebugWorkflow$ - Saved engine instance with ID: 201408110004
-```
-
-```
-$ ../../bin/pio deploy
-```
-
-Retrieve prediction:
-
-```
-$ curl -H "Content-Type: application/json" -d '{ "day": "Mon" }' http://localhost:8000/queries.json
-```
-
-Output:
-
-```
-{"temperature":75.5}
-```
-
-Retrieve prediction:
-
-```
-$ curl -H "Content-Type: application/json" -d '{ "day": "Tue" }' http://localhost:8000/queries.json
-```
-
-Output:
-```
-{"temperature":80.5}
-```
-
-
-Re-train with new data:
-
-```
-$ cd $PIO_HOME/examples/java-local-helloworld
-$ cp ../data/helloworld/data2.csv ../data/helloworld/data.csv
-```
-
-```
-$ ../../bin/pio train
-$ ../../bin/pio deploy
-```
-
-````
-$ curl -H "Content-Type: application/json" -d '{ "day": "Mon" }' http://localhost:8000/queries.json
-
-{"temperature":76.66666666666667}
-````
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/build.sbt b/examples/experimental/java-local-helloworld/build.sbt
deleted file mode 100644
index b246d7f..0000000
--- a/examples/experimental/java-local-helloworld/build.sbt
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "example-java-local-helloworld"
-
-organization := "org.sample"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/engine.json b/examples/experimental/java-local-helloworld/engine.json
deleted file mode 100644
index a00e45b..0000000
--- a/examples/experimental/java-local-helloworld/engine.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "id": "org.sample.java.helloworld",
- "description": "My Hello World Engine",
- "engineFactory": "org.sample.java.helloworld.MyEngineFactory"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-helloworld/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-helloworld/project/assembly.sbt b/examples/experimental/java-local-helloworld/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/java-local-helloworld/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/README.md b/examples/experimental/java-local-regression/README.md
deleted file mode 100644
index ebf45ab..0000000
--- a/examples/experimental/java-local-regression/README.md
+++ /dev/null
@@ -1,96 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Java Local Regression Engine
-
-## Configuration
-
-### Data Source
-
-This sample regression engine reads data from file system.
-
-Edit the file path in `engine.json`, change `filepath` of `datasource` to an absolute path that points to
-[lr_data.py](../data/lr_data.txt)
-
-```
-$ cat engine.json
-...
-"datasource": {
- "filepath": <absolute_path_to_lr_data.txt>
-}
-...
-
-```
-
-### Algorithms
-
-This engine comes with two algorithms.
-
-1. OLS. It is the standard ordinary least square algorithm. It takes no parameter.
-2. Default. This algorithm always return the same value `v` defined in the parameter.
-
-Below is an example of `algorithms` defines in `engine.json`, it invokes three algorithms,
-
-1. The standard OLS algorithm,
-2. A default algorithm which always return 2.0,
-3. A default algorithm which always return 4.0.
-
-```json
-"algorithms": [
- { "name": "OLS", "params": {} },
- { "name": "Default", "params": { "v": 2.0 } },
- { "name": "Default", "params": { "v": 4.0 } }
-]
-```
-
-The `Serving` class returns the average of all these prediction to the user.
-
-## Register engine, train, and deploy.
-
-```
-$ pio build
-$ pio train
-$ pio deploy --port 9997
-```
-
-## Query the Engine Instance
-
-```
-$ curl -X POST http://localhost:9997/queries.json -d \
- '[1.80,0.87,2.41,0.35,-0.21,1.35,0.51,1.55,-0.20,1.32]'
-2.2434392991944025
-```
-
-2.24... is the prediction result.
-
-## Evaluation
-
-You can evaluate the engine instance too.
-```
-$ pio eval --batch JavaRegressionEval \
---metrics-class org.apache.predictionio.examples.java.regression.MeanSquareMetrics
-
-...
-2014-09-24 03:23:07,170 INFO spark.SparkContext - Job finished: collect at Workflow.scala:695, took 0.092829 s
-2014-09-24 03:23:07,284 WARN workflow.CoreWorkflow$ - java.lang.String is not a NiceRendering instance.
-2014-09-24 03:23:07,296 INFO workflow.CoreWorkflow$ - Saved engine instance with ID: OCCUucs7QBOOG--9kIWFEw
-```
-
-And you will see you result in PredictionIO dashboard. To start dashboard:
-```
-$ pio dashboard
-```
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/build.sbt b/examples/experimental/java-local-regression/build.sbt
deleted file mode 100644
index 92d7416..0000000
--- a/examples/experimental/java-local-regression/build.sbt
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "java-local-regression"
-
-organization := "myorg"
-
-version := "0.0.1-SNAPSHOT"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/engine.json b/examples/experimental/java-local-regression/engine.json
deleted file mode 100644
index a80ef7d..0000000
--- a/examples/experimental/java-local-regression/engine.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "id": "java-local-regression",
- "description": "java-local-regression",
- "engineFactory": "org.apache.predictionio.examples.java.regression.EngineFactory",
- "datasource": {
- "params": {
- "filepath": "PATH-TO-lr_data.txt"
- }
- },
- "preparator": {
- "params": {
- "r": 0.8
- }
- },
- "algorithms": [
- {
- "name": "OLS",
- "params": {}
- },
- {
- "name": "Default",
- "params": {
- "v": 2.0
- }
- },
- {
- "name": "Default",
- "params": {
- "v": 4.0
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/project/assembly.sbt b/examples/experimental/java-local-regression/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/java-local-regression/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/DataSource.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/DataSource.java b/examples/experimental/java-local-regression/src/main/java/DataSource.java
deleted file mode 100644
index 62be375..0000000
--- a/examples/experimental/java-local-regression/src/main/java/DataSource.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.LJavaDataSource;
-
-import scala.Tuple2;
-import scala.Tuple3;
-
-import java.io.IOException;
-import java.lang.Iterable;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Pattern;
-
-public class DataSource
- extends LJavaDataSource<DataSourceParams, Integer, TrainingData, Double[], Double> {
-
- private static final Pattern SPACE = Pattern.compile(" ");
-
- public final DataSourceParams dsp;
-
- public DataSource(DataSourceParams dsp) {
- this.dsp = dsp;
- }
-
- public List<Tuple3<Integer, TrainingData, Iterable<Tuple2<Double[], Double>>>> read() {
-
- List<String> lines;
- try {
- lines = Files.readAllLines(Paths.get(dsp.filepath), StandardCharsets.UTF_8);
- } catch (IOException exception) {
- System.out.println("Cannot read file");
- lines = new ArrayList<String>();
- }
-
- int n = lines.size();
-
- int featureCount = SPACE.split(lines.get(0)).length - 1;
-
- Double[][] x = new Double[n][featureCount];
- Double[] y = new Double[n];
-
- for (int i = 0; i < n; i++) {
- String[] line = SPACE.split(lines.get(i), 2);
- y[i] = Double.parseDouble(line[0]);
-
- String[] featureStrs = SPACE.split(line[1]);
-
- for (int j = 0; j < featureCount; j++) {
- x[i][j] = Double.parseDouble(featureStrs[j]);
- }
- }
-
- TrainingData td = new TrainingData(x, y);
-
- List<Tuple2<Double[], Double>> faList = new ArrayList<>();
-
- for (int i = 0; i < 10; i++) {
- faList.add(new Tuple2<Double[], Double>(x[i], y[i]));
- }
-
- List<Tuple3<Integer, TrainingData, Iterable<Tuple2<Double[], Double>>>> results =
- new ArrayList <> ();
-
- results.add(new Tuple3<Integer, TrainingData, Iterable<Tuple2<Double[], Double>>>(
- new Integer(0), td, faList));
-
- return results;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/DataSourceParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/DataSourceParams.java b/examples/experimental/java-local-regression/src/main/java/DataSourceParams.java
deleted file mode 100644
index 1afc919..0000000
--- a/examples/experimental/java-local-regression/src/main/java/DataSourceParams.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class DataSourceParams implements JavaParams {
- public final String filepath;
- public DataSourceParams(String filepath) {
- this.filepath = filepath;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithm.java b/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithm.java
deleted file mode 100644
index 055e5e6..0000000
--- a/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithm.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.LJavaAlgorithm;
-
-// This algorithm is for illustration only. It returns a constant.
-public class DefaultAlgorithm
- extends LJavaAlgorithm<DefaultAlgorithmParams, TrainingData, Object, Double[], Double> {
- public final DefaultAlgorithmParams p;
-
- public DefaultAlgorithm(DefaultAlgorithmParams p) {
- this.p = p;
- }
-
- public Object train(TrainingData data) {
- return null;
- }
-
- public Double predict(Object nullModel, Double[] query) {
- return p.v;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithmParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithmParams.java b/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithmParams.java
deleted file mode 100644
index 28aba37..0000000
--- a/examples/experimental/java-local-regression/src/main/java/DefaultAlgorithmParams.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class DefaultAlgorithmParams implements JavaParams {
- public final double v;
- public DefaultAlgorithmParams(double v) {
- this.v = v;
- }
-
- @Override
- public String toString() {
- return "DefaultAlgorithmParams (k=" + this.v + ")";
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/EngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/EngineFactory.java b/examples/experimental/java-local-regression/src/main/java/EngineFactory.java
deleted file mode 100644
index 1fe03f3..0000000
--- a/examples/experimental/java-local-regression/src/main/java/EngineFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-
-public class EngineFactory implements IJavaEngineFactory {
- public JavaEngine<TrainingData, Integer, TrainingData, Double[], Double, Double> apply() {
- return new JavaEngineBuilder<TrainingData, Integer, TrainingData, Double[], Double, Double> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass(Preparator.class)
- .addAlgorithmClass("OLS", OLSAlgorithm.class)
- .addAlgorithmClass("Default", DefaultAlgorithm.class)
- .servingClass(Serving.class)
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/MeanSquareEvaluator.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/MeanSquareEvaluator.java b/examples/experimental/java-local-regression/src/main/java/MeanSquareEvaluator.java
deleted file mode 100644
index 13577e7..0000000
--- a/examples/experimental/java-local-regression/src/main/java/MeanSquareEvaluator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.JavaEvaluator;
-import java.lang.Iterable;
-import org.apache.predictionio.controller.java.EmptyParams;
-import scala.Tuple2;
-import java.util.List;
-import java.util.ArrayList;
-
-
-public class MeanSquareEvaluator
- extends JavaEvaluator<EmptyParams, Integer,
- Double[], Double, Double, Double, Double, String> {
-
- public Double evaluateUnit(Double[] query, Double prediction, Double actual) {
- return (prediction - actual) * (prediction - actual);
- }
-
- public Double evaluateSet(Integer dp, Iterable<Double> mseSeq) {
- double mse = 0.0;
- int n = 0;
- for (Double e: mseSeq) {
- mse += e;
- n += 1;
- }
- return mse / n;
- }
-
- public String evaluateAll(Iterable<Tuple2<Integer, Double>> input) {
- List<String> l = new ArrayList<String>();
- for (Tuple2<Integer, Double> t : input) {
- l.add("MSE: " + t._2().toString());
- }
- return l.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/OLSAlgorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/OLSAlgorithm.java b/examples/experimental/java-local-regression/src/main/java/OLSAlgorithm.java
deleted file mode 100644
index 6ed38f6..0000000
--- a/examples/experimental/java-local-regression/src/main/java/OLSAlgorithm.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression;
-import org.apache.commons.math3.linear.RealVector;
-import java.util.Arrays;
-import org.apache.predictionio.controller.java.EmptyParams;
-
-import org.apache.predictionio.controller.java.LJavaAlgorithm;
-
-public class OLSAlgorithm
- extends LJavaAlgorithm<EmptyParams, TrainingData, Double[], Double[], Double> {
-
- public Double[] train(TrainingData data) {
- OLSMultipleLinearRegression r = new OLSMultipleLinearRegression();
- // Convert Double[][] to double[][]
- double[][] x = new double[data.r][data.c];
- double[] y = new double[data.r];
-
- for (int i=0; i<data.r; i++) {
- for (int j=0; j<data.c; j++) {
- x[i][j] = data.x[i][j].doubleValue();
- }
- y[i] = data.y[i].doubleValue();
- }
-
- r.newSampleData(y, x);
- // Fixme. Add intercept
- double[] p = r.estimateRegressionParameters();
- Double[] pp = new Double[data.c];
- for (int j=0; j<data.c; j++) {
- pp[j] = p[j];
- }
- System.out.println("Regression Algo: " + Arrays.toString(pp));
- return pp;
- }
-
- public Double predict(Double[] model, Double[] query) {
- double dotProduct = 0.0;
- for (int i = 0; i < model.length; i++) {
- dotProduct += model[i] * query[i];
- }
- return dotProduct;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/Preparator.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/Preparator.java b/examples/experimental/java-local-regression/src/main/java/Preparator.java
deleted file mode 100644
index f4d46c1..0000000
--- a/examples/experimental/java-local-regression/src/main/java/Preparator.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.LJavaPreparator;
-
-// This Preparator is just a proof-of-concept. It removes a fraction of the
-// training data to make training more "efficient".
-public class Preparator extends LJavaPreparator<PreparatorParams, TrainingData, TrainingData> {
- private PreparatorParams pp;
- public Preparator(PreparatorParams pp) {
- this.pp = pp;
- }
-
- public TrainingData prepare(TrainingData td) {
- int n = (int) (td.r * pp.r);
-
- Double[][] x = new Double[n][td.c];
- Double[] y = new Double[n];
- for (int i=0; i<n; i++) {
- for (int j=0; j<td.c; j++) {
- x[i][j] = td.x[i][j];
- }
- y[i] = td.y[i];
- }
- return new TrainingData(x, y);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/PreparatorParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/PreparatorParams.java b/examples/experimental/java-local-regression/src/main/java/PreparatorParams.java
deleted file mode 100644
index e4f45fb..0000000
--- a/examples/experimental/java-local-regression/src/main/java/PreparatorParams.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class PreparatorParams implements JavaParams {
- // Take the r-fraction of data in training.
- public double r = 1.0;
- public PreparatorParams(double r) {
- this.r = r;
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/Run.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/Run.java b/examples/experimental/java-local-regression/src/main/java/Run.java
deleted file mode 100644
index 76601f4..0000000
--- a/examples/experimental/java-local-regression/src/main/java/Run.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaParams;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.LJavaAlgorithm;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import scala.Tuple2;
-
-public class Run {
- // During development, one can build a semi-engine, only add the first few layers. In this
- // particular example, we only add until Algorithm Layer.
- private static class HalfBakedEngineFactory implements IJavaEngineFactory {
- public JavaEngine<TrainingData, Integer, TrainingData, Double[], Double, Double> apply() {
- return new JavaEngineBuilder<TrainingData, Integer, TrainingData, Double[], Double, Double> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass(Preparator.class)
- .addAlgorithmClass("OLS", OLSAlgorithm.class)
- .addAlgorithmClass("Default", DefaultAlgorithm.class)
- .build();
- }
- }
-
- public static void runComponents() throws IOException {
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(new File("../data/lr_data.txt").getCanonicalPath()))
- .preparatorParams(new PreparatorParams(0.3))
- .addAlgorithmParams("OLS", new EmptyParams())
- .addAlgorithmParams("Default", new DefaultAlgorithmParams(0.2))
- .addAlgorithmParams("Default", new DefaultAlgorithmParams(0.4))
- .build();
-
- JavaWorkflow.runEngine(
- (new HalfBakedEngineFactory()).apply(),
- engineParams,
- new WorkflowParamsBuilder().batch("java regression engine").verbose(3).build()
- );
- }
-
- public static void runEngine() throws IOException {
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(new File("../data/lr_data.txt").getCanonicalPath()))
- .preparatorParams(new PreparatorParams(0.3))
- .addAlgorithmParams("OLS", new EmptyParams())
- .addAlgorithmParams("Default", new DefaultAlgorithmParams(0.2))
- .addAlgorithmParams("Default", new DefaultAlgorithmParams(0.4))
- .build();
-
- JavaWorkflow.runEngine(
- (new EngineFactory()).apply(),
- engineParams,
- MeanSquareEvaluator.class,
- new EmptyParams(),
- new WorkflowParamsBuilder().batch("java regression engine").verbose(3).build()
- );
- }
-
- public static void main(String[] args) {
- try {
- runEngine();
- //runComponents();
- } catch (IOException ex) {
- System.out.println(ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/Serving.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/Serving.java b/examples/experimental/java-local-regression/src/main/java/Serving.java
deleted file mode 100644
index 24ad092..0000000
--- a/examples/experimental/java-local-regression/src/main/java/Serving.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import org.apache.predictionio.controller.java.LJavaServing;
-import org.apache.predictionio.controller.java.EmptyParams;
-
-import java.lang.Iterable;
-
-public class Serving extends LJavaServing<EmptyParams, Double[], Double> {
- public Double serve(Double[] query, Iterable<Double> predictions) {
- int n = 0;
- double s = 0.0;
- for (Double d: predictions) {
- n += 1;
- s += d;
- }
- return s / n;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/TrainingData.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/TrainingData.java b/examples/experimental/java-local-regression/src/main/java/TrainingData.java
deleted file mode 100644
index c59d0d7..0000000
--- a/examples/experimental/java-local-regression/src/main/java/TrainingData.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.regression;
-
-import java.io.Serializable;
-import java.util.Arrays;
-
-public class TrainingData implements Serializable {
- public final Double[][] x;
- public final Double[] y;
- public final int r;
- public final int c;
- public TrainingData(Double[][] x, Double[] y) {
- this.x = x;
- this.y = y;
- this.r = x.length;
- this.c = x[0].length;
- }
- @Override public String toString() {
- return "TrainingData: r=" + this.r + ",c=" + this.c;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/examples/algorithmsParams.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/examples/algorithmsParams.json b/examples/experimental/java-local-regression/src/main/java/examples/algorithmsParams.json
deleted file mode 100644
index e376f2e..0000000
--- a/examples/experimental/java-local-regression/src/main/java/examples/algorithmsParams.json
+++ /dev/null
@@ -1,5 +0,0 @@
-[
- {"name": "OLS", "params": {}},
- {"name": "Default", "params": {"v": 0.2}},
- {"name": "Default", "params": {"v": 0.4}}
-]
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/examples/dataSourceParams.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/examples/dataSourceParams.json b/examples/experimental/java-local-regression/src/main/java/examples/dataSourceParams.json
deleted file mode 100644
index 8d87c56..0000000
--- a/examples/experimental/java-local-regression/src/main/java/examples/dataSourceParams.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
- "filepath": "data/lr_data.txt"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/examples/manifest.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/examples/manifest.json b/examples/experimental/java-local-regression/src/main/java/examples/manifest.json
deleted file mode 100644
index 191de43..0000000
--- a/examples/experimental/java-local-regression/src/main/java/examples/manifest.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "id": "org.apache.predictionio.examples.java.regression",
- "version": "0.8.1-SNAPSHOT",
- "name": "Apache Java Regression Engine",
- "engineFactory": "org.apache.predictionio.examples.java.regression.EngineFactory"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-regression/src/main/java/examples/preparatorParams.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-regression/src/main/java/examples/preparatorParams.json b/examples/experimental/java-local-regression/src/main/java/examples/preparatorParams.json
deleted file mode 100644
index f5ac299..0000000
--- a/examples/experimental/java-local-regression/src/main/java/examples/preparatorParams.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
- "r": 0.3
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/README.md b/examples/experimental/java-local-tutorial/README.md
deleted file mode 100644
index 478fa8e..0000000
--- a/examples/experimental/java-local-tutorial/README.md
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-## TODO: update with new engine.json format and CLI! (updates pending)
-
-## Download Data from GroupLens
-
-Execute the following command to download MovieLens 100k to data/ml-100k/.
-```
-$ ./fetch.sh
-```
-
-## Tutorials
-
-The purpose of this tutorial is to help you to get familiar with each components of the PredictionIO framework. It can be found in `docs/manual/tutorials/enginebuilders/stepbystep`.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/build.sbt b/examples/experimental/java-local-tutorial/build.sbt
deleted file mode 100644
index 077fa0c..0000000
--- a/examples/experimental/java-local-tutorial/build.sbt
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "java-local-tutorial"
-
-organization := "org.apache.predictionio"
-
-version := "0.9.1"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.predictionio" %% "engines" % "0.9.1" % "provided",
- "org.apache.mahout" % "mahout-core" % "0.9",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/data/test/ratings.csv
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/data/test/ratings.csv b/examples/experimental/java-local-tutorial/data/test/ratings.csv
deleted file mode 100644
index 126ba55..0000000
--- a/examples/experimental/java-local-tutorial/data/test/ratings.csv
+++ /dev/null
@@ -1,11 +0,0 @@
-1,1,2
-1,2,3
-1,3,4
-2,3,4
-2,4,1
-3,2,2
-3,3,1
-3,4,3
-4,1,5
-4,2,3
-4,4,2
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/fetch.sh
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/fetch.sh b/examples/experimental/java-local-tutorial/fetch.sh
deleted file mode 100755
index fc42dd4..0000000
--- a/examples/experimental/java-local-tutorial/fetch.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-
-TARGET_BASE_DIR="data"
-
-mkdir -p $TARGET_BASE_DIR
-curl http://files.grouplens.org/papers/ml-100k.zip -o $TARGET_BASE_DIR/ml-100k.zip
-cd $TARGET_BASE_DIR
-unzip ml-100k.zip
-
-echo "Data dir: ${TARGET_BASE_DIR}/ml-100k/"
-
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/project/assembly.sbt b/examples/experimental/java-local-tutorial/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/java-local-tutorial/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/.gitignore b/examples/experimental/java-local-tutorial/src/main/java/recommendations/.gitignore
deleted file mode 100644
index cc027f5..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-testdata/ml-100k/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/AlgoParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/AlgoParams.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/AlgoParams.java
deleted file mode 100644
index 05ad8b4..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/AlgoParams.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class AlgoParams implements JavaParams {
- public double threshold;
-
- public AlgoParams(double threshold) {
- this.threshold = threshold;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Algorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Algorithm.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Algorithm.java
deleted file mode 100644
index c2be789..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Algorithm.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import org.apache.predictionio.controller.java.LJavaAlgorithm;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.commons.math3.linear.ArrayRealVector;
-import org.apache.commons.math3.linear.OpenMapRealVector;
-import scala.Tuple2;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Queue;
-import java.util.PriorityQueue;
-import java.util.Comparator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class Algorithm extends
- LJavaAlgorithm<AlgoParams, TrainingData, Model, Query, Float> {
-
- final static Logger logger = LoggerFactory.getLogger(Algorithm.class);
-
- AlgoParams params;
-
- public Algorithm(AlgoParams params) {
- this.params = params;
- }
-
- @Override
- public Model train(TrainingData data) {
- // pre-process
- Map<Integer, Map<Integer, Float>> itemMap = new HashMap<Integer, Map<Integer, Float>>();
- Map<Integer, Integer> userIndexMap = new HashMap<Integer, Integer>();
- Map<Integer, Integer> itemIndexMap = new HashMap<Integer, Integer>();
-
- int itemIndex = 0;
- int userIndex = 0;
- for (TrainingData.Rating r : data.ratings) {
- Map<Integer, Float> userRating = itemMap.get(r.iid);
- if (userRating == null) {
- // new item
- userRating = new HashMap<Integer, Float>();
- itemMap.put(r.iid, userRating);
- itemIndexMap.put(r.iid, itemIndex);
- itemIndex += 1; // increment item index for next item
- }
- userRating.put(r.uid, r.rating);
-
- // update user index
- Integer u = userIndexMap.get(r.uid);
- if (u == null) {
- // new user
- userIndexMap.put(r.uid, userIndex);
- userIndex += 1;
- }
- }
-
- int numOfItems = itemIndexMap.size();
- int numOfUsers = userIndexMap.size();
-
- Map<Integer, RealVector> itemVectors = new HashMap<Integer, RealVector>();
- Map<Integer, RealVector> userHistory = new HashMap<Integer, RealVector>();
-
- for (Map.Entry<Integer, Map<Integer, Float>> entry : itemMap.entrySet()) {
- Integer itemID = entry.getKey();
- Integer iindex = itemIndexMap.get(itemID);
- Map<Integer, Float> userRatingMap = entry.getValue();
- RealVector item = new ArrayRealVector(numOfUsers); // dimension is numOfUsers
- for (Map.Entry<Integer, Float> r : userRatingMap.entrySet()) {
- Integer userID = r.getKey();
- Float rating = r.getValue();
- Integer uindex = userIndexMap.get(userID);
- item.setEntry(uindex, rating);
- // update user History
- RealVector user = userHistory.get(userID);
- if (user == null) {
- user = new OpenMapRealVector(numOfItems);
- userHistory.put(userID, user);
- }
- user.setEntry(iindex, rating);
- }
- itemVectors.put(itemID, item);
- }
-
- // calculate sim
-
- Map<Integer, RealVector> itemSimilarity = new HashMap<Integer, RealVector>();
- List<Integer> item1List = new ArrayList<Integer>(itemIndexMap.keySet());
- List<Integer> item2List = new ArrayList<Integer>(item1List);
-
- int numSimilarItems = 100;
- Comparator<IndexAndScore> comparator = new IndexAndScoreComparator();
- Map<Integer, Queue<IndexAndScore>> topItemSimilarity =
- new HashMap<Integer, Queue<IndexAndScore>>();
-
- for (Integer itemID1 : item1List) {
- item2List.remove(0);
- Integer index1 = itemIndexMap.get(itemID1);
- for (Integer itemID2: item2List) {
- RealVector vector1 = itemVectors.get(itemID1);
- RealVector vector2 = itemVectors.get(itemID2);
- double score = vector1.cosine(vector2);
- if (score > params.threshold) {
- Integer index2 = itemIndexMap.get(itemID2);
- setTopItemSimilarity(topItemSimilarity, itemID1, index2, score, numSimilarItems,
- comparator);
- setTopItemSimilarity(topItemSimilarity, itemID2, index1, score, numSimilarItems,
- comparator);
- }
- }
- }
-
- for (Map.Entry<Integer, Queue<IndexAndScore>> entry : topItemSimilarity.entrySet()) {
- Iterator<IndexAndScore> it = entry.getValue().iterator();
- RealVector vector = new OpenMapRealVector(numOfItems);
- while (it.hasNext()) {
- IndexAndScore d = it.next();
- vector.setEntry(d.index, d.score);
- }
- itemSimilarity.put(entry.getKey(), vector);
- }
-
- return new Model(itemSimilarity, userHistory);
- }
-
- private class IndexAndScore {
- int index;
- double score;
- public IndexAndScore(int index, double score) {
- this.index = index;
- this.score = score;
- }
- }
-
- private class IndexAndScoreComparator implements Comparator<IndexAndScore> {
- @Override
- public int compare(IndexAndScore o1, IndexAndScore o2) {
- int r = 0;
- if (o1.score < o2.score)
- r = -1;
- else if (o1.score > o2.score)
- r = 1;
- return r;
- }
- }
-
- private void setTopItemSimilarity(Map<Integer, Queue<IndexAndScore>> topItemSimilarity,
- Integer itemID1, Integer index2, double score, int capacity,
- Comparator<IndexAndScore> comparator) {
- Queue<IndexAndScore> queue = topItemSimilarity.get(itemID1);
- if (queue == null) {
- queue = new PriorityQueue<IndexAndScore>(capacity, comparator);
- topItemSimilarity.put(itemID1, queue);
- }
- IndexAndScore entry = new IndexAndScore(index2, score);
- if (queue.size() < capacity)
- queue.add(entry);
- else if (comparator.compare(queue.peek(), entry) < 0) {
- queue.poll();
- queue.add(entry);
- }
- }
-
- @Override
- public Float predict(Model model, Query query) {
- RealVector itemVector = model.itemSimilarity.get(query.iid);
- RealVector userVector = model.userHistory.get(query.uid);
- if (itemVector == null) {
- // cold start item, can't be handled by this algo, return hard code value.
- return Float.NaN;
- } else if (userVector == null) {
- // new user, can't be handled by this algo, return hard code value.
- return Float.NaN;
- } else {
- //logger.info("(" + query.uid + "," + query.iid + ")");
- //logger.info(itemVector.toString());
- //logger.info(userVector.toString());
- double accum = 0.0;
- double accumSim = 0.0;
- for (int i = 0; i < itemVector.getDimension(); i++) {
- double weight = itemVector.getEntry(i);
- double rating = userVector.getEntry(i);
- if ((weight != 0) && (rating != 0)) {
- accum += weight * rating;
- accumSim += Math.abs(weight);
- }
- }
-
- if (accumSim == 0.0) {
- return Float.NaN;
- } else {
- return (float) (accum / accumSim);
- }
- }
- }
-
-}
[18/50] [abbrv] incubator-predictionio git commit: [PIO-72] Add JDBC
JARs when they are present
Posted by ch...@apache.org.
[PIO-72] Add JDBC JARs when they are present
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/a8130217
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/a8130217
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/a8130217
Branch: refs/heads/master
Commit: a813021714ae8930c0d7e2061a15ab897005d188
Parents: d894b72
Author: Donald Szeto <do...@apache.org>
Authored: Fri Jul 28 11:08:00 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Fri Jul 28 11:08:00 2017 -0700
----------------------------------------------------------------------
bin/compute-classpath.sh | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a8130217/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index cd0527f..3e59ca7 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -63,9 +63,11 @@ if [ -n "$ES_CONF_DIR" ]; then
fi
if [ -n "$POSTGRES_JDBC_DRIVER" ]; then
CLASSPATH="$CLASSPATH:$POSTGRES_JDBC_DRIVER"
+ ASSEMBLY_JARS="$ASSEMBLY_JARS,$POSTGRES_JDBC_DRIVER"
fi
if [ -n "$MYSQL_JDBC_DRIVER" ]; then
CLASSPATH="$CLASSPATH:$MYSQL_JDBC_DRIVER"
+ ASSEMBLY_JARS="$ASSEMBLY_JARS,$MYSQL_JDBC_DRIVER"
fi
echo "$CLASSPATH"
[33/50] [abbrv] incubator-predictionio git commit: [PIO-115]
Implement Storage app & channel name-to-ID cache
Posted by ch...@apache.org.
[PIO-115] Implement Storage app & channel name-to-ID cache
Closes #424
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/eb613580
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/eb613580
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/eb613580
Branch: refs/heads/master
Commit: eb6135809f66ccd2db71a3f78404a4aa0ccb16cd
Parents: bcc0afe
Author: Mars Hall <ma...@heroku.com>
Authored: Tue Aug 29 11:28:19 2017 -0700
Committer: Mars Hall <ma...@heroku.com>
Committed: Tue Aug 29 11:28:19 2017 -0700
----------------------------------------------------------------------
.../apache/predictionio/data/store/Common.scala | 43 ++++++++++++--------
1 file changed, 25 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/eb613580/data/src/main/scala/org/apache/predictionio/data/store/Common.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/store/Common.scala b/data/src/main/scala/org/apache/predictionio/data/store/Common.scala
index 552064f..7daf19c 100644
--- a/data/src/main/scala/org/apache/predictionio/data/store/Common.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/store/Common.scala
@@ -19,6 +19,7 @@
package org.apache.predictionio.data.store
import org.apache.predictionio.data.storage.Storage
+import scala.collection.mutable
import grizzled.slf4j.Logger
private[predictionio] object Common {
@@ -26,28 +27,34 @@ private[predictionio] object Common {
@transient lazy val logger = Logger[this.type]
@transient lazy private val appsDb = Storage.getMetaDataApps()
@transient lazy private val channelsDb = Storage.getMetaDataChannels()
+ // Memoize app & channel name-to-ID resolution to avoid excessive storage IO
+ @transient lazy val appNameToIdCache =
+ mutable.Map[(String, Option[String]), (Int, Option[Int])]()
/* throw exception if invalid app name or channel name */
def appNameToId(appName: String, channelName: Option[String]): (Int, Option[Int]) = {
- val appOpt = appsDb.getByName(appName)
-
- appOpt.map { app =>
- val channelMap: Map[String, Int] = channelsDb.getByAppid(app.id)
- .map(c => (c.name, c.id)).toMap
-
- val channelId: Option[Int] = channelName.map { ch =>
- if (channelMap.contains(ch)) {
- channelMap(ch)
- } else {
- logger.error(s"Invalid channel name ${ch}.")
- throw new IllegalArgumentException(s"Invalid channel name ${ch}.")
+ appNameToIdCache.getOrElseUpdate((appName, channelName), {
+ val appOpt = appsDb.getByName(appName)
+
+ appOpt.map { app =>
+ val channelMap: Map[String, Int] = channelsDb.getByAppid(app.id)
+ .map(c => (c.name, c.id)).toMap
+
+ val channelId: Option[Int] = channelName.map { ch =>
+ if (channelMap.contains(ch)) {
+ channelMap(ch)
+ } else {
+ logger.error(s"Invalid channel name ${ch}.")
+ throw new IllegalArgumentException(s"Invalid channel name ${ch}.")
+ }
}
- }
- (app.id, channelId)
- }.getOrElse {
- logger.error(s"Invalid app name ${appName}")
- throw new IllegalArgumentException(s"Invalid app name ${appName}")
- }
+ appNameToIdCache((appName, channelName)) = (app.id, channelId)
+ (app.id, channelId)
+ }.getOrElse {
+ logger.error(s"Invalid app name ${appName}")
+ throw new IllegalArgumentException(s"Invalid app name ${appName}")
+ }
+ })
}
}
[03/50] [abbrv] incubator-predictionio git commit: Fix use of case
class in similarproduct examples
Posted by ch...@apache.org.
Fix use of case class in similarproduct examples
Closes #414
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/83f5ef70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/83f5ef70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/83f5ef70
Branch: refs/heads/master
Commit: 83f5ef706b6bedcfb00c3dd522f4fd82057b3009
Parents: 5aea927
Author: Naoki Takezoe <ta...@apache.org>
Authored: Wed Jul 19 10:27:07 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Wed Jul 19 10:27:07 2017 +0900
----------------------------------------------------------------------
.../source/templates/similarproduct/dase.html.md.erb | 14 +++++++-------
.../multi-events-multi-algos.html.md.erb | 2 +-
.../similarproduct/return-item-properties.html.md.erb | 2 +-
3 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/83f5ef70/docs/manual/source/templates/similarproduct/dase.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/templates/similarproduct/dase.html.md.erb b/docs/manual/source/templates/similarproduct/dase.html.md.erb
index a7db937..d9dd677 100644
--- a/docs/manual/source/templates/similarproduct/dase.html.md.erb
+++ b/docs/manual/source/templates/similarproduct/dase.html.md.erb
@@ -35,7 +35,7 @@ case class Query(
categories: Option[Set[String]],
whiteList: Option[Set[String]],
blackList: Option[Set[String]]
-) extends Serializable
+)
```
The `PredictedResult` case class defines the format of **predicted result**,
@@ -55,12 +55,12 @@ with:
```scala
case class PredictedResult(
itemScores: Array[ItemScore]
-) extends Serializable
+)
case class ItemScore(
item: String,
score: Double
-) extends Serializable
+)
```
Finally, `SimilarProductEngine` is the *Engine Factory* that defines the
@@ -531,13 +531,13 @@ INFO: You can easily modify `isCandidate()` checking or `whiteList` generation i
val topScores = getTopN(filteredScore, query.num)(ord).toArray
val itemScores = topScores.map { case (i, s) =>
- new ItemScore(
+ ItemScore(
item = model.itemIntStringMap(i),
score = s
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
```
@@ -545,13 +545,13 @@ Note that the item IDs in top N results are the `Int` indices. You map them back
```scala
val itemScores = topScores.map { case (i, s) =>
- new ItemScore(
+ ItemScore(
item = model.itemIntStringMap(i),
score = s
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
```
PredictionIO passes the returned `PredictedResult` object to *Serving*.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/83f5ef70/docs/manual/source/templates/similarproduct/multi-events-multi-algos.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/templates/similarproduct/multi-events-multi-algos.html.md.erb b/docs/manual/source/templates/similarproduct/multi-events-multi-algos.html.md.erb
index 9c93750..f863df1 100644
--- a/docs/manual/source/templates/similarproduct/multi-events-multi-algos.html.md.erb
+++ b/docs/manual/source/templates/similarproduct/multi-events-multi-algos.html.md.erb
@@ -331,7 +331,7 @@ class Serving
.take(query.num)
.map { case (k,v) => ItemScore(k, v) }
- new PredictedResult(combined)
+ PredictedResult(combined)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/83f5ef70/docs/manual/source/templates/similarproduct/return-item-properties.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/templates/similarproduct/return-item-properties.html.md.erb b/docs/manual/source/templates/similarproduct/return-item-properties.html.md.erb
index a23518e..3f70e6c 100644
--- a/docs/manual/source/templates/similarproduct/return-item-properties.html.md.erb
+++ b/docs/manual/source/templates/similarproduct/return-item-properties.html.md.erb
@@ -102,7 +102,7 @@ Modify how to create the ItemScore object using the properties.
val itemScores = topScores.map { case (i, s) =>
// MODIFIED
val it = model.items(i)
- new ItemScore(
+ ItemScore(
item = model.itemIntStringMap(i),
title = it.title,
date = it.date,
[37/50] [abbrv] incubator-predictionio git commit: [PIO-118]
ClassCastException from NullWritable to Text in ESEventsUtil
Posted by ch...@apache.org.
[PIO-118] ClassCastException from NullWritable to Text in ESEventsUtil
Closes #429
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/69c5e3b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/69c5e3b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/69c5e3b7
Branch: refs/heads/master
Commit: 69c5e3b777500b0ecc23516d4ae11c2fc95505c8
Parents: 6cf89c5
Author: Shinsuke Sugaya <sh...@apache.org>
Authored: Fri Sep 8 09:02:15 2017 +0900
Committer: Shinsuke Sugaya <sh...@apache.org>
Committed: Fri Sep 8 09:02:15 2017 +0900
----------------------------------------------------------------------
.../data/storage/elasticsearch/ESEventsUtil.scala | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/69c5e3b7/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
index 749ab49..38dec8a 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
@@ -51,11 +51,9 @@ object ESEventsUtil {
}
def getOptStringCol(col: String): Option[String] = {
- val r = result.get(new Text(col))
- if (r == null) {
- None
- } else {
- Some(r.asInstanceOf[Text].toString())
+ result.get(new Text(col)) match {
+ case x if x.isInstanceOf[Text] => Some(x.asInstanceOf[Text].toString)
+ case _ => None
}
}
[22/50] [abbrv] incubator-predictionio git commit: [PIO-63]Add
incubator logo and disclaimer to web site
Posted by ch...@apache.org.
[PIO-63]Add incubator logo and disclaimer to web site
Closes #405
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/55ce7b87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/55ce7b87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/55ce7b87
Branch: refs/heads/master
Commit: 55ce7b87d6816ed532838202f350eb3d36f48d64
Parents: 83f5ef7
Author: Naoki Takezoe <ta...@apache.org>
Authored: Thu Aug 3 15:03:47 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Thu Aug 3 15:03:47 2017 +0900
----------------------------------------------------------------------
.../source/images/logos/apache_incubator.png | Bin 0 -> 14557 bytes
docs/manual/source/partials/_footer.html.slim | 9 +++++++--
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/55ce7b87/docs/manual/source/images/logos/apache_incubator.png
----------------------------------------------------------------------
diff --git a/docs/manual/source/images/logos/apache_incubator.png b/docs/manual/source/images/logos/apache_incubator.png
new file mode 100644
index 0000000..f189e1b
Binary files /dev/null and b/docs/manual/source/images/logos/apache_incubator.png differ
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/55ce7b87/docs/manual/source/partials/_footer.html.slim
----------------------------------------------------------------------
diff --git a/docs/manual/source/partials/_footer.html.slim b/docs/manual/source/partials/_footer.html.slim
index feb1d0c..7aa22b2 100644
--- a/docs/manual/source/partials/_footer.html.slim
+++ b/docs/manual/source/partials/_footer.html.slim
@@ -4,7 +4,7 @@ footer
.container
.seperator
.row
- .col-md-6.col-xs-6.footer-link-column
+ .col-md-6.footer-link-column
.footer-link-column-row
h4 Community
ul
@@ -13,7 +13,7 @@ footer
li: a href="//github.com/apache/incubator-predictionio" target="blank" GitHub
li: a href="mailto:user-subscribe@predictionio.incubator.apache.org" target="blank" Subscribe to User Mailing List
li: a href="//stackoverflow.com/questions/tagged/predictionio" target="blank" Stackoverflow
- .col-md-6.col-xs-6.footer-link-column
+ .col-md-6.footer-link-column
.footer-link-column-row
h4 Contribute
ul
@@ -21,6 +21,11 @@ footer
li: a href="//github.com/apache/incubator-predictionio" target="blank" Source Code
li: a href="//issues.apache.org/jira/browse/PIO" target="blank" Bug Tracker
li: a href="mailto:dev-subscribe@predictionio.incubator.apache.org" target="blank" Subscribe to Development Mailing List
+ .row
+ .col-md-12.footer-link-column
+ a.pull-right href="http://incubator.apache.org/projects/predictionio.html"
+ = image_tag 'logos/apache_incubator.png', alt: 'Apache Incubator'
+ span Apache PredictionIO is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
#footer-bottom
.container
[30/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by ch...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/08ebd6b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/08ebd6b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/08ebd6b5
Branch: refs/heads/master
Commit: 08ebd6b5538b09be303915c9487725f24b5675af
Parents: 6789dbe 350b183
Author: Naoki Takezoe <ta...@apache.org>
Authored: Tue Aug 29 10:06:34 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Tue Aug 29 10:06:34 2017 +0900
----------------------------------------------------------------------
docs/manual/source/datacollection/eventmodel.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/08ebd6b5/docs/manual/source/datacollection/eventmodel.html.md.erb
----------------------------------------------------------------------
[50/50] [abbrv] incubator-predictionio git commit: Update
upgrade.html.md for breaking changes
Posted by ch...@apache.org.
Update upgrade.html.md for breaking changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/018ea8e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/018ea8e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/018ea8e3
Branch: refs/heads/master
Commit: 018ea8e34261f0929ad6d4c669fe80d7520bae16
Parents: 9340561
Author: Chan Lee <ch...@gmail.com>
Authored: Sun Sep 17 10:03:07 2017 -0700
Committer: Chan Lee <ch...@gmail.com>
Committed: Sun Sep 17 10:03:40 2017 -0700
----------------------------------------------------------------------
docs/manual/source/resources/upgrade.html.md | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/018ea8e3/docs/manual/source/resources/upgrade.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/resources/upgrade.html.md b/docs/manual/source/resources/upgrade.html.md
index f708fb8..2533c6d 100644
--- a/docs/manual/source/resources/upgrade.html.md
+++ b/docs/manual/source/resources/upgrade.html.md
@@ -36,6 +36,13 @@ To upgrade and use new version of PredictionIO, do the following:
In addition, please take notes of the following for specific version upgrade.
+## Upgrade to 0.12.0
+
+In 0.12.0, Elasticsearch 5.x client has been reimplemented as a singleton.
+Engine templates directly using Elasticsearch 5.x StorageClient require
+update for compatibility. See [pull request]
+(https://github.com/apache/incubator-predictionio/pull/421) for details.
+
## Upgrade to 0.11.0
Starting from 0.11.0, PredictionIO no longer bundles any JDBC drivers in the
[34/50] [abbrv] incubator-predictionio git commit: [PIO-106,
PIO-114] Elasticsearch 5.x singleton client with authentication
Posted by ch...@apache.org.
[PIO-106,PIO-114] Elasticsearch 5.x singleton client with authentication
Closes #421
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/bf84ede6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/bf84ede6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/bf84ede6
Branch: refs/heads/master
Commit: bf84ede6fe475ec591e784eb453c6194befb8515
Parents: eb61358
Author: Mars Hall <ma...@heroku.com>
Authored: Tue Aug 29 11:38:34 2017 -0700
Committer: Mars Hall <ma...@heroku.com>
Committed: Tue Aug 29 11:38:34 2017 -0700
----------------------------------------------------------------------
bin/compute-classpath.sh | 21 ++-
conf/pio-env.sh.template | 3 +
.../predictionio/workflow/BatchPredict.scala | 140 ++++++++++---------
.../workflow/CleanupFunctions.scala | 65 +++++++++
.../predictionio/workflow/CoreWorkflow.scala | 66 +++++----
.../predictionio/data/storage/Storage.scala | 8 +-
storage/elasticsearch/build.sbt | 8 +-
.../storage/elasticsearch/ESAccessKeys.scala | 55 +++-----
.../data/storage/elasticsearch/ESApps.scala | 55 +++-----
.../data/storage/elasticsearch/ESChannels.scala | 50 +++----
.../elasticsearch/ESEngineInstances.scala | 84 ++++-------
.../elasticsearch/ESEvaluationInstances.scala | 71 ++++------
.../data/storage/elasticsearch/ESLEvents.scala | 49 +++++--
.../data/storage/elasticsearch/ESPEvents.scala | 29 ++--
.../storage/elasticsearch/ESSequences.scala | 30 ++--
.../storage/elasticsearch/StorageClient.scala | 69 ++++++++-
.../elasticsearch/StorageClientSpec.scala | 67 +++++++++
.../elasticsearch/StorageTestUtils.scala | 28 ++++
storage/hdfs/project/build.properties | 1 +
.../tools/export/EventsToFile.scala | 62 ++++----
.../predictionio/tools/imprt/FileToEvents.scala | 64 +++++----
21 files changed, 596 insertions(+), 429 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 3e59ca7..7a38e0b 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -42,8 +42,25 @@ ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -)
# Build up classpath
CLASSPATH="${PIO_CONF_DIR}"
-CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*"
-CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*"
+
+# stable classpath for plugin JARs
+if [ -d "${FWDIR}/plugins" ]; then
+ lib_plugin_jars=`ls "${FWDIR}"/plugins/*`
+ lib_plugin_classpath=''
+ for J in $lib_plugin_jars; do
+ lib_plugin_classpath="${lib_plugin_classpath}:${J}"
+ done
+ CLASSPATH="$CLASSPATH${lib_plugin_classpath}"
+fi
+
+# stable classpath for Spark JARs
+lib_spark_jars=`ls "${assembly_folder}"/spark/*.jar`
+lib_spark_classpath=''
+for J in $lib_spark_jars; do
+ lib_spark_classpath="${lib_spark_classpath}:${J}"
+done
+CLASSPATH="$CLASSPATH${lib_spark_classpath}"
+
CLASSPATH="$CLASSPATH:${MAIN_JAR}"
# Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/conf/pio-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/pio-env.sh.template b/conf/pio-env.sh.template
index 832b422..0b6b5b9 100644
--- a/conf/pio-env.sh.template
+++ b/conf/pio-env.sh.template
@@ -90,6 +90,9 @@ PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
# PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
# PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http
# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-5.4.1
+# Optional basic HTTP auth
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_USERNAME=my-name
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_PASSWORD=my-secret
# Elasticsearch 1.x Example
# PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
# PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=<elasticsearch_cluster_name>
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala b/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
index 5420638..69525b1 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
@@ -28,6 +28,7 @@ import org.apache.predictionio.controller.{Engine, Utils}
import org.apache.predictionio.core.{BaseAlgorithm, BaseServing, Doer}
import org.apache.predictionio.data.storage.{EngineInstance, Storage}
import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
+import org.apache.predictionio.workflow.CleanupFunctions
import org.apache.spark.rdd.RDD
import org.json4s._
import org.json4s.native.JsonMethods._
@@ -146,84 +147,89 @@ object BatchPredict extends Logging {
engineInstance: EngineInstance,
engine: Engine[_, _, _, Q, P, _]): Unit = {
- val engineParams = engine.engineInstanceToEngineParams(
- engineInstance, config.jsonExtractor)
+ try {
+ val engineParams = engine.engineInstanceToEngineParams(
+ engineInstance, config.jsonExtractor)
- val kryo = KryoInstantiator.newKryoInjection
+ val kryo = KryoInstantiator.newKryoInjection
- val modelsFromEngineInstance =
- kryo.invert(modeldata.get(engineInstance.id).get.models).get.
- asInstanceOf[Seq[Any]]
+ val modelsFromEngineInstance =
+ kryo.invert(modeldata.get(engineInstance.id).get.models).get.
+ asInstanceOf[Seq[Any]]
- val prepareSparkContext = WorkflowContext(
- batch = engineInstance.engineFactory,
- executorEnv = engineInstance.env,
- mode = "Batch Predict (model)",
- sparkEnv = engineInstance.sparkConf)
+ val prepareSparkContext = WorkflowContext(
+ batch = engineInstance.engineFactory,
+ executorEnv = engineInstance.env,
+ mode = "Batch Predict (model)",
+ sparkEnv = engineInstance.sparkConf)
- val models = engine.prepareDeploy(
- prepareSparkContext,
- engineParams,
- engineInstance.id,
- modelsFromEngineInstance,
- params = WorkflowParams()
- )
+ val models = engine.prepareDeploy(
+ prepareSparkContext,
+ engineParams,
+ engineInstance.id,
+ modelsFromEngineInstance,
+ params = WorkflowParams()
+ )
- val algorithms = engineParams.algorithmParamsList.map { case (n, p) =>
- Doer(engine.algorithmClassMap(n), p)
- }
+ val algorithms = engineParams.algorithmParamsList.map { case (n, p) =>
+ Doer(engine.algorithmClassMap(n), p)
+ }
- val servingParamsWithName = engineParams.servingParams
+ val servingParamsWithName = engineParams.servingParams
- val serving = Doer(engine.servingClassMap(servingParamsWithName._1),
- servingParamsWithName._2)
+ val serving = Doer(engine.servingClassMap(servingParamsWithName._1),
+ servingParamsWithName._2)
- val runSparkContext = WorkflowContext(
- batch = engineInstance.engineFactory,
- executorEnv = engineInstance.env,
- mode = "Batch Predict (runner)",
- sparkEnv = engineInstance.sparkConf)
+ val runSparkContext = WorkflowContext(
+ batch = engineInstance.engineFactory,
+ executorEnv = engineInstance.env,
+ mode = "Batch Predict (runner)",
+ sparkEnv = engineInstance.sparkConf)
- val inputRDD: RDD[String] = runSparkContext.
- textFile(config.inputFilePath).
- filter(_.trim.nonEmpty)
- val queriesRDD: RDD[String] = config.queryPartitions match {
- case Some(p) => inputRDD.repartition(p)
- case None => inputRDD
- }
+ val inputRDD: RDD[String] = runSparkContext.
+ textFile(config.inputFilePath).
+ filter(_.trim.nonEmpty)
+ val queriesRDD: RDD[String] = config.queryPartitions match {
+ case Some(p) => inputRDD.repartition(p)
+ case None => inputRDD
+ }
- val predictionsRDD: RDD[String] = queriesRDD.map { queryString =>
- val jsonExtractorOption = config.jsonExtractor
- // Extract Query from Json
- val query = JsonExtractor.extract(
- jsonExtractorOption,
- queryString,
- algorithms.head.queryClass,
- algorithms.head.querySerializer,
- algorithms.head.gsonTypeAdapterFactories
- )
- // Deploy logic. First call Serving.supplement, then Algo.predict,
- // finally Serving.serve.
- val supplementedQuery = serving.supplementBase(query)
- // TODO: Parallelize the following.
- val predictions = algorithms.zip(models).map { case (a, m) =>
- a.predictBase(m, supplementedQuery)
+ val predictionsRDD: RDD[String] = queriesRDD.map { queryString =>
+ val jsonExtractorOption = config.jsonExtractor
+ // Extract Query from Json
+ val query = JsonExtractor.extract(
+ jsonExtractorOption,
+ queryString,
+ algorithms.head.queryClass,
+ algorithms.head.querySerializer,
+ algorithms.head.gsonTypeAdapterFactories
+ )
+ // Deploy logic. First call Serving.supplement, then Algo.predict,
+ // finally Serving.serve.
+ val supplementedQuery = serving.supplementBase(query)
+ // TODO: Parallelize the following.
+ val predictions = algorithms.zip(models).map { case (a, m) =>
+ a.predictBase(m, supplementedQuery)
+ }
+ // Notice that it is by design to call Serving.serve with the
+ // *original* query.
+ val prediction = serving.serveBase(query, predictions)
+ // Combine query with prediction, so the batch results are
+ // self-descriptive.
+ val predictionJValue = JsonExtractor.toJValue(
+ jsonExtractorOption,
+ Map("query" -> query,
+ "prediction" -> prediction),
+ algorithms.head.querySerializer,
+ algorithms.head.gsonTypeAdapterFactories)
+ // Return JSON string
+ compact(render(predictionJValue))
}
- // Notice that it is by design to call Serving.serve with the
- // *original* query.
- val prediction = serving.serveBase(query, predictions)
- // Combine query with prediction, so the batch results are
- // self-descriptive.
- val predictionJValue = JsonExtractor.toJValue(
- jsonExtractorOption,
- Map("query" -> query,
- "prediction" -> prediction),
- algorithms.head.querySerializer,
- algorithms.head.gsonTypeAdapterFactories)
- // Return JSON string
- compact(render(predictionJValue))
- }
- predictionsRDD.saveAsTextFile(config.outputFilePath)
+ predictionsRDD.saveAsTextFile(config.outputFilePath)
+
+ } finally {
+ CleanupFunctions.run()
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/core/src/main/scala/org/apache/predictionio/workflow/CleanupFunctions.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CleanupFunctions.scala b/core/src/main/scala/org/apache/predictionio/workflow/CleanupFunctions.scala
new file mode 100644
index 0000000..bdd8b01
--- /dev/null
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CleanupFunctions.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.workflow
+
+/** :: DeveloperApi ::
+ * Singleton object that collects anonymous functions to be
+ * executed to allow the process to end gracefully.
+ *
+ * For example, the Elasticsearch REST storage client
+ * maintains an internal connection pool that must
+ * be closed to allow the process to exit.
+ */
+object CleanupFunctions {
+ @volatile private var functions: Seq[() => Unit] = Seq.empty[() => Unit]
+
+ /** Add a function to be called during cleanup.
+ *
+ * {{{
+ * import org.apache.predictionio.workflow.CleanupFunctions
+ *
+ * CleanupFunctions.add { MyStorageClass.close }
+ * }}}
+ *
+ * @param anonymous function containing cleanup code.
+ */
+ def add(f: () => Unit): Seq[() => Unit] = {
+ functions = functions :+ f
+ functions
+ }
+
+ /** Call all cleanup functions in order added.
+ *
+ * {{{
+ * import org.apache.predictionio.workflow.CleanupFunctions
+ *
+ * try {
+ * // Much code that needs cleanup
+ * // whether successful or error thrown.
+ * } finally {
+ * CleanupFunctions.run()
+ * }
+ * }}}
+ *
+ * @param anonymous function containing cleanup code.
+ */
+ def run(): Unit = {
+ functions.foreach { f => f() }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/core/src/main/scala/org/apache/predictionio/workflow/CoreWorkflow.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CoreWorkflow.scala b/core/src/main/scala/org/apache/predictionio/workflow/CoreWorkflow.scala
index d956fd7..65270a2 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/CoreWorkflow.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CoreWorkflow.scala
@@ -96,6 +96,7 @@ object CoreWorkflow {
}
} finally {
logger.debug("Stopping SparkContext")
+ CleanupFunctions.run()
sc.stop()
}
}
@@ -123,40 +124,43 @@ object CoreWorkflow {
env,
params.sparkEnv,
mode.capitalize)
- val evaluationInstanceId = evaluationInstances.insert(evaluationInstance)
- logger.info(s"Starting evaluation instance ID: $evaluationInstanceId")
-
- val evaluatorResult: BaseEvaluatorResult = EvaluationWorkflow.runEvaluation(
- sc,
- evaluation,
- engine,
- engineParamsList,
- evaluator,
- params)
-
- if (evaluatorResult.noSave) {
- logger.info(s"This evaluation result is not inserted into database: $evaluatorResult")
- } else {
- val evaluatedEvaluationInstance = evaluationInstance.copy(
- status = "EVALCOMPLETED",
- id = evaluationInstanceId,
- endTime = DateTime.now,
- evaluatorResults = evaluatorResult.toOneLiner,
- evaluatorResultsHTML = evaluatorResult.toHTML,
- evaluatorResultsJSON = evaluatorResult.toJSON
- )
-
- logger.info(s"Updating evaluation instance with result: $evaluatorResult")
+ try {
+ val evaluationInstanceId = evaluationInstances.insert(evaluationInstance)
+
+ logger.info(s"Starting evaluation instance ID: $evaluationInstanceId")
+
+ val evaluatorResult: BaseEvaluatorResult = EvaluationWorkflow.runEvaluation(
+ sc,
+ evaluation,
+ engine,
+ engineParamsList,
+ evaluator,
+ params)
+
+ if (evaluatorResult.noSave) {
+ logger.info(s"This evaluation result is not inserted into database: $evaluatorResult")
+ } else {
+ val evaluatedEvaluationInstance = evaluationInstance.copy(
+ status = "EVALCOMPLETED",
+ id = evaluationInstanceId,
+ endTime = DateTime.now,
+ evaluatorResults = evaluatorResult.toOneLiner,
+ evaluatorResultsHTML = evaluatorResult.toHTML,
+ evaluatorResultsJSON = evaluatorResult.toJSON
+ )
+
+ logger.info(s"Updating evaluation instance with result: $evaluatorResult")
+
+ evaluationInstances.update(evaluatedEvaluationInstance)
+ }
+ logger.info("runEvaluation completed")
- evaluationInstances.update(evaluatedEvaluationInstance)
+ } finally {
+ logger.debug("Stop SparkContext")
+ CleanupFunctions.run()
+ sc.stop()
}
-
- logger.debug("Stop SparkContext")
-
- sc.stop()
-
- logger.info("runEvaluation completed")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
index 52442a6..70e1973 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
@@ -19,6 +19,7 @@
package org.apache.predictionio.data.storage
import grizzled.slf4j.Logging
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.predictionio.annotation.DeveloperApi
import scala.concurrent.ExecutionContext.Implicits.global
@@ -214,7 +215,8 @@ object Storage extends Logging {
}
} catch {
case e: Throwable =>
- error(e.getMessage)
+ val stackTrace = ExceptionUtils.getStackTrace(e)
+ error(s"${e.getMessage}\n${stackTrace}\n\n")
errors += 1
r -> DataObjectMeta("", "")
}
@@ -282,7 +284,9 @@ object Storage extends Logging {
Some(ClientMeta(sourceType, client, clientConfig))
} catch {
case e: Throwable =>
- error(s"Error initializing storage client for source ${k}", e)
+ val stackTrace = ExceptionUtils.getStackTrace(e)
+ error(s"Error initializing storage client for source ${k}.\n" +
+ s"${stackTrace}\n\n")
errors += 1
None
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index da4842e..b60e86e 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -30,7 +30,7 @@ libraryDependencies ++= Seq(
"org.elasticsearch" %% elasticsearchSparkArtifact.value % elasticsearchVersion.value
exclude("org.apache.spark", "*"),
"org.elasticsearch" % "elasticsearch-hadoop-mr" % elasticsearchVersion.value,
- "org.scalatest" %% "scalatest" % "2.1.7" % "test")
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
parallelExecution in Test := false
@@ -39,8 +39,10 @@ pomExtra := childrenPomExtra.value
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
assemblyShadeRules in assembly := Seq(
- ShadeRule.rename("org.apache.http.**" -> "shadeio.data.http.@1").inAll
-)
+ ShadeRule.rename("org.apache.http.**" ->
+ "org.apache.predictionio.shaded.org.apache.http.@1").inAll,
+ ShadeRule.rename("org.elasticsearch.client.**" ->
+ "org.apache.predictionio.shaded.org.elasticsearch.client.@1").inAll)
// skip test in assembly
test in assembly := {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
index 98c2781..73ef1d0 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
@@ -27,36 +27,30 @@ import org.apache.http.util.EntityUtils
import org.apache.predictionio.data.storage.AccessKey
import org.apache.predictionio.data.storage.AccessKeys
import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.{ResponseException, RestClient}
import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.write
import grizzled.slf4j.Logging
-import org.elasticsearch.client.ResponseException
/** Elasticsearch implementation of AccessKeys. */
-class ESAccessKeys(client: ESClient, config: StorageClientConfig, index: String)
+class ESAccessKeys(client: RestClient, config: StorageClientConfig, index: String)
extends AccessKeys with Logging {
implicit val formats = DefaultFormats.lossless
private val estype = "accesskeys"
- val restClient = client.open()
- try {
- ESUtils.createIndex(restClient, index,
- ESUtils.getNumberOfShards(config, index.toUpperCase),
- ESUtils.getNumberOfReplicas(config, index.toUpperCase))
- val mappingJson =
- (estype ->
- ("_all" -> ("enabled" -> false)) ~
- ("properties" ->
- ("key" -> ("type" -> "keyword")) ~
- ("events" -> ("type" -> "keyword"))))
- ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
- } finally {
- restClient.close()
- }
+ ESUtils.createIndex(client, index,
+ ESUtils.getNumberOfShards(config, index.toUpperCase),
+ ESUtils.getNumberOfReplicas(config, index.toUpperCase))
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> false)) ~
+ ("properties" ->
+ ("key" -> ("type" -> "keyword")) ~
+ ("events" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(client, index, estype, compact(render(mappingJson)))
def insert(accessKey: AccessKey): Option[String] = {
val key = if (accessKey.key.isEmpty) generateKey else accessKey.key
@@ -68,9 +62,8 @@ class ESAccessKeys(client: ESClient, config: StorageClientConfig, index: String)
if (id.isEmpty) {
return None
}
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"GET",
s"/$index/$estype/$id",
Map.empty[String, String].asJava)
@@ -92,50 +85,41 @@ class ESAccessKeys(client: ESClient, config: StorageClientConfig, index: String)
case e: IOException =>
error(s"Failed to access to /$index/$estype/$id", e)
None
- } finally {
- restClient.close()
}
}
def getAll(): Seq[AccessKey] = {
- val restClient = client.open()
try {
val json =
("query" ->
("match_all" -> List.empty))
- ESUtils.getAll[AccessKey](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[AccessKey](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error("Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
def getByAppid(appid: Int): Seq[AccessKey] = {
- val restClient = client.open()
try {
val json =
("query" ->
("term" ->
("appid" -> appid)))
- ESUtils.getAll[AccessKey](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[AccessKey](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error("Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
def update(accessKey: AccessKey): Unit = {
val id = accessKey.key
- val restClient = client.open()
try {
val entity = new NStringEntity(write(accessKey), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava,
@@ -151,15 +135,12 @@ class ESAccessKeys(client: ESClient, config: StorageClientConfig, index: String)
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
- } finally {
- restClient.close()
}
}
def delete(id: String): Unit = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"DELETE",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava)
@@ -173,8 +154,6 @@ class ESAccessKeys(client: ESClient, config: StorageClientConfig, index: String)
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/id", e)
- } finally {
- restClient.close()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
index 6afed12..ba48065 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
@@ -27,37 +27,31 @@ import org.apache.http.util.EntityUtils
import org.apache.predictionio.data.storage.App
import org.apache.predictionio.data.storage.Apps
import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.{ResponseException, RestClient}
import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.write
import grizzled.slf4j.Logging
-import org.elasticsearch.client.ResponseException
/** Elasticsearch implementation of Items. */
-class ESApps(client: ESClient, config: StorageClientConfig, index: String)
+class ESApps(client: RestClient, config: StorageClientConfig, index: String)
extends Apps with Logging {
implicit val formats = DefaultFormats.lossless
private val estype = "apps"
private val seq = new ESSequences(client, config, index)
- val restClient = client.open()
- try {
- ESUtils.createIndex(restClient, index,
- ESUtils.getNumberOfShards(config, index.toUpperCase),
- ESUtils.getNumberOfReplicas(config, index.toUpperCase))
- val mappingJson =
- (estype ->
- ("_all" -> ("enabled" -> false)) ~
- ("properties" ->
- ("id" -> ("type" -> "keyword")) ~
- ("name" -> ("type" -> "keyword"))))
- ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
- } finally {
- restClient.close()
- }
+ ESUtils.createIndex(client, index,
+ ESUtils.getNumberOfShards(config, index.toUpperCase),
+ ESUtils.getNumberOfReplicas(config, index.toUpperCase))
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> false)) ~
+ ("properties" ->
+ ("id" -> ("type" -> "keyword")) ~
+ ("name" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(client, index, estype, compact(render(mappingJson)))
def insert(app: App): Option[Int] = {
val id = app.id match {
@@ -77,9 +71,8 @@ class ESApps(client: ESClient, config: StorageClientConfig, index: String)
}
def get(id: Int): Option[App] = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"GET",
s"/$index/$estype/$id",
Map.empty[String, String].asJava)
@@ -101,20 +94,17 @@ class ESApps(client: ESClient, config: StorageClientConfig, index: String)
case e: IOException =>
error(s"Failed to access to /$index/$estype/$id", e)
None
- } finally {
- restClient.close()
}
}
def getByName(name: String): Option[App] = {
- val restClient = client.open()
try {
val json =
("query" ->
("term" ->
("name" -> name)))
val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/_search",
Map.empty[String, String].asJava,
@@ -131,33 +121,27 @@ class ESApps(client: ESClient, config: StorageClientConfig, index: String)
case e: IOException =>
error(s"Failed to access to /$index/$estype/_search", e)
None
- } finally {
- restClient.close()
}
}
def getAll(): Seq[App] = {
- val restClient = client.open()
try {
val json =
("query" ->
("match_all" -> Nil))
- ESUtils.getAll[App](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[App](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error("Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
def update(app: App): Unit = {
val id = app.id.toString
- val restClient = client.open()
try {
val entity = new NStringEntity(write(app), ContentType.APPLICATION_JSON);
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava,
@@ -173,15 +157,12 @@ class ESApps(client: ESClient, config: StorageClientConfig, index: String)
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
- } finally {
- restClient.close()
}
}
def delete(id: Int): Unit = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"DELETE",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava)
@@ -195,8 +176,6 @@ class ESApps(client: ESClient, config: StorageClientConfig, index: String)
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/id", e)
- } finally {
- restClient.close()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
index c142beb..b5eb5c8 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
@@ -27,35 +27,29 @@ import org.apache.http.util.EntityUtils
import org.apache.predictionio.data.storage.Channel
import org.apache.predictionio.data.storage.Channels
import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.{ResponseException, RestClient}
import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.write
import grizzled.slf4j.Logging
-import org.elasticsearch.client.ResponseException
-class ESChannels(client: ESClient, config: StorageClientConfig, index: String)
+class ESChannels(client: RestClient, config: StorageClientConfig, index: String)
extends Channels with Logging {
implicit val formats = DefaultFormats.lossless
private val estype = "channels"
private val seq = new ESSequences(client, config, index)
-
- val restClient = client.open()
- try {
- ESUtils.createIndex(restClient, index,
- ESUtils.getNumberOfShards(config, index.toUpperCase),
- ESUtils.getNumberOfReplicas(config, index.toUpperCase))
- val mappingJson =
- (estype ->
- ("_all" -> ("enabled" -> false)) ~
- ("properties" ->
- ("name" -> ("type" -> "keyword"))))
- ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
- } finally {
- restClient.close()
- }
+
+ ESUtils.createIndex(client, index,
+ ESUtils.getNumberOfShards(config, index.toUpperCase),
+ ESUtils.getNumberOfReplicas(config, index.toUpperCase))
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> false)) ~
+ ("properties" ->
+ ("name" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(client, index, estype, compact(render(mappingJson)))
def insert(channel: Channel): Option[Int] = {
val id = channel.id match {
@@ -75,9 +69,8 @@ class ESChannels(client: ESClient, config: StorageClientConfig, index: String)
}
def get(id: Int): Option[Channel] = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"GET",
s"/$index/$estype/$id",
Map.empty[String, String].asJava)
@@ -99,34 +92,28 @@ class ESChannels(client: ESClient, config: StorageClientConfig, index: String)
case e: IOException =>
error(s"Failed to access to /$index/$estype/$id", e)
None
- } finally {
- restClient.close()
}
}
def getByAppid(appid: Int): Seq[Channel] = {
- val restClient = client.open()
try {
val json =
("query" ->
("term" ->
("appid" -> appid)))
- ESUtils.getAll[Channel](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[Channel](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error(s"Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
def update(channel: Channel): Boolean = {
val id = channel.id.toString
- val restClient = client.open()
try {
val entity = new NStringEntity(write(channel), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava,
@@ -144,15 +131,12 @@ class ESChannels(client: ESClient, config: StorageClientConfig, index: String)
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
false
- } finally {
- restClient.close()
}
}
def delete(id: Int): Unit = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"DELETE",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava)
@@ -166,8 +150,6 @@ class ESChannels(client: ESClient, config: StorageClientConfig, index: String)
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
- } finally {
- restClient.close()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
index de474c1..eec5b64 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
@@ -28,46 +28,40 @@ import org.apache.predictionio.data.storage.EngineInstance
import org.apache.predictionio.data.storage.EngineInstanceSerializer
import org.apache.predictionio.data.storage.EngineInstances
import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.{ResponseException, RestClient}
import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.write
import grizzled.slf4j.Logging
-import org.elasticsearch.client.ResponseException
-class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: String)
+class ESEngineInstances(client: RestClient, config: StorageClientConfig, index: String)
extends EngineInstances with Logging {
implicit val formats = DefaultFormats + new EngineInstanceSerializer
private val estype = "engine_instances"
-
- val restClient = client.open()
- try {
- ESUtils.createIndex(restClient, index,
- ESUtils.getNumberOfShards(config, index.toUpperCase),
- ESUtils.getNumberOfReplicas(config, index.toUpperCase))
- val mappingJson =
- (estype ->
- ("_all" -> ("enabled" -> false)) ~
- ("properties" ->
- ("status" -> ("type" -> "keyword")) ~
- ("startTime" -> ("type" -> "date")) ~
- ("endTime" -> ("type" -> "date")) ~
- ("engineId" -> ("type" -> "keyword")) ~
- ("engineVersion" -> ("type" -> "keyword")) ~
- ("engineVariant" -> ("type" -> "keyword")) ~
- ("engineFactory" -> ("type" -> "keyword")) ~
- ("batch" -> ("type" -> "keyword")) ~
- ("dataSourceParams" -> ("type" -> "keyword")) ~
- ("preparatorParams" -> ("type" -> "keyword")) ~
- ("algorithmsParams" -> ("type" -> "keyword")) ~
- ("servingParams" -> ("type" -> "keyword")) ~
- ("status" -> ("type" -> "keyword"))))
- ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
- } finally {
- restClient.close()
- }
+
+ ESUtils.createIndex(client, index,
+ ESUtils.getNumberOfShards(config, index.toUpperCase),
+ ESUtils.getNumberOfReplicas(config, index.toUpperCase))
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> false)) ~
+ ("properties" ->
+ ("status" -> ("type" -> "keyword")) ~
+ ("startTime" -> ("type" -> "date")) ~
+ ("endTime" -> ("type" -> "date")) ~
+ ("engineId" -> ("type" -> "keyword")) ~
+ ("engineVersion" -> ("type" -> "keyword")) ~
+ ("engineVariant" -> ("type" -> "keyword")) ~
+ ("engineFactory" -> ("type" -> "keyword")) ~
+ ("batch" -> ("type" -> "keyword")) ~
+ ("dataSourceParams" -> ("type" -> "keyword")) ~
+ ("preparatorParams" -> ("type" -> "keyword")) ~
+ ("algorithmsParams" -> ("type" -> "keyword")) ~
+ ("servingParams" -> ("type" -> "keyword")) ~
+ ("status" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(client, index, estype, compact(render(mappingJson)))
def insert(i: EngineInstance): String = {
val id = i.id match {
@@ -88,10 +82,9 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
}
def preInsert(): Option[String] = {
- val restClient = client.open()
try {
val entity = new NStringEntity("{}", ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/",
Map("refresh" -> "true").asJava,
@@ -109,15 +102,12 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
case e: IOException =>
error(s"Failed to create $index/$estype", e)
None
- } finally {
- restClient.close()
}
}
def get(id: String): Option[EngineInstance] = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"GET",
s"/$index/$estype/$id",
Map.empty[String, String].asJava)
@@ -139,24 +129,19 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
case e: IOException =>
error(s"Failed to access to /$index/$estype/$id", e)
None
- } finally {
- restClient.close()
}
}
def getAll(): Seq[EngineInstance] = {
- val restClient = client.open()
try {
val json =
("query" ->
("match_all" -> List.empty))
- ESUtils.getAll[EngineInstance](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[EngineInstance](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error("Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
@@ -164,7 +149,6 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
engineId: String,
engineVersion: String,
engineVariant: String): Seq[EngineInstance] = {
- val restClient = client.open()
try {
val json =
("query" ->
@@ -181,13 +165,11 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
("sort" -> List(
("startTime" ->
("order" -> "desc"))))
- ESUtils.getAll[EngineInstance](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[EngineInstance](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error(s"Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
@@ -202,10 +184,9 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
def update(i: EngineInstance): Unit = {
val id = i.id
- val restClient = client.open()
try {
val entity = new NStringEntity(write(i), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava,
@@ -221,15 +202,12 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
- } finally {
- restClient.close()
}
}
def delete(id: String): Unit = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"DELETE",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava)
@@ -243,8 +221,6 @@ class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: St
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
- } finally {
- restClient.close()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
index 9b19cf4..1706583 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
@@ -29,43 +29,37 @@ import org.apache.predictionio.data.storage.EvaluationInstanceSerializer
import org.apache.predictionio.data.storage.EvaluationInstances
import org.apache.predictionio.data.storage.StorageClientConfig
import org.apache.predictionio.data.storage.StorageClientException
-import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.{ResponseException, RestClient}
import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.write
import grizzled.slf4j.Logging
-import org.elasticsearch.client.ResponseException
-class ESEvaluationInstances(client: ESClient, config: StorageClientConfig, index: String)
+class ESEvaluationInstances(client: RestClient, config: StorageClientConfig, index: String)
extends EvaluationInstances with Logging {
implicit val formats = DefaultFormats + new EvaluationInstanceSerializer
private val estype = "evaluation_instances"
private val seq = new ESSequences(client, config, index)
-
- val restClient = client.open()
- try {
- ESUtils.createIndex(restClient, index,
- ESUtils.getNumberOfShards(config, index.toUpperCase),
- ESUtils.getNumberOfReplicas(config, index.toUpperCase))
- val mappingJson =
- (estype ->
- ("_all" -> ("enabled" -> false)) ~
- ("properties" ->
- ("status" -> ("type" -> "keyword")) ~
- ("startTime" -> ("type" -> "date")) ~
- ("endTime" -> ("type" -> "date")) ~
- ("evaluationClass" -> ("type" -> "keyword")) ~
- ("engineParamsGeneratorClass" -> ("type" -> "keyword")) ~
- ("batch" -> ("type" -> "keyword")) ~
- ("evaluatorResults" -> ("type" -> "text")) ~
- ("evaluatorResultsHTML" -> ("enabled" -> false)) ~
- ("evaluatorResultsJSON" -> ("enabled" -> false))))
- ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
- } finally {
- restClient.close()
- }
+
+ ESUtils.createIndex(client, index,
+ ESUtils.getNumberOfShards(config, index.toUpperCase),
+ ESUtils.getNumberOfReplicas(config, index.toUpperCase))
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> false)) ~
+ ("properties" ->
+ ("status" -> ("type" -> "keyword")) ~
+ ("startTime" -> ("type" -> "date")) ~
+ ("endTime" -> ("type" -> "date")) ~
+ ("evaluationClass" -> ("type" -> "keyword")) ~
+ ("engineParamsGeneratorClass" -> ("type" -> "keyword")) ~
+ ("batch" -> ("type" -> "keyword")) ~
+ ("evaluatorResults" -> ("type" -> "text")) ~
+ ("evaluatorResultsHTML" -> ("enabled" -> false)) ~
+ ("evaluatorResultsJSON" -> ("enabled" -> false))))
+ ESUtils.createMapping(client, index, estype, compact(render(mappingJson)))
def insert(i: EvaluationInstance): String = {
val id = i.id match {
@@ -85,9 +79,8 @@ class ESEvaluationInstances(client: ESClient, config: StorageClientConfig, index
}
def get(id: String): Option[EvaluationInstance] = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"GET",
s"/$index/$estype/$id",
Map.empty[String, String].asJava)
@@ -109,29 +102,23 @@ class ESEvaluationInstances(client: ESClient, config: StorageClientConfig, index
case e: IOException =>
error(s"Failed to access to /$index/$estype/$id", e)
None
- } finally {
- restClient.close()
}
}
def getAll(): Seq[EvaluationInstance] = {
- val restClient = client.open()
try {
val json =
("query" ->
("match_all" -> List.empty))
- ESUtils.getAll[EvaluationInstance](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[EvaluationInstance](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error("Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
def getCompleted(): Seq[EvaluationInstance] = {
- val restClient = client.open()
try {
val json =
("query" ->
@@ -140,22 +127,19 @@ class ESEvaluationInstances(client: ESClient, config: StorageClientConfig, index
("sort" ->
("startTime" ->
("order" -> "desc")))
- ESUtils.getAll[EvaluationInstance](restClient, index, estype, compact(render(json)))
+ ESUtils.getAll[EvaluationInstance](client, index, estype, compact(render(json)))
} catch {
case e: IOException =>
error("Failed to access to /$index/$estype/_search", e)
Nil
- } finally {
- restClient.close()
}
}
def update(i: EvaluationInstance): Unit = {
val id = i.id
- val restClient = client.open()
try {
val entity = new NStringEntity(write(i), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava,
@@ -171,15 +155,12 @@ class ESEvaluationInstances(client: ESClient, config: StorageClientConfig, index
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
- } finally {
- restClient.close()
}
}
def delete(id: String): Unit = {
- val restClient = client.open()
try {
- val response = restClient.performRequest(
+ val response = client.performRequest(
"DELETE",
s"/$index/$estype/$id",
Map("refresh" -> "true").asJava)
@@ -193,8 +174,6 @@ class ESEvaluationInstances(client: ESClient, config: StorageClientConfig, index
} catch {
case e: IOException =>
error(s"Failed to update $index/$estype/$id", e)
- } finally {
- restClient.close()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
index 6c0c4a7..5e1f4c1 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
@@ -28,6 +28,7 @@ import org.apache.http.util.EntityUtils
import org.apache.predictionio.data.storage.Event
import org.apache.predictionio.data.storage.LEvents
import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.client.{ResponseException, RestClient}
import org.joda.time.DateTime
import org.json4s._
import org.json4s.JsonDSL._
@@ -37,10 +38,9 @@ import org.json4s.ext.JodaTimeSerializers
import grizzled.slf4j.Logging
import org.apache.http.message.BasicHeader
-class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: String)
+class ESLEvents(val client: RestClient, config: StorageClientConfig, val index: String)
extends LEvents with Logging {
implicit val formats = DefaultFormats.lossless ++ JodaTimeSerializers.all
- val restClient = client.open()
def getEsType(appId: Int, channelId: Option[Int] = None): String = {
channelId.map { ch =>
@@ -52,7 +52,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
override def init(appId: Int, channelId: Option[Int] = None): Boolean = {
val estype = getEsType(appId, channelId)
- ESUtils.createIndex(restClient, index,
+ ESUtils.createIndex(client, index,
ESUtils.getNumberOfShards(config, index.toUpperCase),
ESUtils.getNumberOfReplicas(config, index.toUpperCase))
val json =
@@ -71,7 +71,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
("tags" -> ("type" -> "keyword")) ~
("prId" -> ("type" -> "keyword")) ~
("creationTime" -> ("type" -> "date"))))
- ESUtils.createMapping(restClient, index, estype, compact(render(json)))
+ ESUtils.createMapping(client, index, estype, compact(render(json)))
true
}
@@ -82,7 +82,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
("query" ->
("match_all" -> List.empty))
val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
- restClient.performRequest(
+ client.performRequest(
"POST",
s"/$index/$estype/_delete_by_query",
Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
@@ -99,9 +99,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
}
}
- override def close(): Unit = {
- restClient.close()
- }
+ override def close(): Unit = {}
override def futureInsert(
event: Event,
@@ -126,7 +124,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
("creationTime" -> ESUtils.formatUTCDateTime(event.creationTime)) ~
("properties" -> write(event.properties.toJObject))
val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/$id",
Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
@@ -185,7 +183,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
}.mkString("", "\n", "\n")
val entity = new StringEntity(json)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
"/_bulk",
Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
@@ -215,6 +213,29 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
}
}
+ private def exists(client: RestClient, estype: String, id: Int): Boolean = {
+ try {
+ client.performRequest(
+ "GET",
+ s"/$index/$estype/$id",
+ Map.empty[String, String].asJava).getStatusLine.getStatusCode match {
+ case 200 => true
+ case _ => false
+ }
+ } catch {
+ case e: ResponseException =>
+ e.getResponse.getStatusLine.getStatusCode match {
+ case 404 => false
+ case _ =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ false
+ }
+ case e: IOException =>
+ error(s"Failed to access to $index/$estype/$id", e)
+ false
+ }
+ }
+
override def futureGet(
eventId: String,
appId: Int,
@@ -227,7 +248,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
("term" ->
("eventId" -> eventId)))
val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/_search",
Map.empty[String, String].asJava,
@@ -260,7 +281,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
("term" ->
("eventId" -> eventId)))
val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/_delete_by_query",
Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
@@ -301,8 +322,8 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
startTime, untilTime, entityType, entityId,
eventNames, targetEntityType, targetEntityId, reversed)
limit.getOrElse(20) match {
- case -1 => ESUtils.getEventAll(restClient, index, estype, query).toIterator
- case size => ESUtils.getEvents(restClient, index, estype, query, size).toIterator
+ case -1 => ESUtils.getEventAll(client, index, estype, query).toIterator
+ case size => ESUtils.getEvents(client, index, estype, query, size).toIterator
}
} catch {
case e: IOException =>
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
index 9f0a188..75f7639 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
@@ -41,7 +41,7 @@ import org.json4s.native.JsonMethods._
import org.json4s.ext.JodaTimeSerializers
-class ESPEvents(client: ESClient, config: StorageClientConfig, index: String)
+class ESPEvents(client: RestClient, config: StorageClientConfig, index: String)
extends PEvents {
implicit val formats = DefaultFormats.lossless ++ JodaTimeSerializers.all
@@ -107,8 +107,6 @@ class ESPEvents(client: ESClient, config: StorageClientConfig, index: String)
eventIds: RDD[String],
appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
val estype = getEsType(appId, channelId)
- val restClient = client.open()
- try {
eventIds.foreachPartition { iter =>
iter.foreach { eventId =>
try {
@@ -117,28 +115,23 @@ class ESPEvents(client: ESClient, config: StorageClientConfig, index: String)
("term" ->
("eventId" -> eventId)))
val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/_delete_by_query",
Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
entity)
- val jsonResponse = parse(EntityUtils.toString(response.getEntity))
- val result = (jsonResponse \ "result").extract[String]
- result match {
- case "deleted" => true
- case _ =>
- logger.error(s"[$result] Failed to update $index/$estype:$eventId")
- false
- }
- } catch {
- case e: IOException =>
- logger.error(s"Failed to update $index/$estype:$eventId", e)
- false
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "deleted" =>
+ case _ =>
+ logger.error(s"[$result] Failed to update $index/$estype:$eventId")
}
+ } catch {
+ case e: IOException =>
+ logger.error(s"Failed to update $index/$estype:$eventId", e)
}
}
- } finally {
- restClient.close()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
index 9fd31a3..018ef85 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
@@ -35,30 +35,24 @@ import org.json4s.native.Serialization.write
import grizzled.slf4j.Logging
-class ESSequences(client: ESClient, config: StorageClientConfig, index: String) extends Logging {
+class ESSequences(client: RestClient, config: StorageClientConfig, index: String) extends Logging {
implicit val formats = DefaultFormats
private val estype = "sequences"
- val restClient = client.open()
- try {
- ESUtils.createIndex(restClient, index,
- ESUtils.getNumberOfShards(config, index.toUpperCase),
- ESUtils.getNumberOfReplicas(config, index.toUpperCase))
- val mappingJson =
- (estype ->
- ("_all" -> ("enabled" -> false)) ~
- ("properties" ->
- ("n" -> ("enabled" -> false))))
- ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
- } finally {
- restClient.close()
- }
+ ESUtils.createIndex(client, index,
+ ESUtils.getNumberOfShards(config, index.toUpperCase),
+ ESUtils.getNumberOfReplicas(config, index.toUpperCase))
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> false)) ~
+ ("properties" ->
+ ("n" -> ("enabled" -> false))))
+ ESUtils.createMapping(client, index, estype, compact(render(mappingJson)))
def genNext(name: String): Long = {
- val restClient = client.open()
try {
val entity = new NStringEntity(write("n" -> name), ContentType.APPLICATION_JSON)
- val response = restClient.performRequest(
+ val response = client.performRequest(
"POST",
s"/$index/$estype/$name",
Map("refresh" -> "false").asJava,
@@ -76,8 +70,6 @@ class ESSequences(client: ESClient, config: StorageClientConfig, index: String)
} catch {
case e: IOException =>
throw new StorageClientException(s"Failed to update $index/$estype/$name", e)
- } finally {
- restClient.close()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
index 647d180..d2c69b9 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
@@ -18,27 +18,84 @@
package org.apache.predictionio.data.storage.elasticsearch
import org.apache.http.HttpHost
+import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials}
+import org.apache.http.impl.client.BasicCredentialsProvider
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.apache.predictionio.data.storage.BaseStorageClient
import org.apache.predictionio.data.storage.StorageClientConfig
import org.apache.predictionio.data.storage.StorageClientException
+import org.apache.predictionio.workflow.CleanupFunctions
import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback
import grizzled.slf4j.Logging
-case class ESClient(hosts: Seq[HttpHost]) {
- def open(): RestClient = {
+object ESClient extends Logging {
+ private var _sharedRestClient: Option[RestClient] = None
+
+ def open(
+ hosts: Seq[HttpHost],
+ basicAuth: Option[(String, String)] = None): RestClient = {
try {
- RestClient.builder(hosts: _*).build()
+ val newClient = _sharedRestClient match {
+ case Some(c) => c
+ case None => {
+ var builder = RestClient.builder(hosts: _*)
+ builder = basicAuth match {
+ case Some((username, password)) => builder.setHttpClientConfigCallback(
+ new BasicAuthProvider(username, password))
+ case None => builder}
+ builder.build()
+ }
+ }
+ _sharedRestClient = Some(newClient)
+ newClient
} catch {
case e: Throwable =>
throw new StorageClientException(e.getMessage, e)
}
}
+
+ def close(): Unit = {
+ _sharedRestClient.foreach { client =>
+ client.close()
+ _sharedRestClient = None
+ }
+ }
}
-class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
- with Logging {
+class StorageClient(val config: StorageClientConfig)
+ extends BaseStorageClient with Logging {
+
override val prefix = "ES"
- val client = ESClient(ESUtils.getHttpHosts(config))
+ val usernamePassword = (
+ config.properties.get("USERNAME"),
+ config.properties.get("PASSWORD"))
+ val optionalBasicAuth: Option[(String, String)] = usernamePassword match {
+ case (None, None) => None
+ case (username, password) => Some(
+ (username.getOrElse(""), password.getOrElse("")))
+ }
+
+ CleanupFunctions.add { ESClient.close }
+
+ val client = ESClient.open(ESUtils.getHttpHosts(config), optionalBasicAuth)
+}
+
+class BasicAuthProvider(
+ val username: String,
+ val password: String)
+ extends HttpClientConfigCallback {
+
+ val credentialsProvider = new BasicCredentialsProvider()
+ credentialsProvider.setCredentials(
+ AuthScope.ANY,
+ new UsernamePasswordCredentials(username, password))
+
+ override def customizeHttpClient(
+ httpClientBuilder: HttpAsyncClientBuilder
+ ): HttpAsyncClientBuilder = {
+ httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClientSpec.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClientSpec.scala b/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClientSpec.scala
new file mode 100644
index 0000000..91cc4d6
--- /dev/null
+++ b/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClientSpec.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import org.apache.predictionio.data.storage.{App, Apps, Storage, StorageClientConfig}
+import org.elasticsearch.client.{RestClient, Response}
+import scala.collection.JavaConverters._
+
+import org.specs2._
+import org.specs2.specification.Step
+
+class ElasticsearchStorageClientSpec extends Specification {
+ def is = s2"""
+
+ PredictionIO Storage Elasticsearch REST Client Specification ${getESClient}
+
+ """
+
+ def getESClient = sequential ^ s2"""
+
+ StorageClient should
+ - initialize metadata store ${initMetadataStore(appsDO)}
+
+ """
+
+ def initMetadataStore(appsDO: Apps) = sequential ^ s2"""
+
+ creates an app ${createsApp(appsDO)}
+ gets apps ${getApps(appsDO)}
+
+ """
+
+ val indexName = "test_pio_storage_meta_" + hashCode
+
+ def appsDO: Apps = Storage.getDataObject[Apps](StorageTestUtils.elasticsearchSourceName, indexName)
+
+ def createsApp(appsDO: Apps) = {
+ val newId: Int = 123
+ val newApp: App = App(newId, "test1", Some("App for ElasticsearchStorageClientSpec"))
+ val id: Option[Int] = appsDO.insert(newApp)
+ val createdApp: Option[App] = appsDO.get(id.get)
+ createdApp.get.id mustEqual newId
+ }
+
+ def getApps(appsDO: Apps) = {
+ val apps: Seq[App] = appsDO.getAll()
+ println(s"Storage.config ${Storage.config}")
+ println(s"getApps ${apps}")
+ apps must beAnInstanceOf[Seq[App]]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageTestUtils.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageTestUtils.scala b/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageTestUtils.scala
new file mode 100644
index 0000000..f891c58
--- /dev/null
+++ b/storage/elasticsearch/src/test/scala/org/apache/predictionio/data/storage/elasticsearch/StorageTestUtils.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+object StorageTestUtils {
+ val elasticsearchSourceName = "ELASTICSEARCH"
+
+ def dropESIndex(namespace: String): Unit = {
+ // TODO
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/storage/hdfs/project/build.properties
----------------------------------------------------------------------
diff --git a/storage/hdfs/project/build.properties b/storage/hdfs/project/build.properties
new file mode 100644
index 0000000..64317fd
--- /dev/null
+++ b/storage/hdfs/project/build.properties
@@ -0,0 +1 @@
+sbt.version=0.13.15
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala b/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
index c101d3f..0372a44 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
@@ -25,6 +25,7 @@ import org.apache.predictionio.data.SparkVersionDependent
import org.apache.predictionio.tools.Runner
import org.apache.predictionio.workflow.WorkflowContext
import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.workflow.CleanupFunctions
import grizzled.slf4j.Logging
import org.apache.spark.sql.SaveMode
@@ -69,40 +70,45 @@ object EventsToFile extends Logging {
}
}
parser.parse(args, EventsToFileArgs()) map { args =>
- // get channelId
- val channels = Storage.getMetaDataChannels
- val channelMap = channels.getByAppid(args.appId).map(c => (c.name, c.id)).toMap
+ try {
+ // get channelId
+ val channels = Storage.getMetaDataChannels
+ val channelMap = channels.getByAppid(args.appId).map(c => (c.name, c.id)).toMap
- val channelId: Option[Int] = args.channel.map { ch =>
- if (!channelMap.contains(ch)) {
- error(s"Channel ${ch} doesn't exist in this app.")
- sys.exit(1)
+ val channelId: Option[Int] = args.channel.map { ch =>
+ if (!channelMap.contains(ch)) {
+ error(s"Channel ${ch} doesn't exist in this app.")
+ sys.exit(1)
+ }
+
+ channelMap(ch)
}
- channelMap(ch)
- }
+ val channelStr = args.channel.map(n => " Channel " + n).getOrElse("")
- val channelStr = args.channel.map(n => " Channel " + n).getOrElse("")
+ WorkflowUtils.modifyLogging(verbose = args.verbose)
+ @transient lazy implicit val formats = Utils.json4sDefaultFormats +
+ new EventJson4sSupport.APISerializer
+ val sc = WorkflowContext(
+ mode = "Export",
+ batch = "App ID " + args.appId + channelStr,
+ executorEnv = Runner.envStringToMap(args.env))
+ val sqlSession = SparkVersionDependent.sqlSession(sc)
+ val events = Storage.getPEvents()
+ val eventsRdd = events.find(appId = args.appId, channelId = channelId)(sc)
+ val jsonStringRdd = eventsRdd.map(write(_))
+ if (args.format == "json") {
+ jsonStringRdd.saveAsTextFile(args.outputPath)
+ } else {
+ val jsonDf = sqlSession.read.json(jsonStringRdd)
+ jsonDf.write.mode(SaveMode.ErrorIfExists).parquet(args.outputPath)
+ }
+ info(s"Events are exported to ${args.outputPath}/.")
+ info("Done.")
- WorkflowUtils.modifyLogging(verbose = args.verbose)
- @transient lazy implicit val formats = Utils.json4sDefaultFormats +
- new EventJson4sSupport.APISerializer
- val sc = WorkflowContext(
- mode = "Export",
- batch = "App ID " + args.appId + channelStr,
- executorEnv = Runner.envStringToMap(args.env))
- val sqlSession = SparkVersionDependent.sqlSession(sc)
- val events = Storage.getPEvents()
- val eventsRdd = events.find(appId = args.appId, channelId = channelId)(sc)
- val jsonStringRdd = eventsRdd.map(write(_))
- if (args.format == "json") {
- jsonStringRdd.saveAsTextFile(args.outputPath)
- } else {
- val jsonDf = sqlSession.read.json(jsonStringRdd)
- jsonDf.write.mode(SaveMode.ErrorIfExists).parquet(args.outputPath)
+ } finally {
+ CleanupFunctions.run()
}
- info(s"Events are exported to ${args.outputPath}/.")
- info("Done.")
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bf84ede6/tools/src/main/scala/org/apache/predictionio/tools/imprt/FileToEvents.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/imprt/FileToEvents.scala b/tools/src/main/scala/org/apache/predictionio/tools/imprt/FileToEvents.scala
index 4b333ab..11f5a52 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/imprt/FileToEvents.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/imprt/FileToEvents.scala
@@ -25,6 +25,7 @@ import org.apache.predictionio.data.storage.Storage
import org.apache.predictionio.tools.Runner
import org.apache.predictionio.workflow.WorkflowContext
import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.workflow.CleanupFunctions
import grizzled.slf4j.Logging
import org.json4s.native.Serialization._
@@ -66,41 +67,46 @@ object FileToEvents extends Logging {
}
}
parser.parse(args, FileToEventsArgs()) map { args =>
- // get channelId
- val channels = Storage.getMetaDataChannels
- val channelMap = channels.getByAppid(args.appId).map(c => (c.name, c.id)).toMap
+ try {
+ // get channelId
+ val channels = Storage.getMetaDataChannels
+ val channelMap = channels.getByAppid(args.appId).map(c => (c.name, c.id)).toMap
- val channelId: Option[Int] = args.channel.map { ch =>
- if (!channelMap.contains(ch)) {
- error(s"Channel ${ch} doesn't exist in this app.")
- sys.exit(1)
+ val channelId: Option[Int] = args.channel.map { ch =>
+ if (!channelMap.contains(ch)) {
+ error(s"Channel ${ch} doesn't exist in this app.")
+ sys.exit(1)
+ }
+
+ channelMap(ch)
}
- channelMap(ch)
- }
+ val channelStr = args.channel.map(n => " Channel " + n).getOrElse("")
- val channelStr = args.channel.map(n => " Channel " + n).getOrElse("")
+ WorkflowUtils.modifyLogging(verbose = args.verbose)
+ @transient lazy implicit val formats = Utils.json4sDefaultFormats +
+ new EventJson4sSupport.APISerializer
+ val sc = WorkflowContext(
+ mode = "Import",
+ batch = "App ID " + args.appId + channelStr,
+ executorEnv = Runner.envStringToMap(args.env))
+ val rdd = sc.textFile(args.inputPath).filter(_.trim.nonEmpty).map { json =>
+ Try(read[Event](json)).recoverWith {
+ case e: Throwable =>
+ error(s"\nmalformed json => $json")
+ Failure(e)
+ }.get
+ }
+ val events = Storage.getPEvents()
+ events.write(events = rdd,
+ appId = args.appId,
+ channelId = channelId)(sc)
+ info("Events are imported.")
+ info("Done.")
- WorkflowUtils.modifyLogging(verbose = args.verbose)
- @transient lazy implicit val formats = Utils.json4sDefaultFormats +
- new EventJson4sSupport.APISerializer
- val sc = WorkflowContext(
- mode = "Import",
- batch = "App ID " + args.appId + channelStr,
- executorEnv = Runner.envStringToMap(args.env))
- val rdd = sc.textFile(args.inputPath).filter(_.trim.nonEmpty).map { json =>
- Try(read[Event](json)).recoverWith {
- case e: Throwable =>
- error(s"\nmalformed json => $json")
- Failure(e)
- }.get
+ } finally {
+ CleanupFunctions.run()
}
- val events = Storage.getPEvents()
- events.write(events = rdd,
- appId = args.appId,
- channelId = channelId)(sc)
- info("Events are imported.")
- info("Done.")
}
}
}
[42/50] [abbrv] incubator-predictionio git commit: Update
templates.yaml - change user name
Posted by ch...@apache.org.
Update templates.yaml - change user name
Closes #431
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/47506576
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/47506576
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/47506576
Branch: refs/heads/master
Commit: 47506576342367aa07f0e53a7983bbf059b82d5f
Parents: 9c68d86
Author: Tomasz Stęczniewski <to...@gmail.com>
Authored: Tue Sep 12 20:19:53 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Sep 12 20:20:24 2017 -0700
----------------------------------------------------------------------
docs/manual/source/gallery/templates.yaml | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/47506576/docs/manual/source/gallery/templates.yaml
----------------------------------------------------------------------
diff --git a/docs/manual/source/gallery/templates.yaml b/docs/manual/source/gallery/templates.yaml
index 9f0ec1d..4642bf6 100644
--- a/docs/manual/source/gallery/templates.yaml
+++ b/docs/manual/source/gallery/templates.yaml
@@ -449,7 +449,7 @@
- template:
name: Deeplearning4j RNTN
- repo: "https://github.com/ts335793/template-scala-parallel-dl4j-rntn"
+ repo: "https://github.com/thomasste/template-scala-parallel-dl4j-rntn"
description: |-
Recursive Neural Tensor Network algorithm is supervised learning algorithm used to predict sentiment of sentences. This template is based on deeplearning4j RNTN example: https://github.com/SkymindIO/deeplearning4j-nlp-examples/tree/master/src/main/java/org/deeplearning4j/rottentomatoes/rntn. It's goal is to show how to integrate deeplearning4j library with PredictionIO.
tags: [classification, nlp]
@@ -459,7 +459,7 @@
status: alpha
pio_min_version: 0.9.2
apache_pio_convesion_required: "requires conversion"
- support_link: '<a href="https://github.com/ts335793/template-scala-parallel-dl4j-rntn/issues">Github issues</a>'
+ support_link: '<a href="https://github.com/thomasste/template-scala-parallel-dl4j-rntn/issues">Github issues</a>'
- template:
name: classifier-kafka-streaming-template
@@ -601,7 +601,7 @@
- template:
name: Spark Deeplearning4j Word2Vec
- repo: "https://github.com/ts335793/template-scala-spark-dl4j-word2vec"
+ repo: "https://github.com/thomasste/template-scala-spark-dl4j-word2vec"
description: |-
This template shows how to integrate Deeplearnign4j spark api with PredictionIO on example of app which uses Word2Vec algorithm to predict nearest words.
tags: [nlp]
@@ -611,7 +611,7 @@
status: stable
pio_min_version: 0.9.2
apache_pio_convesion_required: "requires conversion"
- support_link: '<a href="https://github.com/ts335793/template-scala-spark-dl4j-word2vec/issues">Github issues</a>'
+ support_link: '<a href="https://github.com/thomasste/template-scala-spark-dl4j-word2vec/issues">Github issues</a>'
- template:
name: Sentiment Analysis Template
@@ -629,7 +629,7 @@
- template:
name: Recursive Neural Networks (Sentiment Analysis)
- repo: "https://github.com/ts335793/template-scala-rnn"
+ repo: "https://github.com/thomasste/template-scala-rnn"
description: |-
Predicting sentiment of phrases with use of Recursive Neural Network algorithm and OpenNLP parser.
tags: [nlp]
@@ -639,7 +639,7 @@
status: stable
pio_min_version: 0.9.2
apache_pio_convesion_required: "requires conversion"
- support_link: '<a href="https://github.com/ts335793/template-scala-rnn/issues">Github issues</a>'
+ support_link: '<a href="https://github.com/thomasste/template-scala-rnn/issues">Github issues</a>'
- template:
name: CoreNLP Text Classification
[31/50] [abbrv] incubator-predictionio git commit: Fix use of case
class in textclassifier example
Posted by ch...@apache.org.
Fix use of case class in textclassifier example
Closes #426
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/9c68d86c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/9c68d86c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/9c68d86c
Branch: refs/heads/master
Commit: 9c68d86c61a06b08d618b8cd8177c9d76cd86618
Parents: 350b183
Author: Naoki Takezoe <ta...@apache.org>
Authored: Tue Aug 29 14:03:36 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Tue Aug 29 14:03:36 2017 +0900
----------------------------------------------------------------------
docs/manual/source/demo/textclassification.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/9c68d86c/docs/manual/source/demo/textclassification.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/demo/textclassification.html.md.erb b/docs/manual/source/demo/textclassification.html.md.erb
index 788178d..9830d3d 100644
--- a/docs/manual/source/demo/textclassification.html.md.erb
+++ b/docs/manual/source/demo/textclassification.html.md.erb
@@ -570,7 +570,7 @@ Once you have a vector of class probabilities, you can classify the text documen
def predict(doc : String) : PredictedResult = {
val x: Array[Double] = getScores(doc)
val y: (Double, Double) = (nb.labels zip x).maxBy(_._2)
- new PredictedResult(pd.categoryMap.getOrElse(y._1, ""), y._2)
+ PredictedResult(pd.categoryMap.getOrElse(y._1, ""), y._2)
}
```
[29/50] [abbrv] incubator-predictionio git commit: Update
eventmodel.html.md.erb
Posted by ch...@apache.org.
Update eventmodel.html.md.erb
Closes #422
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/350b183c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/350b183c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/350b183c
Branch: refs/heads/master
Commit: 350b183cacb96122dd3136f48c731f3ac17aa57c
Parents: 8e96d1b
Author: Nilmax Moura <ni...@gmail.com>
Authored: Tue Aug 29 10:03:06 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Tue Aug 29 10:04:45 2017 +0900
----------------------------------------------------------------------
docs/manual/source/datacollection/eventmodel.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/350b183c/docs/manual/source/datacollection/eventmodel.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/datacollection/eventmodel.html.md.erb b/docs/manual/source/datacollection/eventmodel.html.md.erb
index f40880c..47aee72 100644
--- a/docs/manual/source/datacollection/eventmodel.html.md.erb
+++ b/docs/manual/source/datacollection/eventmodel.html.md.erb
@@ -25,7 +25,7 @@ This section explains how to model your application data as events.
For example, your application may have users and some items which the user can interact with. Then you can model them as two entity types: **user** and **item** and the entityId can uniquely identify the entity within each entityType (e.g. user with ID 1, item with ID 1).
-An entity may peform some events (e.g user 1 does something), and entity may have properties associated with it (e.g. user may have gender, age, email etc). Hence, **events** involve **entities** and there are two types of events, respectively:
+An entity may peform some events (e.g user 1 does something), and entity may have properties associated with it (e.g. user may have gender, age, email etc). Hence, **events** involve **entities** and there are three types of events, respectively:
1. Generic events performed by an entity.
2. Special events for recording changes of an entity's properties
[40/50] [abbrv] incubator-predictionio git commit: [PIO-120] ensures
the train process exits gracefully after an Elasticsearch connection error
Posted by ch...@apache.org.
[PIO-120] ensures the train process exits gracefully after an Elasticsearch connection error
Closes #432
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/34518365
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/34518365
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/34518365
Branch: refs/heads/master
Commit: 3451836584e16f324dc90f973846618e6a5bc3c9
Parents: a0a2c12
Author: Mars Hall <ma...@heroku.com>
Authored: Mon Sep 11 17:13:26 2017 -0700
Committer: Mars Hall <ma...@heroku.com>
Committed: Mon Sep 11 17:13:26 2017 -0700
----------------------------------------------------------------------
.../predictionio/workflow/CreateWorkflow.scala | 238 ++++++++++---------
1 file changed, 121 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/34518365/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala b/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
index 303ed06..c5f32a6 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
@@ -134,144 +134,148 @@ object CreateWorkflow extends Logging {
}
def main(args: Array[String]): Unit = {
- val wfcOpt = parser.parse(args, WorkflowConfig())
- if (wfcOpt.isEmpty) {
- logger.error("WorkflowConfig is empty. Quitting")
- return
- }
+ try {
+ val wfcOpt = parser.parse(args, WorkflowConfig())
+ if (wfcOpt.isEmpty) {
+ logger.error("WorkflowConfig is empty. Quitting")
+ return
+ }
- val wfc = wfcOpt.get
+ val wfc = wfcOpt.get
- WorkflowUtils.modifyLogging(wfc.verbose)
+ WorkflowUtils.modifyLogging(wfc.verbose)
- val evaluation = wfc.evaluationClass.map { ec =>
- try {
- WorkflowUtils.getEvaluation(ec, getClass.getClassLoader)._2
- } catch {
- case e @ (_: ClassNotFoundException | _: NoSuchMethodException) =>
- error(s"Unable to obtain evaluation $ec. Aborting workflow.", e)
- sys.exit(1)
+ val evaluation = wfc.evaluationClass.map { ec =>
+ try {
+ WorkflowUtils.getEvaluation(ec, getClass.getClassLoader)._2
+ } catch {
+ case e @ (_: ClassNotFoundException | _: NoSuchMethodException) =>
+ error(s"Unable to obtain evaluation $ec. Aborting workflow.", e)
+ sys.exit(1)
+ }
}
- }
- val engineParamsGenerator = wfc.engineParamsGeneratorClass.map { epg =>
- try {
- WorkflowUtils.getEngineParamsGenerator(epg, getClass.getClassLoader)._2
- } catch {
- case e @ (_: ClassNotFoundException | _: NoSuchMethodException) =>
- error(s"Unable to obtain engine parameters generator $epg. " +
- "Aborting workflow.", e)
- sys.exit(1)
+ val engineParamsGenerator = wfc.engineParamsGeneratorClass.map { epg =>
+ try {
+ WorkflowUtils.getEngineParamsGenerator(epg, getClass.getClassLoader)._2
+ } catch {
+ case e @ (_: ClassNotFoundException | _: NoSuchMethodException) =>
+ error(s"Unable to obtain engine parameters generator $epg. " +
+ "Aborting workflow.", e)
+ sys.exit(1)
+ }
}
- }
- val pioEnvVars = wfc.env.map { e =>
- e.split(',').flatMap { p =>
- p.split('=') match {
- case Array(k, v) => List(k -> v)
- case _ => Nil
- }
- }.toMap
- }.getOrElse(Map.empty)
+ val pioEnvVars = wfc.env.map { e =>
+ e.split(',').flatMap { p =>
+ p.split('=') match {
+ case Array(k, v) => List(k -> v)
+ case _ => Nil
+ }
+ }.toMap
+ }.getOrElse(Map.empty)
- if (evaluation.isEmpty) {
- val variantJson = parse(stringFromFile(wfc.engineVariant))
- val engineFactory = if (wfc.engineFactory == "") {
- variantJson \ "engineFactory" match {
+ if (evaluation.isEmpty) {
+ val variantJson = parse(stringFromFile(wfc.engineVariant))
+ val engineFactory = if (wfc.engineFactory == "") {
+ variantJson \ "engineFactory" match {
+ case JString(s) => s
+ case _ =>
+ error("Unable to read engine factory class name from " +
+ s"${wfc.engineVariant}. Aborting.")
+ sys.exit(1)
+ }
+ } else wfc.engineFactory
+ val variantId = variantJson \ "id" match {
case JString(s) => s
case _ =>
- error("Unable to read engine factory class name from " +
+ error("Unable to read engine variant ID from " +
s"${wfc.engineVariant}. Aborting.")
sys.exit(1)
}
- } else wfc.engineFactory
- val variantId = variantJson \ "id" match {
- case JString(s) => s
- case _ =>
- error("Unable to read engine variant ID from " +
- s"${wfc.engineVariant}. Aborting.")
- sys.exit(1)
- }
- val (engineLanguage, engineFactoryObj) = try {
- WorkflowUtils.getEngine(engineFactory, getClass.getClassLoader)
- } catch {
- case e @ (_: ClassNotFoundException | _: NoSuchMethodException) =>
- error(s"Unable to obtain engine: ${e.getMessage}. Aborting workflow.")
- sys.exit(1)
- }
+ val (engineLanguage, engineFactoryObj) = try {
+ WorkflowUtils.getEngine(engineFactory, getClass.getClassLoader)
+ } catch {
+ case e @ (_: ClassNotFoundException | _: NoSuchMethodException) =>
+ error(s"Unable to obtain engine: ${e.getMessage}. Aborting workflow.")
+ sys.exit(1)
+ }
- val engine: BaseEngine[_, _, _, _] = engineFactoryObj()
+ val engine: BaseEngine[_, _, _, _] = engineFactoryObj()
- val customSparkConf = WorkflowUtils.extractSparkConf(variantJson)
- val workflowParams = WorkflowParams(
- verbose = wfc.verbosity,
- skipSanityCheck = wfc.skipSanityCheck,
- stopAfterRead = wfc.stopAfterRead,
- stopAfterPrepare = wfc.stopAfterPrepare,
- sparkEnv = WorkflowParams().sparkEnv ++ customSparkConf)
+ val customSparkConf = WorkflowUtils.extractSparkConf(variantJson)
+ val workflowParams = WorkflowParams(
+ verbose = wfc.verbosity,
+ skipSanityCheck = wfc.skipSanityCheck,
+ stopAfterRead = wfc.stopAfterRead,
+ stopAfterPrepare = wfc.stopAfterPrepare,
+ sparkEnv = WorkflowParams().sparkEnv ++ customSparkConf)
- // Evaluator Not Specified. Do training.
- if (!engine.isInstanceOf[Engine[_,_,_,_,_,_]]) {
- throw new NoSuchMethodException(s"Engine $engine is not trainable")
- }
+ // Evaluator Not Specified. Do training.
+ if (!engine.isInstanceOf[Engine[_,_,_,_,_,_]]) {
+ throw new NoSuchMethodException(s"Engine $engine is not trainable")
+ }
- val trainableEngine = engine.asInstanceOf[Engine[_, _, _, _, _, _]]
+ val trainableEngine = engine.asInstanceOf[Engine[_, _, _, _, _, _]]
- val engineParams = if (wfc.engineParamsKey == "") {
- trainableEngine.jValueToEngineParams(variantJson, wfc.jsonExtractor)
- } else {
- engineFactoryObj.engineParams(wfc.engineParamsKey)
- }
+ val engineParams = if (wfc.engineParamsKey == "") {
+ trainableEngine.jValueToEngineParams(variantJson, wfc.jsonExtractor)
+ } else {
+ engineFactoryObj.engineParams(wfc.engineParamsKey)
+ }
- val engineInstance = EngineInstance(
- id = "",
- status = "INIT",
- startTime = DateTime.now,
- endTime = DateTime.now,
- engineId = wfc.engineId,
- engineVersion = wfc.engineVersion,
- engineVariant = variantId,
- engineFactory = engineFactory,
- batch = wfc.batch,
- env = pioEnvVars,
- sparkConf = workflowParams.sparkEnv,
- dataSourceParams =
- JsonExtractor.paramToJson(wfc.jsonExtractor, engineParams.dataSourceParams),
- preparatorParams =
- JsonExtractor.paramToJson(wfc.jsonExtractor, engineParams.preparatorParams),
- algorithmsParams =
- JsonExtractor.paramsToJson(wfc.jsonExtractor, engineParams.algorithmParamsList),
- servingParams =
- JsonExtractor.paramToJson(wfc.jsonExtractor, engineParams.servingParams))
+ val engineInstance = EngineInstance(
+ id = "",
+ status = "INIT",
+ startTime = DateTime.now,
+ endTime = DateTime.now,
+ engineId = wfc.engineId,
+ engineVersion = wfc.engineVersion,
+ engineVariant = variantId,
+ engineFactory = engineFactory,
+ batch = wfc.batch,
+ env = pioEnvVars,
+ sparkConf = workflowParams.sparkEnv,
+ dataSourceParams =
+ JsonExtractor.paramToJson(wfc.jsonExtractor, engineParams.dataSourceParams),
+ preparatorParams =
+ JsonExtractor.paramToJson(wfc.jsonExtractor, engineParams.preparatorParams),
+ algorithmsParams =
+ JsonExtractor.paramsToJson(wfc.jsonExtractor, engineParams.algorithmParamsList),
+ servingParams =
+ JsonExtractor.paramToJson(wfc.jsonExtractor, engineParams.servingParams))
- val engineInstanceId = Storage.getMetaDataEngineInstances.insert(
- engineInstance)
+ val engineInstanceId = Storage.getMetaDataEngineInstances.insert(
+ engineInstance)
- CoreWorkflow.runTrain(
- env = pioEnvVars,
- params = workflowParams,
- engine = trainableEngine,
- engineParams = engineParams,
- engineInstance = engineInstance.copy(id = engineInstanceId))
- } else {
- val workflowParams = WorkflowParams(
- verbose = wfc.verbosity,
- skipSanityCheck = wfc.skipSanityCheck,
- stopAfterRead = wfc.stopAfterRead,
- stopAfterPrepare = wfc.stopAfterPrepare,
- sparkEnv = WorkflowParams().sparkEnv)
- val evaluationInstance = EvaluationInstance(
- evaluationClass = wfc.evaluationClass.get,
- engineParamsGeneratorClass = wfc.engineParamsGeneratorClass.get,
- batch = wfc.batch,
- env = pioEnvVars,
- sparkConf = workflowParams.sparkEnv
- )
- Workflow.runEvaluation(
- evaluation = evaluation.get,
- engineParamsGenerator = engineParamsGenerator.get,
- evaluationInstance = evaluationInstance,
- params = workflowParams)
+ CoreWorkflow.runTrain(
+ env = pioEnvVars,
+ params = workflowParams,
+ engine = trainableEngine,
+ engineParams = engineParams,
+ engineInstance = engineInstance.copy(id = engineInstanceId))
+ } else {
+ val workflowParams = WorkflowParams(
+ verbose = wfc.verbosity,
+ skipSanityCheck = wfc.skipSanityCheck,
+ stopAfterRead = wfc.stopAfterRead,
+ stopAfterPrepare = wfc.stopAfterPrepare,
+ sparkEnv = WorkflowParams().sparkEnv)
+ val evaluationInstance = EvaluationInstance(
+ evaluationClass = wfc.evaluationClass.get,
+ engineParamsGeneratorClass = wfc.engineParamsGeneratorClass.get,
+ batch = wfc.batch,
+ env = pioEnvVars,
+ sparkConf = workflowParams.sparkEnv
+ )
+ Workflow.runEvaluation(
+ evaluation = evaluation.get,
+ engineParamsGenerator = engineParamsGenerator.get,
+ evaluationInstance = evaluationInstance,
+ params = workflowParams)
+ }
+ } finally {
+ CleanupFunctions.run()
}
}
}
[17/50] [abbrv] incubator-predictionio git commit: [PIO-72] Fix class
loading for pio-shell
Posted by ch...@apache.org.
[PIO-72] Fix class loading for pio-shell
Closes #401
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d894b72a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d894b72a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d894b72a
Branch: refs/heads/master
Commit: d894b72af8975aef872aae5311f3c7f75aa00961
Parents: cc4e2e0
Author: Chan Lee <ch...@gmail.com>
Authored: Thu Jul 27 14:54:37 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Jul 27 14:54:37 2017 -0700
----------------------------------------------------------------------
bin/compute-classpath.sh | 23 +++++++++-----------
bin/pio-class | 2 --
bin/pio-shell | 5 ++---
tests/build_docker.sh | 13 ++++++-----
.../org/apache/predictionio/tools/Common.scala | 19 +++++++++-------
5 files changed, 31 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 032d111..cd0527f 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -17,20 +17,11 @@
# limitations under the License.
#
-SCALA_VERSION=2.10
-
# Figure out where PredictionIO is installed
FWDIR="$(cd `dirname $0`/..; pwd)"
. ${FWDIR}/bin/load-pio-env.sh
-# Build up classpath
-CLASSPATH="${PIO_CONF_DIR}"
-
-CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*:${FWDIR}/lib/spark/*"
-
-ASSEMBLY_DIR="${FWDIR}/assembly"
-
if [ -n "$JAVA_HOME" ]; then
JAR_CMD="$JAVA_HOME/bin/jar"
else
@@ -41,12 +32,19 @@ fi
if [ -f "${FWDIR}/RELEASE" ]; then
assembly_folder="${FWDIR}"/lib
else
- assembly_folder="${ASSEMBLY_DIR}"
+ assembly_folder="${FWDIR}"/assembly/src/universal/lib
fi
-ASSEMBLY_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null)
+MAIN_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null)
+DATA_JARS=$(ls "${assembly_folder}"/spark/pio-data-*assembly*.jar 2>/dev/null)
+# Comma-separated list of assembly jars for submitting to spark-shell
+ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -)
-CLASSPATH="$CLASSPATH:${ASSEMBLY_JAR}"
+# Build up classpath
+CLASSPATH="${PIO_CONF_DIR}"
+CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*"
+CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*"
+CLASSPATH="$CLASSPATH:${MAIN_JAR}"
# Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this
# assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts
@@ -70,5 +68,4 @@ if [ -n "$MYSQL_JDBC_DRIVER" ]; then
CLASSPATH="$CLASSPATH:$MYSQL_JDBC_DRIVER"
fi
-
echo "$CLASSPATH"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-class
----------------------------------------------------------------------
diff --git a/bin/pio-class b/bin/pio-class
index 8bde407..94cc520 100755
--- a/bin/pio-class
+++ b/bin/pio-class
@@ -22,8 +22,6 @@ case "`uname`" in
CYGWIN*) cygwin=true;;
esac
-SCALA_VERSION=2.10
-
# Figure out where PredictionIO is installed
FWDIR="$(cd `dirname $0`/..; pwd)"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-shell
----------------------------------------------------------------------
diff --git a/bin/pio-shell b/bin/pio-shell
index d47008f..5a5745a 100755
--- a/bin/pio-shell
+++ b/bin/pio-shell
@@ -55,11 +55,10 @@ fi
if [[ "$1" == "--with-spark" ]]
then
echo "Starting the PIO shell with the Apache Spark Shell."
- # compute the $ASSEMPLY_JAR, the location of the assemply jar, with
- # bin/compute-classpath.sh
+ # Get paths of assembly jars to pass to spark-shell
. ${PIO_HOME}/bin/compute-classpath.sh
shift
- $SPARK_HOME/bin/spark-shell --jars $ASSEMBLY_JAR $@
+ ${SPARK_HOME}/bin/spark-shell --jars ${ASSEMBLY_JARS} $@
else
echo -e "\033[0;33mStarting the PIO shell without Apache Spark.\033[0m"
echo -e "\033[0;33mIf you need the Apache Spark library, run 'pio-shell --with-spark [spark-submit arguments...]'.\033[0m"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tests/build_docker.sh
----------------------------------------------------------------------
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index d467e97..b17a304 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -38,11 +38,14 @@ fi
-Dhadoop.version=$PIO_HADOOP_VERSION \
-Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
sbt/sbt clean storage/clean
-rm -rf assembly/*.jar
-cp dist/lib/*.jar assembly/
-rm -rf lib/spark
-mkdir -p lib/spark
-cp dist/lib/spark/*.jar lib/spark
+
+assembly_folder=assembly/src/universal/lib
+rm -rf ${assembly_folder}/*.jar
+rm -rf ${assembly_folder}/spark
+mkdir -p ${assembly_folder}/spark
+
+cp dist/lib/*.jar ${assembly_folder}
+cp dist/lib/spark/*.jar ${assembly_folder}/spark
rm *.tar.gz
docker build -t predictionio/pio .
popd
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
index 0ddd644..047b8dd 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -65,6 +65,15 @@ object Common extends EitherLogging {
}
}
+ def getCoreDir(pioHome: String): String = {
+ if (new File(pioHome + File.separator + "RELEASE").exists) {
+ pioHome + File.separator + "lib"
+ } else {
+ Array(pioHome, "assembly", "src", "universal", "lib")
+ .mkString(File.separator)
+ }
+ }
+
def getEngineDirPath(directory: Option[String]): String = {
new File(directory.getOrElse(".")).getCanonicalPath
}
@@ -97,18 +106,12 @@ object Common extends EitherLogging {
def jarFilesAt(path: File): Array[File] = path.listFiles filter {
_.getName.toLowerCase.endsWith(".jar")
}
- jarFilesAt(new File(pioHome, "lib/spark"))
+ jarFilesAt(new File(getCoreDir(pioHome) + File.separator + "spark"))
}
def coreAssembly(pioHome: String): Expected[File] = {
val core = s"pio-assembly-${BuildInfo.version}.jar"
- val coreDir =
- if (new File(pioHome + File.separator + "RELEASE").exists) {
- new File(pioHome + File.separator + "lib")
- } else {
- new File(pioHome + File.separator + "assembly")
- }
- val coreFile = new File(coreDir, core)
+ val coreFile = new File(getCoreDir(pioHome), core)
if (coreFile.exists) {
Right(coreFile)
} else {
[35/50] [abbrv] incubator-predictionio git commit: [PIO-99] Check for
engine compile error before proceeding to build
Posted by ch...@apache.org.
[PIO-99] Check for engine compile error before proceeding to build
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/8dcd9481
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/8dcd9481
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/8dcd9481
Branch: refs/heads/master
Commit: 8dcd9481501d6f793438f8e92cc5cdec9818616e
Parents: bf84ede
Author: Chan Lee <ch...@gmail.com>
Authored: Tue Sep 5 14:31:35 2017 -0700
Committer: Chan Lee <ch...@gmail.com>
Committed: Tue Sep 5 21:11:22 2017 -0700
----------------------------------------------------------------------
.../apache/predictionio/tools/commands/Engine.scala | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/8dcd9481/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
index e3460a5..8380695 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -150,13 +150,16 @@ object Engine extends EitherLogging {
case Left(err) => return Left(err)
case Right(_) =>
- compile(buildArgs, pioHome, engineDirPath, verbose)
- info("Looking for an engine...")
- val jarFiles = jarFilesForScala(engineDirPath)
- if (jarFiles.isEmpty) {
- return logAndFail("No engine found. Your build might have failed. Aborting.")
+ compile(buildArgs, pioHome, engineDirPath, verbose) match {
+ case Left(err) => return Left(err)
+ case Right(_) =>
+ info("Looking for an engine...")
+ val jarFiles = jarFilesForScala(engineDirPath)
+ if (jarFiles.isEmpty) {
+ return logAndFail("No engine found. Your build might have failed. Aborting.")
+ }
+ jarFiles foreach { f => info(s"Found ${f.getName}") }
}
- jarFiles foreach { f => info(s"Found ${f.getName}")}
}
logAndSucceed("Build finished successfully.")
}
[24/50] [abbrv] incubator-predictionio git commit: [PIO-66] Document
JIRA processes
Posted by ch...@apache.org.
[PIO-66] Document JIRA processes
Closes #417
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/8e96d1b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/8e96d1b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/8e96d1b5
Branch: refs/heads/master
Commit: 8e96d1b59c90b0834cf55ce33ba8224dbb7e19f8
Parents: 55ce7b8
Author: shimamoto <sh...@apache.org>
Authored: Thu Aug 3 11:42:41 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Aug 3 11:42:41 2017 -0700
----------------------------------------------------------------------
docs/manual/data/nav/main.yml | 3 ++
.../source/community/contribute-code.html.md | 30 ++++++++++++----
docs/manual/source/resources/release.html.md | 38 ++++++++++++++++++++
3 files changed, 65 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/8e96d1b5/docs/manual/data/nav/main.yml
----------------------------------------------------------------------
diff --git a/docs/manual/data/nav/main.yml b/docs/manual/data/nav/main.yml
index 245fbaf..33c05eb 100644
--- a/docs/manual/data/nav/main.yml
+++ b/docs/manual/data/nav/main.yml
@@ -319,6 +319,9 @@ root:
url: '#'
children:
-
+ body: 'Release Cadence'
+ url: '/resources/release/'
+ -
body: 'Developing Engines with IntelliJ IDEA'
url: '/resources/intellij/'
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/8e96d1b5/docs/manual/source/community/contribute-code.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/community/contribute-code.html.md b/docs/manual/source/community/contribute-code.html.md
index f1c1810..d295427 100644
--- a/docs/manual/source/community/contribute-code.html.md
+++ b/docs/manual/source/community/contribute-code.html.md
@@ -52,6 +52,8 @@ bug fixes will follow a different path.
## How to Add / Propose a New Feature
+Before adding new features into JIRA, please check that the feature does not currently exist in JIRA.
+
1. To propose a new feature, simply
[subscribe](mailto:dev-subscribe@predictionio.incubator.apache.org) and post
your proposal to [Apache PredictionIO (incubating) Development Mailing List]
@@ -64,6 +66,26 @@ bug fixes will follow a different path.
(incubating) Development Mailing List as primary mode of communication. You
never know if someone else has a better idea. ;)
+### Adding ticket to JIRA
+
+1. Add a descriptive Summary and a detailed description
+2. Set Issue Type to Bug, Improvement, New Feature, Test or Wish
+3. Set Priority to Blocker, Critical, Major, Minor or Trivial
+4. Fill out Affects Version with the version of PredictionIO you are currently using
+5. Fill out Environment if needed for description of your bug / feature
+6. Please leave other fields blank
+
+### Triaging JIRA
+
+Tickets will be triaged by PredictionIO committers.
+
+- **Target Version**: Either a particular version or `Future` if to be done later
+ + Once a fix has been committed, the Fix Version will filled in with the appropriate release
+
+- **Component**: Each ticket will be annotated with one or more of the following Components
+ + **Core**: affects the main code branch / will be part of a release
+ + **Documentation**: affects the documents / will be pushed to livedoc branch
+ + **Templates**: affects one of the separate github repositories for a template
## How to Issue a Pull Request
@@ -159,10 +181,6 @@ layer for event store and metadata store.
Source code for http://predictionio.incubator.apache.org site, and any other
documentation support files.
-### engines
-
-Obsolete built-in engines. To be removed.
-
### examples
Complete code examples showing Apache PredictionIO (incubating)'s application.
@@ -171,9 +189,9 @@ Complete code examples showing Apache PredictionIO (incubating)'s application.
Embedded SBT (Simple Build Tool) launcher.
-### templates
+### storage
-Starting point of building your custom engine.
+Storage implementations.
### tools
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/8e96d1b5/docs/manual/source/resources/release.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/resources/release.html.md b/docs/manual/source/resources/release.html.md
new file mode 100644
index 0000000..87f8680
--- /dev/null
+++ b/docs/manual/source/resources/release.html.md
@@ -0,0 +1,38 @@
+---
+title: Release Cadence
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+## Versions
+
+Each PIO release will be versioned: `<MAJOR>`.`<FEATURE>`.`<MAINTENANCE>`
+
+- **MAJOR**: Major releases are not on any particular cadence. Major releases are likely to be
+ stable over a long period (one year or more)
+- **FEATURE**: We are targeting new feature releases every two months and will include new
+ features, improvements and bug fixes
+- **MAINTENANCE**: This is reserved for urgent bug fixes on the current release
+
+## Release Cadence
+
+Feature releases will occur every two months. Maintenance releases will be ad-hoc, as needed.
+
+- At the beginning of each release, committers nominate features for the upcoming release
+- At end of week, share JIRA link with dev user group to invite comments on the list
+- Committers modify the target list as necessary after comments are integrated and assign them to devs
[46/50] [abbrv] incubator-predictionio git commit: Prepare
0.12.0-incubating-rc1
Posted by ch...@apache.org.
Prepare 0.12.0-incubating-rc1
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/954f234b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/954f234b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/954f234b
Branch: refs/heads/master
Commit: 954f234b204ab6ca4b50ec951353232e2517d3c9
Parents: 2d95348
Author: Chan Lee <ch...@gmail.com>
Authored: Tue Sep 12 23:08:16 2017 -0700
Committer: Chan Lee <ch...@gmail.com>
Committed: Tue Sep 12 23:08:16 2017 -0700
----------------------------------------------------------------------
KEYS | 57 ++++++++++++++++++++
build.sbt | 2 +-
.../engines/recommendation-engine/build.sbt | 2 +-
3 files changed, 59 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/954f234b/KEYS
----------------------------------------------------------------------
diff --git a/KEYS b/KEYS
index 08286b8..0dcd781 100644
--- a/KEYS
+++ b/KEYS
@@ -129,3 +129,60 @@ MV2jufd24oqFWQmddOCCkIPYeDOavZvcxYevdftY3LX1B4WbiZOMHqh/HDH2NkyW
93sbykwywNMHkdFPE0109cM=
=QAQh
-----END PGP PUBLIC KEY BLOCK-----
+pub 4096R/4719A8F4 2017-09-12 [expires: 2021-09-12]
+uid [ultimate] Chan Lee <ch...@apache.org>
+sig 3 4719A8F4 2017-09-12 Chan Lee <ch...@apache.org>
+sub 4096R/A18B1E8F 2017-09-12 [expires: 2021-09-12]
+sig 4719A8F4 2017-09-12 Chan Lee <ch...@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFm4YoUBEAC3CFA/xIiqn+NoqSB8ya+mgnwfuL7XoRcR0gaZQ5BNXMjRZcqw
+5On1v2TTcXo6LcD/g7oxdBYWzUaubsCSmZzMp+cT4w4bmLr3bSZol0akNq4n2MgT
+q7jXOhXDhMdKzIdxuJe7wGtFGLjm8Macc6576MEYq1AUtdDNYuEMWr5PmQwsbNQQ
+CBkH8007trDlPygvzh+w/tLOHNbIv3ynHCIYeY/vYpm3XLHPEF0CjV05sDUMdHkb
+l5s9xYFOfPT6JqXEC1gnjanSvQ20MlLwk1D9AYlkTir+a17/igi2S5YGBUHXvxvU
+7xJrNfN9xDklUMUkHcyZNNWgIr1U7U//4hKm/3D2ele0AmMtL+Y0MmnSKvqXMiRf
+PTON/6rBwK4WdKsYtq1e2nUWv5Btfzb/oEovh8nmJwrpWUi4oBo8q7lBj9PEb1yY
+QXBnSjzg2C4zx/z8O1aHx0zb8njTMwknH0Ii/0ukMyCvC1y9yoDTEcD01ctivxwq
+h4I547vuzVQA7LrPob8fa003R5jR95+bvfedR4beJlDb4uTIS7654Usk58xzCyYt
+hVz7YrD8Zn1u841YYqw52pQ+d29OpVC6z/IaEN06v77TIW6uJVPxdQU/tUJM+fvT
+kFVVU8qcGo5SjPUFyc66s9eJ3NRBdoRQO8ls3YO9JxMNSd4wtSysj9vKOwARAQAB
+tB1DaGFuIExlZSA8Y2hhbmxlZUBhcGFjaGUub3JnPokCPQQTAQoAJwUCWbhihQIb
+AwUJB4YfgAULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRB+I2PYRxmo9Eq3D/9Q
+Tg45t1DXEq911JAfCDA/nIFNBaziPt3FIdKeWR3EGxwTie/KrSH7sL6ou40+GnSb
+DcWxfq3CobFkTyaAAF7NGt3cmP5e7H02EaEqrSfewUnV0eZlZOb7qoc4qWoG8vGe
+OwQQEY48DFxmzBzSZSyUvtgJQBX08mlE4jtEMh0QngkxyH5qGi+NEfbvCZaRWf/3
+nfErYSfFenQqqRmXol69zGJ+MPXDZLCBdpayTjNZRRs9Bw4B/VVm3zGgdm2GZ/2U
+G4aPNNBnqDBi6WDBqC8RwOtWcyvHKCXXu2suHTN79SIWhksuPQBILI0lMEfrdsdk
+76WUlgHLvdUAjb/g4a+cV7XkbSstGbMvKi7foM5V2o5FppaL7BApMdIZUN5xTAsh
+zbJ/Wyru69SgNYm/4byWOFyzZBvZLtodqhT/BUHerwOO2ZNtRijwct8Q6o7PePbu
+K/35KTCTmU5D+91igIa72981ePnShuG1qx1v9a3wpoENYRt6YPchW9vhnErKdh3L
+nMfgcobI0HESQDx2Si8n8YGYIwXNeUAdDhkuhULXs22XIAay503R5tYiBm3mTaRR
+q25Tl1uSBeMeNGRoGOfHl7s8oIx/kH54PzgmEbLtEUs4SaUUW1l92ShQgkj87lKX
+Zytqx27HUTwkUz3AIxBHs9VloJ2Sh/cmMXB70uTV1LkCDQRZuGKFARAAzPli/9oU
+QLC5Bp75UDIRLQk4kDqyaSfCgcNTuqsXxL0UR/IoICz8S4qo1UccwO6LoUNH5L7j
+EFHdeG5P2a/Y/WQUIduwGs6M4Btb8vC9Q6kw2P+6R7gCoq9L6QKGOWjOwCbd7H1e
+YnxB7IizTYzgHq8llr3V7VXXkBSytCMOXdCxOMwoSyNqJdQtrrV1XdzPSwaHZ5vn
+i+VMuKVM3XlS3cB/KhJTJckWfaQzHMWmCYZytz5GwSJuMz37/YYFKTBPvYfk6z3F
+kVnpYTK/TZ9ZalbVUQY39Xk9MRhF22FbY1/kdmTQaA9+lxEONvmULXo/vmZIrJlS
+ua+DdUhrbmz8E6zLvqsH5HcPqJx7nbv/wcjcp4jRPk8vUNu3OO7a4ClFBymJzwL3
+k9bgCoVEY/fo71jVSpeajzWR3psJjOFLIOufNortYx3AjdVR2TkyKko0k0nFJ3FV
+P2MN86nwCh89kU581Q/XLDmZFrlrqFek6V+J2fu4l42Bu4/rWlML4AGst7p7UsYU
+y/TbF9wJhHgpSV/70gcS2jW26xZpcjNM09fgGlaJFzeqCrxkuIfx5CbdJzkAeFFC
+3Nq9o6rcLZXAqVGlLJLl2hGF7mnrzm+7RpZkoIXwX0RyJVVLaz4e6TYXXygYTaD3
+kQK6fNabA1Yo+SvRiQLBGM9oAaNoe8ajfWEAEQEAAYkCJQQYAQoADwUCWbhihQIb
+DAUJB4YfgAAKCRB+I2PYRxmo9KlYD/0aP3YrtNjHdlmLMit9z1jS4hATMdIohS2M
+5woZkNogmol2SUXLWz0jGNSKlsPvmlFfD7BEJtzoal27GL5/LTu53a7XXtKMdPrT
+RF8ZHQd4SkvAmNySPAZo10utskWnkEBtVntHT4/T4KYaum+f+9DbmVLmBIvOMzWU
+qN5QIYI1HcXw/flY/7F/jH3pZWR1IZ+lgF8AERldYnMUYn+PmEo0YCsu2MulLkfr
+2puXx5k0U/dO2Ljnrai878maLuSH100T8nFnY0lJEvwTb1xns4Hszum//9JOk1kJ
+MpaD7iWKW8BtkN5bouWxT95Igi9I+9nxpDTF47s5fqCbhG7EGI7jeAJYaPUYkzof
+P/k4K44d4jdmPyxCDxcA2KhxU+8QaYYuKgKX51UZzYR2Xx9YKG5IJTsbZNeKvUku
+2T7F+k4B2Z/XbvjzBtsfdQx0hqiDwfw2ZOMTFiaRQKzTbuJ4jb9QoNZazAiuR6gO
+A9pyEhk6ZBWq9zw00ym3W5IGy/YTS/3buDOfgLL82Axp9TEEJo7vk6JN0I4nJScN
+CDBXY/trGluhnOmV1mQ2lJnq4FJ2sJSIVcbYTNahAD3cJ8ub+XFgyeMno0OTlrVg
+LxEW+096aYvYBaxyb9GdKZzUplRHOZWg89DjGM5EVIy8yST1XDyCCMQeK/XglWCA
+gslTn9eM5A==
+=Jacn
+-----END PGP PUBLIC KEY BLOCK-----
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/954f234b/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 1a439c2..5a38e6f 100644
--- a/build.sbt
+++ b/build.sbt
@@ -46,7 +46,7 @@ lazy val scalaSparkDepsVersion = Map(
name := "apache-predictionio-parent"
-version in ThisBuild := "0.11.1-SNAPSHOT"
+version in ThisBuild := "0.12.0-incubating"
organization in ThisBuild := "org.apache.predictionio"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/954f234b/tests/pio_tests/engines/recommendation-engine/build.sbt
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/build.sbt b/tests/pio_tests/engines/recommendation-engine/build.sbt
index e9e2436..70c11eb 100644
--- a/tests/pio_tests/engines/recommendation-engine/build.sbt
+++ b/tests/pio_tests/engines/recommendation-engine/build.sbt
@@ -26,6 +26,6 @@ name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.1-SNAPSHOT" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided",
"org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided")
[06/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/build.sbt b/examples/experimental/scala-parallel-similarproduct-localmodel/build.sbt
deleted file mode 100644
index 87207e1..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-similarproduct"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/data/import_eventserver.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/data/import_eventserver.py b/examples/experimental/scala-parallel-similarproduct-localmodel/data/import_eventserver.py
deleted file mode 100644
index 6107d1c..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/data/import_eventserver.py
+++ /dev/null
@@ -1,90 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Import sample data for similar product engine
-"""
-
-import predictionio
-import argparse
-import random
-
-SEED = 3
-
-def import_events(client):
- random.seed(SEED)
- count = 0
- print client.get_status()
- print "Importing data..."
-
- # generate 10 users, with user ids u1,u2,....,u10
- user_ids = ["u%s" % i for i in range(1, 11)]
- for user_id in user_ids:
- print "Set user", user_id
- client.create_event(
- event="$set",
- entity_type="user",
- entity_id=user_id
- )
- count += 1
-
- # generate 50 items, with item ids i1,i2,....,i50
- # random assign 1 to 4 categories among c1-c6 to items
- categories = ["c%s" % i for i in range(1, 7)]
- item_ids = ["i%s" % i for i in range(1, 51)]
- for item_id in item_ids:
- print "Set item", item_id
- client.create_event(
- event="$set",
- entity_type="item",
- entity_id=item_id,
- properties={
- "categories" : random.sample(categories, random.randint(1, 4))
- }
- )
- count += 1
-
- # each user randomly viewed 10 items
- for user_id in user_ids:
- for viewed_item in random.sample(item_ids, 10):
- print "User", user_id ,"views item", viewed_item
- client.create_event(
- event="view",
- entity_type="user",
- entity_id=user_id,
- target_entity_type="item",
- target_entity_id=viewed_item
- )
- count += 1
-
- print "%s events are imported." % count
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description="Import sample data for similar product engine")
- parser.add_argument('--access_key', default='invald_access_key')
- parser.add_argument('--url', default="http://localhost:7070")
-
- args = parser.parse_args()
- print args
-
- client = predictionio.EventClient(
- access_key=args.access_key,
- url=args.url,
- threads=5,
- qsize=500)
- import_events(client)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/data/send_query.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/data/send_query.py b/examples/experimental/scala-parallel-similarproduct-localmodel/data/send_query.py
deleted file mode 100644
index 8678b15..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/data/send_query.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Send sample query to prediction engine
-"""
-
-import predictionio
-engine_client = predictionio.EngineClient(url="http://localhost:8000")
-print engine_client.send_query({"items": ["i1", "i3"], "num": 4})
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/engine.json b/examples/experimental/scala-parallel-similarproduct-localmodel/engine.json
deleted file mode 100644
index c55849f..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/engine.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.template.similarproduct.SimilarProductEngine",
- "datasource": {
- "params" : {
- "appId": 9
- }
- },
- "algorithms": [
- {
- "name": "als",
- "params": {
- "rank": 10,
- "numIterations" : 20,
- "lambda": 0.01,
- "seed": 3
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/project/assembly.sbt b/examples/experimental/scala-parallel-similarproduct-localmodel/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/ALSAlgorithm.scala b/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/ALSAlgorithm.scala
deleted file mode 100644
index 62c0c00..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/ALSAlgorithm.scala
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.P2LAlgorithm // ADDED
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.IPersistentModel
-import org.apache.predictionio.controller.IPersistentModelLoader
-import org.apache.predictionio.data.storage.BiMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.mllib.recommendation.ALS
-import org.apache.spark.mllib.recommendation.{Rating => MLlibRating}
-
-import grizzled.slf4j.Logger
-
-import scala.collection.mutable.PriorityQueue
-
-case class ALSAlgorithmParams(
- rank: Int,
- numIterations: Int,
- lambda: Double,
- seed: Option[Long]) extends Params
-
-class ALSLocalModel( // MODIFIED
- val productFeatures: Map[Int, Array[Double]], // MODIFIED
- val itemStringIntMap: BiMap[String, Int],
- val items: Map[Int, Item]
-) extends Serializable { // MODIFIED
-
- // MODIFIED
-
- @transient lazy val itemIntStringMap = itemStringIntMap.inverse
-
- override def toString = {
- s" productFeatures: [${productFeatures.size}]" + // MODIFIED
- s"(${productFeatures.take(2).toList}...)" +
- s" itemStringIntMap: [${itemStringIntMap.size}]" +
- s"(${itemStringIntMap.take(2).toString}...)]" +
- s" items: [${items.size}]" +
- s"(${items.take(2).toString}...)]"
- }
-}
-
-/**
- * Use ALS to build item x feature matrix
- */
-class ALSAlgorithm(val ap: ALSAlgorithmParams)
- // MODIFIED
- extends P2LAlgorithm[PreparedData, ALSLocalModel, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: PreparedData): ALSLocalModel = { // MODIFIED
- require(!data.viewEvents.take(1).isEmpty,
- s"viewEvents in PreparedData cannot be empty." +
- " Please check if DataSource generates TrainingData" +
- " and Preprator generates PreparedData correctly.")
- require(!data.users.take(1).isEmpty,
- s"users in PreparedData cannot be empty." +
- " Please check if DataSource generates TrainingData" +
- " and Preprator generates PreparedData correctly.")
- require(!data.items.take(1).isEmpty,
- s"items in PreparedData cannot be empty." +
- " Please check if DataSource generates TrainingData" +
- " and Preprator generates PreparedData correctly.")
- // create User and item's String ID to integer index BiMap
- val userStringIntMap = BiMap.stringInt(data.users.keys)
- val itemStringIntMap = BiMap.stringInt(data.items.keys)
-
- // collect Item as Map and convert ID to Int index
- val items: Map[Int, Item] = data.items.map { case (id, item) =>
- (itemStringIntMap(id), item)
- }.collectAsMap.toMap
-
- val mllibRatings = data.viewEvents
- .map { r =>
- // Convert user and item String IDs to Int index for MLlib
- val uindex = userStringIntMap.getOrElse(r.user, -1)
- val iindex = itemStringIntMap.getOrElse(r.item, -1)
-
- if (uindex == -1)
- logger.info(s"Couldn't convert nonexistent user ID ${r.user}"
- + " to Int index.")
-
- if (iindex == -1)
- logger.info(s"Couldn't convert nonexistent item ID ${r.item}"
- + " to Int index.")
-
- ((uindex, iindex), 1)
- }.filter { case ((u, i), v) =>
- // keep events with valid user and item index
- (u != -1) && (i != -1)
- }.reduceByKey(_ + _) // aggregate all view events of same user-item pair
- .map { case ((u, i), v) =>
- // MLlibRating requires integer index for user and item
- MLlibRating(u, i, v)
- }
-
- // MLLib ALS cannot handle empty training data.
- require(!mllibRatings.take(1).isEmpty,
- s"mllibRatings cannot be empty." +
- " Please check if your events contain valid user and item ID.")
-
- // seed for MLlib ALS
- val seed = ap.seed.getOrElse(System.nanoTime)
-
- val m = ALS.trainImplicit(
- ratings = mllibRatings,
- rank = ap.rank,
- iterations = ap.numIterations,
- lambda = ap.lambda,
- blocks = -1,
- alpha = 1.0,
- seed = seed)
-
- new ALSLocalModel( // MODIFIED
- productFeatures = m.productFeatures.collectAsMap.toMap, // MODIFIED
- itemStringIntMap = itemStringIntMap,
- items = items
- )
- }
-
- // MODIFIED
- def predict(model: ALSLocalModel, query: Query): PredictedResult = {
-
- // convert items to Int index
- val queryList: Set[Int] = query.items.map(model.itemStringIntMap.get(_))
- .flatten.toSet
-
- // MODIFIED
- val queryFeatures: Vector[Array[Double]] = queryList.toVector
- .map { item =>
- // productFeatures may not contain the requested item
- val qf: Option[Array[Double]] = model.productFeatures.get(item)
- qf
- }.flatten
-
- val whiteList: Option[Set[Int]] = query.whiteList.map( set =>
- set.map(model.itemStringIntMap.get(_)).flatten
- )
- val blackList: Option[Set[Int]] = query.blackList.map ( set =>
- set.map(model.itemStringIntMap.get(_)).flatten
- )
-
- val ord = Ordering.by[(Int, Double), Double](_._2).reverse
-
- val indexScores: Array[(Int, Double)] = if (queryFeatures.isEmpty) {
- logger.info(s"No productFeatures vector for query items ${query.items}.")
- Array[(Int, Double)]()
- } else {
- model.productFeatures // MODIFIED
- .mapValues { f =>
- queryFeatures.map{ qf =>
- cosine(qf, f)
- }.reduce(_ + _)
- }
- .filter(_._2 > 0) // keep items with score > 0
- .toArray // MODIFIED
- }
-
- val filteredScore = indexScores.view.filter { case (i, v) =>
- isCandidateItem(
- i = i,
- items = model.items,
- categories = query.categories,
- queryList = queryList,
- whiteList = whiteList,
- blackList = blackList
- )
- }
-
- val topScores = getTopN(filteredScore, query.num)(ord).toArray
-
- val itemScores = topScores.map { case (i, s) =>
- new ItemScore(
- item = model.itemIntStringMap(i),
- score = s
- )
- }
-
- new PredictedResult(itemScores)
- }
-
- private
- def getTopN[T](s: Seq[T], n: Int)(implicit ord: Ordering[T]): Seq[T] = {
-
- val q = PriorityQueue()
-
- for (x <- s) {
- if (q.size < n)
- q.enqueue(x)
- else {
- // q is full
- if (ord.compare(x, q.head) < 0) {
- q.dequeue()
- q.enqueue(x)
- }
- }
- }
-
- q.dequeueAll.toSeq.reverse
- }
-
- private
- def cosine(v1: Array[Double], v2: Array[Double]): Double = {
- val size = v1.size
- var i = 0
- var n1: Double = 0
- var n2: Double = 0
- var d: Double = 0
- while (i < size) {
- n1 += v1(i) * v1(i)
- n2 += v2(i) * v2(i)
- d += v1(i) * v2(i)
- i += 1
- }
- val n1n2 = (math.sqrt(n1) * math.sqrt(n2))
- if (n1n2 == 0) 0 else (d / n1n2)
- }
-
- private
- def isCandidateItem(
- i: Int,
- items: Map[Int, Item],
- categories: Option[Set[String]],
- queryList: Set[Int],
- whiteList: Option[Set[Int]],
- blackList: Option[Set[Int]]
- ): Boolean = {
- whiteList.map(_.contains(i)).getOrElse(true) &&
- blackList.map(!_.contains(i)).getOrElse(true) &&
- // discard items in query as well
- (!queryList.contains(i)) &&
- // filter categories
- categories.map { cat =>
- items(i).categories.map { itemCat =>
- // keep this item if has ovelap categories with the query
- !(itemCat.toSet.intersect(cat).isEmpty)
- }.getOrElse(false) // discard this item if it has no categories
- }.getOrElse(true)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/DataSource.scala
deleted file mode 100644
index af8b974..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-case class DataSourceParams(appId: Int) extends Params
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- val eventsDb = Storage.getPEvents()
-
- // create a RDD of (entityID, User)
- val usersRDD: RDD[(String, User)] = eventsDb.aggregateProperties(
- appId = dsp.appId,
- entityType = "user"
- )(sc).map { case (entityId, properties) =>
- val user = try {
- User()
- } catch {
- case e: Exception => {
- logger.error(s"Failed to get properties ${properties} of" +
- s" user ${entityId}. Exception: ${e}.")
- throw e
- }
- }
- (entityId, user)
- }
-
- // create a RDD of (entityID, Item)
- val itemsRDD: RDD[(String, Item)] = eventsDb.aggregateProperties(
- appId = dsp.appId,
- entityType = "item"
- )(sc).map { case (entityId, properties) =>
- val item = try {
- // Assume categories is optional property of item.
- Item(categories = properties.getOpt[List[String]]("categories"))
- } catch {
- case e: Exception => {
- logger.error(s"Failed to get properties ${properties} of" +
- s" item ${entityId}. Exception: ${e}.")
- throw e
- }
- }
- (entityId, item)
- }
-
- // get all "user" "view" "item" events
- val viewEventsRDD: RDD[ViewEvent] = eventsDb.find(
- appId = dsp.appId,
- entityType = Some("user"),
- eventNames = Some(List("view")),
- // targetEntityType is optional field of an event.
- targetEntityType = Some(Some("item")))(sc)
- // eventsDb.find() returns RDD[Event]
- .map { event =>
- val viewEvent = try {
- event.event match {
- case "view" => ViewEvent(
- user = event.entityId,
- item = event.targetEntityId.get,
- t = event.eventTime.getMillis)
- case _ => throw new Exception(s"Unexpected event ${event} is read.")
- }
- } catch {
- case e: Exception => {
- logger.error(s"Cannot convert ${event} to ViewEvent." +
- s" Exception: ${e}.")
- throw e
- }
- }
- viewEvent
- }
-
- new TrainingData(
- users = usersRDD,
- items = itemsRDD,
- viewEvents = viewEventsRDD
- )
- }
-}
-
-case class User()
-
-case class Item(categories: Option[List[String]])
-
-case class ViewEvent(user: String, item: String, t: Long)
-
-class TrainingData(
- val users: RDD[(String, User)],
- val items: RDD[(String, Item)],
- val viewEvents: RDD[ViewEvent]
-) extends Serializable {
- override def toString = {
- s"users: [${users.count()} (${users.take(2).toList}...)]" +
- s"items: [${items.count()} (${items.take(2).toList}...)]" +
- s"viewEvents: [${viewEvents.count()}] (${viewEvents.take(2).toList}...)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Engine.scala
deleted file mode 100644
index 52b19fe..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-
-case class Query(
- items: List[String],
- num: Int,
- categories: Option[Set[String]],
- whiteList: Option[Set[String]],
- blackList: Option[Set[String]]
-)
-
-case class PredictedResult(
- itemScores: Array[ItemScore]
-)
-
-case class ItemScore(
- item: String,
- score: Double
-)
-
-object SimilarProductEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- classOf[Preparator],
- Map("als" -> classOf[ALSAlgorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Preparator.scala
deleted file mode 100644
index e3394b0..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.PPreparator
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(
- users = trainingData.users,
- items = trainingData.items,
- viewEvents = trainingData.viewEvents)
- }
-}
-
-class PreparedData(
- val users: RDD[(String, User)],
- val items: RDD[(String, Item)],
- val viewEvents: RDD[ViewEvent]
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Serving.scala
deleted file mode 100644
index 3e115d5..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/build.sbt b/examples/experimental/scala-parallel-trim-app/build.sbt
deleted file mode 100644
index fdce0d6..0000000
--- a/examples/experimental/scala-parallel-trim-app/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-vanilla"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/engine.json b/examples/experimental/scala-parallel-trim-app/engine.json
deleted file mode 100644
index 5695856..0000000
--- a/examples/experimental/scala-parallel-trim-app/engine.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.apache.predictionio.examples.experimental.trimapp.VanillaEngine",
- "datasource": {
- "params" : {
- "srcAppId": 13,
- "dstAppId": 15,
- "startTime": "2014-07-01T00:00:00.000Z",
- "untilTime": "2015-01-01T00:00:00.000Z"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/project/assembly.sbt b/examples/experimental/scala-parallel-trim-app/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-trim-app/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/src/main/scala/Algorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/src/main/scala/Algorithm.scala b/examples/experimental/scala-parallel-trim-app/src/main/scala/Algorithm.scala
deleted file mode 100644
index f46a7b7..0000000
--- a/examples/experimental/scala-parallel-trim-app/src/main/scala/Algorithm.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.trimapp
-
-import org.apache.predictionio.controller.P2LAlgorithm
-import org.apache.predictionio.controller.Params
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-//case class AlgorithmParams(mult: Int) extends Params
-
-//class Algorithm(val ap: AlgorithmParams)
-class Algorithm
- extends P2LAlgorithm[TrainingData, Model, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: TrainingData): Model = {
- new Model
- }
-
- def predict(model: Model, query: Query): PredictedResult = {
- // Prefix the query with the model data
- PredictedResult(p = "")
- }
-}
-
-class Model extends Serializable {
- override def toString = "Model"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-trim-app/src/main/scala/DataSource.scala
deleted file mode 100644
index 4f03dba..0000000
--- a/examples/experimental/scala-parallel-trim-app/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.trimapp
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import com.github.nscala_time.time.Imports._
-
-import grizzled.slf4j.Logger
-
-case class DataSourceParams(
- srcAppId: Int,
- dstAppId: Int,
- startTime: Option[DateTime],
- untilTime: Option[DateTime]
-) extends Params
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- val eventsDb = Storage.getPEvents()
- logger.info(s"TrimApp: $dsp")
-
-
- logger.info(s"Read events from appId ${dsp.srcAppId}")
- val srcEvents: RDD[Event] = eventsDb.find(
- appId = dsp.srcAppId,
- startTime = dsp.startTime,
- untilTime = dsp.untilTime
- )(sc)
-
- val dstEvents: Array[Event] = eventsDb.find(appId = dsp.dstAppId)(sc).take(1)
-
- if (dstEvents.size > 0) {
- throw new Exception(s"DstApp ${dsp.dstAppId} is not empty. Quitting.")
- }
-
- logger.info(s"Write events to appId ${dsp.dstAppId}")
- eventsDb.write(srcEvents, dsp.dstAppId)(sc)
-
- logger.info(s"Finish writing events to appId ${dsp.dstAppId}")
-
- new TrainingData()
- }
-}
-
-class TrainingData(
-) extends Serializable {
- override def toString = ""
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-trim-app/src/main/scala/Engine.scala
deleted file mode 100644
index d45baf3..0000000
--- a/examples/experimental/scala-parallel-trim-app/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.trimapp
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller._
-
-case class Query(q: String) extends Serializable
-
-case class PredictedResult(p: String) extends Serializable
-
-object VanillaEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- PIdentityPreparator(classOf[DataSource]),
- Map("" -> classOf[Algorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-trim-app/src/main/scala/Preparator.scala
deleted file mode 100644
index fd8efde..0000000
--- a/examples/experimental/scala-parallel-trim-app/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.trimapp
-
-import org.apache.predictionio.controller.PPreparator
-import org.apache.predictionio.data.storage.Event
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-/*
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(events = trainingData.events)
- }
-}
-
-class PreparedData(
- val events: RDD[Event]
-) extends Serializable
-*/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-trim-app/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-trim-app/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-trim-app/src/main/scala/Serving.scala
deleted file mode 100644
index 4fc376f..0000000
--- a/examples/experimental/scala-parallel-trim-app/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.trimapp
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override
- def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-recommendations/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-recommendations/README.md b/examples/experimental/scala-recommendations/README.md
deleted file mode 100644
index 1bb80da..0000000
--- a/examples/experimental/scala-recommendations/README.md
+++ /dev/null
@@ -1,146 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-Distributed Recommendation Engine with RDD-based Model using MLlib's ALS
-========================================================================
-
-This document describes a recommendation engine that is based on Apache Spark's
-MLlib collaborative filtering algorithm.
-
-
-Prerequisite
-------------
-
-Make sure you have built PredictionIO and setup storage described
-[here](/README.md).
-
-
-High Level Description
-----------------------
-
-This engine demonstrates how one can integrate MLlib's algorithms that produce
-an RDD-based model, deploy it in production and serve real-time queries.
-
-For details about MLlib's collaborative filtering algorithms, please refer to
-https://spark.apache.org/docs/latest/mllib-collaborative-filtering.html.
-
-All code definition can be found [here](src/main/scala/Run.scala).
-
-
-### Data Source
-
-Training data is located at `/examples/data/movielens.txt`. Values are delimited
-by double colons (::). The first column are user IDs. The second column are item
-IDs. The third column are ratings. In this example, they are represented as
-`RDD[Rating]`, as described in the official MLlib guide.
-
-
-### Preparator
-
-The preparator in this example is an identity function, i.e. no further
-preparation is done on the training data.
-
-
-### Algorithm
-
-This example engine contains one single algorithm that wraps around MLlib. The
-`train()` method simply calls MLlib's `ALS.train()` method.
-
-
-### Serving
-
-This example engine uses `FirstServing`, which serves only predictions from the
-first algorithm. Since there is only one algorithm in this engine, predictions
-from MLlib's ALS algorithm will be served.
-
-
-Training a Model
-----------------
-
-This example provides a set of ready-to-use parameters for each component
-mentioned in the previous section. They are located inside the `params`
-subdirectory.
-
-Before training, you must let PredictionIO know about the engine. Run the
-following command to build and register the engine.
-```
-$ cd $PIO_HOME/examples/scala-recommendations
-$ ../../bin/pio build
-```
-where `$PIO_HOME` is the root directory of the PredictionIO code tree.
-
-To start training, use the following command. You need to install the
-[`gfortran`](https://github.com/mikiobraun/jblas/wiki/Missing-Libraries)
-runtime library if it is not already present on your nodes. For Debian and
-Ubuntu systems this would be "`sudo apt-get install libgfortran3`".
-```
-$ cd $PIO_HOME/examples/scala-recommendations
-$ ../../bin/pio train
-```
-This will train a model and save it in PredictionIO's metadata storage. Notice
-that when the run is completed, it will display a run ID, like below.
-```
-2014-08-27 23:13:54,596 INFO SparkContext - Job finished: saveAsObjectFile at Run.scala:68, took 0.299989372 s
-2014-08-27 23:13:54,736 INFO APIDebugWorkflow$ - Saved engine instance with ID: txHBY2XRQTKFnxC-lYoVgA
-```
-
-
-Deploying a Real-time Prediction Server
----------------------------------------
-
-Following from instructions above, you should have trained a model. Use the
-following command to start a server.
-```
-$ cd $PIO_HOME/examples/scala-recommendations
-$ ../../bin/pio deploy
-```
-This will create a server that by default binds to http://localhost:8000. You
-can visit that page in your web browser to check its status.
-
-To perform real-time predictions, try the following. This predicts on how user 1 will rate item (movie) 4. As in all collaborative filtering algorithms, it will not handle the case of a cold user (when the user has not rated any movies).
-```
-$ curl -H "Content-Type: application/json" -d '[1,4]' http://localhost:8000/queries.json
-```
-Congratulations! You have just trained an ALS model and is able to perform real
-time prediction distributed across an Apache Spark cluster!
-
-
-Production Prediction Server Deployment
----------------------------------------
-
-Prediction servers support reloading models on the fly with the latest completed
-run.
-
-1. Assuming you already have a running prediction server from the previous
- section, go to http://localhost:8000 to check its status. Take note of the
- **Run ID** at the top.
-
-2. Run training and deploy again. There is no need to manually terminate the previous deploy instance.
-
- ```
- $ cd $PIO_HOME/examples/scala-recommendations
- $ ../../bin/pio train
- $ ../../bin/pio deploy
- ```
-
-3. Refresh the page at http://localhost:8000, you should see the prediction
- server status page with a new **Run ID** at the top.
-
-Congratulations! You have just experienced a production-ready setup that can
-reload itself automatically after every training! Simply add the training or
-evaluation command to your *crontab*, and your setup will be able to re-deploy
-itself automatically at a regular interval.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-recommendations/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-recommendations/build.sbt b/examples/experimental/scala-recommendations/build.sbt
deleted file mode 100644
index bda1930..0000000
--- a/examples/experimental/scala-recommendations/build.sbt
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "example-scala-recommendations"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "commons-io" % "commons-io" % "2.4",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided",
- "org.json4s" %% "json4s-native" % "3.2.10")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-recommendations/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-recommendations/engine.json b/examples/experimental/scala-recommendations/engine.json
deleted file mode 100644
index a4c30e4..0000000
--- a/examples/experimental/scala-recommendations/engine.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "id": "default",
- "description": "Item Recommendations examples",
- "engineFactory": "org.apache.spark.mllib.recommendation.engine.RecommendationEngine",
- "datasource": {
- "params": {
- "filepath": "../data/movielens.txt"
- }
- },
- "algorithms": [
- {
- "name": "",
- "params": {
- "rank": 10,
- "numIterations": 20,
- "lambda": 0.01,
- "persistModel": true
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-recommendations/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-recommendations/project/assembly.sbt b/examples/experimental/scala-recommendations/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-recommendations/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-recommendations/src/main/scala/Run.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-recommendations/src/main/scala/Run.scala b/examples/experimental/scala-recommendations/src/main/scala/Run.scala
deleted file mode 100644
index 7493d05..0000000
--- a/examples/experimental/scala-recommendations/src/main/scala/Run.scala
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.mllib.recommendation.engine
-
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.IPersistentModel
-import org.apache.predictionio.controller.IPersistentModelLoader
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.PAlgorithm
-import org.apache.predictionio.controller.PIdentityPreparator
-import org.apache.predictionio.controller.LFirstServing
-import org.apache.predictionio.controller.Utils
-import org.apache.predictionio.controller.Workflow
-import org.apache.predictionio.controller.WorkflowParams
-
-import org.apache.commons.io.FileUtils
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.mllib.recommendation.ALS
-import org.apache.spark.mllib.recommendation.Rating
-import org.apache.spark.mllib.recommendation.MatrixFactorizationModel
-import org.json4s._
-
-import scala.io.Source
-
-import java.io.File
-
-case class DataSourceParams(val filepath: String) extends Params
-
-case class DataSource(val dsp: DataSourceParams)
- extends PDataSource[DataSourceParams, Null, RDD[Rating], (Int, Int), Double] {
-
- override
- def read(sc: SparkContext)
- : Seq[(Null, RDD[Rating], RDD[((Int, Int), Double)])] = {
- val data = sc.textFile(dsp.filepath)
- val ratings: RDD[Rating] = data.map(_.split("::") match {
- case Array(user, item, rate) =>
- Rating(user.toInt, item.toInt, rate.toDouble)
- })
-
- val featureTargets: RDD[((Int, Int), Double)] = ratings.map {
- case Rating(user, product, rate) => ((user, product), rate)
- }
-
- Seq((null, ratings, featureTargets))
- }
-}
-
-case class AlgorithmParams(
- val rank: Int = 10,
- val numIterations: Int = 20,
- val lambda: Double = 0.01,
- val persistModel: Boolean = false) extends Params
-
-class PMatrixFactorizationModel(rank: Int,
- userFeatures: RDD[(Int, Array[Double])],
- productFeatures: RDD[(Int, Array[Double])])
- extends MatrixFactorizationModel(rank, userFeatures, productFeatures)
- with IPersistentModel[AlgorithmParams] {
- def save(id: String, params: AlgorithmParams, sc: SparkContext): Boolean = {
- if (params.persistModel) {
- sc.parallelize(Seq(rank)).saveAsObjectFile(s"/tmp/${id}/rank")
- userFeatures.saveAsObjectFile(s"/tmp/${id}/userFeatures")
- productFeatures.saveAsObjectFile(s"/tmp/${id}/productFeatures")
- }
- params.persistModel
- }
-}
-
-object PMatrixFactorizationModel
- extends IPersistentModelLoader[AlgorithmParams, PMatrixFactorizationModel] {
- def apply(id: String, params: AlgorithmParams, sc: Option[SparkContext]) = {
- new PMatrixFactorizationModel(
- rank = sc.get.objectFile[Int](s"/tmp/${id}/rank").first,
- userFeatures = sc.get.objectFile(s"/tmp/${id}/userFeatures"),
- productFeatures = sc.get.objectFile(s"/tmp/${id}/productFeatures"))
- }
-}
-
-class ALSAlgorithm(val ap: AlgorithmParams)
- extends PAlgorithm[AlgorithmParams, RDD[Rating],
- PMatrixFactorizationModel, (Int, Int), Double] {
-
- def train(data: RDD[Rating]): PMatrixFactorizationModel = {
- val m = ALS.train(data, ap.rank, ap.numIterations, ap.lambda)
- new PMatrixFactorizationModel(
- rank = m.rank,
- userFeatures = m.userFeatures,
- productFeatures = m.productFeatures)
- }
-
- override
- def batchPredict(
- model: PMatrixFactorizationModel,
- feature: RDD[(Long, (Int, Int))]): RDD[(Long, Double)] = {
- val indexlessFeature = feature.values
-
- val prediction: RDD[Rating] = model.predict(indexlessFeature)
-
- val p: RDD[((Int, Int), Double)] = prediction.map {
- r => ((r.user, r.product), r.rating)
- }
-
- feature.map{ _.swap }
- .join(p)
- .map { case (up, (fi, r)) => (fi,r) }
- }
-
- def predict(
- model: PMatrixFactorizationModel, feature: (Int, Int)): Double = {
- model.predict(feature._1, feature._2)
- }
-
- @transient override lazy val querySerializer =
- Utils.json4sDefaultFormats + new Tuple2IntSerializer
-}
-
-object Run {
- def main(args: Array[String]) {
- val dsp = DataSourceParams("data/movielens.txt")
- val ap = AlgorithmParams()
-
- Workflow.run(
- dataSourceClassOpt = Some(classOf[DataSource]),
- dataSourceParams = dsp,
- preparatorClassOpt = Some(PIdentityPreparator(classOf[DataSource])),
- algorithmClassMapOpt = Some(Map("" -> classOf[ALSAlgorithm])),
- algorithmParamsList = Seq(("", ap)),
- servingClassOpt = Some(LFirstServing(classOf[ALSAlgorithm])),
- params = WorkflowParams(
- batch = "Imagine: P Recommendations",
- verbose = 1
- )
- )
- }
-}
-
-object RecommendationEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- PIdentityPreparator(classOf[DataSource]),
- Map("" -> classOf[ALSAlgorithm]),
- LFirstServing(classOf[ALSAlgorithm]))
- }
-}
-
-
-class Tuple2IntSerializer extends CustomSerializer[(Int, Int)](format => (
- {
- case JArray(List(JInt(x), JInt(y))) => (x.intValue, y.intValue)
- },
- {
- case x: (Int, Int) => JArray(List(JInt(x._1), JInt(x._2)))
- }
-))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/build.sbt b/examples/experimental/scala-refactor-test/build.sbt
deleted file mode 100644
index c3935fe..0000000
--- a/examples/experimental/scala-refactor-test/build.sbt
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-vanilla"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- //"org.apache.predictionio" %% "core" % "0.8.6" % "provided",
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/engine.json b/examples/experimental/scala-refactor-test/engine.json
deleted file mode 100644
index 4cfb64d..0000000
--- a/examples/experimental/scala-refactor-test/engine.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "pio.refactor.VanillaEngine",
- "datasource": {
- "params" : {
- "appId": 1
- }
- },
- "algorithms": [
- {
- "name": "algo",
- "params": {
- "mult" : 1
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/project/assembly.sbt b/examples/experimental/scala-refactor-test/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-refactor-test/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/src/main/scala/Algorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/src/main/scala/Algorithm.scala b/examples/experimental/scala-refactor-test/src/main/scala/Algorithm.scala
deleted file mode 100644
index 51d329e..0000000
--- a/examples/experimental/scala-refactor-test/src/main/scala/Algorithm.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pio.refactor
-
-import org.apache.predictionio.controller.P2LAlgorithm
-import org.apache.predictionio.controller.Params
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-case class AlgorithmParams(mult: Int) extends Params
-
-class Algorithm(val ap: AlgorithmParams)
- // extends PAlgorithm if Model contains RDD[]
- extends P2LAlgorithm[TrainingData, Model, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: TrainingData): Model = {
- // Simply count number of events
- // and multiple it by the algorithm parameter
- // and store the number as model
- val count = data.events.reduce(_ + _) * ap.mult
- logger.error("Algo.train")
- new Model(mc = count)
- }
-
- def predict(model: Model, query: Query): PredictedResult = {
- // Prefix the query with the model data
- //val result = s"${model.mc}-${query.q}"
- logger.error("Algo.predict")
- PredictedResult(p = model.mc + query.q)
- }
-}
-
-class Model(val mc: Int) extends Serializable {
- override def toString = s"mc=${mc}"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/src/main/scala/DataSource.scala b/examples/experimental/scala-refactor-test/src/main/scala/DataSource.scala
deleted file mode 100644
index 1b25e42..0000000
--- a/examples/experimental/scala-refactor-test/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pio.refactor
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller._
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-//case class DataSourceParams(appId: Int) extends Params
-
-class DataSource
- extends PDataSource[
- TrainingData,
- EmptyEvaluationInfo,
- Query,
- ActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- new TrainingData(
- events = sc.parallelize(0 until 100)
- )
- }
-
- override
- def readEval(sc: SparkContext)
- : Seq[(TrainingData, EmptyEvaluationInfo, RDD[(Query, ActualResult)])] =
- {
- logger.error("Datasource!!!")
- (0 until 3).map { ex =>
- (
- readTraining(sc),
- new EmptyEvaluationInfo(),
- sc
- .parallelize((0 until 20)
- .map {i => (Query(i), new ActualResult())}))
- }
- }
-}
-
-class TrainingData(
- val events: RDD[Int]
-) extends Serializable {
- override def toString = {
- s"events: [${events.count()}] (${events.take(2).toList}...)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/src/main/scala/Engine.scala b/examples/experimental/scala-refactor-test/src/main/scala/Engine.scala
deleted file mode 100644
index b064004..0000000
--- a/examples/experimental/scala-refactor-test/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pio.refactor
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller._
-//import org.apache.predictionio.workflow.CoreWorkflow
-import grizzled.slf4j.Logger
-
-case class Query(q: Int)
-
-case class PredictedResult(p: Int)
-
-case class ActualResult()
-
-object VanillaEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- //classOf[Preparator],
- PIdentityPreparator(classOf[DataSource]),
- Map("algo" -> classOf[Algorithm]),
- classOf[Serving])
- }
-}
-
-object Runner {
- @transient lazy val logger = Logger[this.type]
-
- def main(args: Array[String]) {
- val engine = VanillaEngine()
- val engineParams = EngineParams(
- algorithmParamsList = Seq(("algo", AlgorithmParams(2)))
- )
-
- logger.error("Runner. before evaluation!!!")
- val evaluator = new VanillaEvaluator()
-
- logger.error("Runner before runEval!!!")
- Workflow.runEval(
- engine = engine,
- engineParams = engineParams,
- evaluator = evaluator,
- evaluatorParams = EmptyParams())
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/src/main/scala/Evaluator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/src/main/scala/Evaluator.scala b/examples/experimental/scala-refactor-test/src/main/scala/Evaluator.scala
deleted file mode 100644
index 495a025..0000000
--- a/examples/experimental/scala-refactor-test/src/main/scala/Evaluator.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pio.refactor
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller._
-
-class VanillaEvaluator
- extends Evaluator[EmptyEvaluationInfo, Query, PredictedResult,
- ActualResult, Int, Int, String] {
-
- def evaluateUnit(q: Query, p: PredictedResult, a: ActualResult): Int = {
- q.q - p.p
- }
-
- def evaluateSet(evalInfo: EmptyEvaluationInfo, eus: Seq[Int]): Int = eus.sum
-
- def evaluateAll(input: Seq[(EmptyEvaluationInfo, Int)]): String = {
- val sum = input.map(_._2).sum
- s"VanillaEvaluator(${input.size}, $sum)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/src/main/scala/Preparator.scala b/examples/experimental/scala-refactor-test/src/main/scala/Preparator.scala
deleted file mode 100644
index ca05459..0000000
--- a/examples/experimental/scala-refactor-test/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
-package pio.refactor
-
-import org.apache.predictionio.controller.PPreparator
-import org.apache.predictionio.data.storage.Event
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(events = trainingData.events)
- }
-}
-
-class PreparedData(
- val events: RDD[Event]
-) extends Serializable
-
-*/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-refactor-test/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-refactor-test/src/main/scala/Serving.scala b/examples/experimental/scala-refactor-test/src/main/scala/Serving.scala
deleted file mode 100644
index 839ab1e..0000000
--- a/examples/experimental/scala-refactor-test/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package pio.refactor
-
-import org.apache.predictionio.controller.LServing
-import grizzled.slf4j.Logger
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
- override def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- logger.error("Serving.serve")
- predictedResults.head
- }
-}
[41/50] [abbrv] incubator-predictionio git commit: [PIO-59] Use
/dev/urandom to create access keys.
Posted by ch...@apache.org.
[PIO-59] Use /dev/urandom to create access keys.
Creating access keys based on /dev/random may take a long time
on virtualized or clound environment for example more than
10 minutes.
Closes #367
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/bd034f55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/bd034f55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/bd034f55
Branch: refs/heads/master
Commit: bd034f55dfc3205fd4e23805912262e11171473d
Parents: 3451836
Author: Juha Syrjälä <ju...@iki.fi>
Authored: Tue Sep 12 19:37:57 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Sep 12 19:39:18 2017 -0700
----------------------------------------------------------------------
.../scala/org/apache/predictionio/data/storage/AccessKeys.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bd034f55/data/src/main/scala/org/apache/predictionio/data/storage/AccessKeys.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/AccessKeys.scala b/data/src/main/scala/org/apache/predictionio/data/storage/AccessKeys.scala
index b96bd82..ca1c8b3 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/AccessKeys.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/AccessKeys.scala
@@ -66,7 +66,7 @@ trait AccessKeys {
/** Default implementation of key generation */
def generateKey: String = {
- val sr = SecureRandom.getInstanceStrong
+ val sr = new SecureRandom
val srBytes = Array.fill(48)(0.toByte)
sr.nextBytes(srBytes)
Base64.encodeBase64URLSafeString(srBytes) match {
[44/50] [abbrv] incubator-predictionio git commit: minor typo with
'following'
Posted by ch...@apache.org.
minor typo with 'following'
Closes #434
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/951d7a0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/951d7a0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/951d7a0a
Branch: refs/heads/master
Commit: 951d7a0a00a0d18cd13000c21e060e1bf1794e39
Parents: aa86def
Author: aayush kumar <aa...@gmail.com>
Authored: Tue Sep 12 20:38:13 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Sep 12 20:38:40 2017 -0700
----------------------------------------------------------------------
docs/manual/source/install/install-sourcecode.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/951d7a0a/docs/manual/source/install/install-sourcecode.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/install/install-sourcecode.html.md.erb b/docs/manual/source/install/install-sourcecode.html.md.erb
index 23c5503..f8410af 100644
--- a/docs/manual/source/install/install-sourcecode.html.md.erb
+++ b/docs/manual/source/install/install-sourcecode.html.md.erb
@@ -19,7 +19,7 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-INFO: Assuming you are following the directory structure in the followoing,
+INFO: Assuming you are following the directory structure in the following,
replace `/home/abc` with your own home directory wherever you see it.
## Downloading Source Code
[07/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Serving.scala
deleted file mode 100644
index 48fda35..0000000
--- a/examples/experimental/scala-parallel-recommendation-custom-datasource/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/.gitignore b/examples/experimental/scala-parallel-recommendation-entitymap/.gitignore
deleted file mode 100644
index ea4e89d..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-data/sample_movielens_data.txt
-manifest.json
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/build.sbt b/examples/experimental/scala-parallel-recommendation-entitymap/build.sbt
deleted file mode 100644
index 91e94f2..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-recommendation"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/data/import_eventserver.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/data/import_eventserver.py b/examples/experimental/scala-parallel-recommendation-entitymap/data/import_eventserver.py
deleted file mode 100644
index 57a9528..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/data/import_eventserver.py
+++ /dev/null
@@ -1,107 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Import sample data for recommendation engine
-"""
-
-import predictionio
-import argparse
-import random
-
-SEED = 3
-
-def import_events(client):
- random.seed(SEED)
- count = 0
- print "Importing data..."
-
- # generate 10 users, with user uid1,2,....,10
- # with some random attributes
- user_ids = [ ("uid"+str(i)) for i in range(1, 11)]
- for user_id in user_ids:
- print "Set user", user_id
- client.create_event(
- event="$set",
- entity_type="user",
- entity_id=user_id,
- properties={
- "attr0" : float(random.randint(0, 4)),
- "attr1" : random.randint(10, 14),
- "attr2" : random.randint(20, 24)
- }
- )
- count += 1
-
- # generate 50 items, with iid1,2,....,50
- # with some randome attributes
- item_ids = [ ("iid"+str(i)) for i in range(1, 51)]
- for item_id in item_ids:
- print "Set item", item_id
- client.create_event(
- event="$set",
- entity_type="item",
- entity_id=item_id,
- properties={
- "attrA" : random.choice(["something1", "something2", "valueX"]),
- "attrB" : random.randint(10, 30),
- "attrC" : random.choice([True, False])
- }
- )
- count += 1
-
- # each user randomly rate or buy 10 items
- for user_id in user_ids:
- for viewed_item in random.sample(item_ids, 10):
- if (random.randint(0, 1) == 1):
- print "User", user_id ,"rates item", viewed_item
- client.create_event(
- event="rate",
- entity_type="user",
- entity_id=user_id,
- target_entity_type="item",
- target_entity_id=item_id,
- properties= { "rating" : float(random.randint(1, 6)) }
- )
- else:
- print "User", user_id ,"buys item", viewed_item
- client.create_event(
- event="buy",
- entity_type="user",
- entity_id=user_id,
- target_entity_type="item",
- target_entity_id=item_id
- )
- count += 1
-
- print "%s events are imported." % count
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description="Import sample data for recommendation engine")
- parser.add_argument('--access_key', default='invald_access_key')
- parser.add_argument('--url', default="http://localhost:7070")
-
- args = parser.parse_args()
- print args
-
- client = predictionio.EventClient(
- access_key=args.access_key,
- url=args.url,
- threads=5,
- qsize=500)
- import_events(client)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/data/send_query.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/data/send_query.py b/examples/experimental/scala-parallel-recommendation-entitymap/data/send_query.py
deleted file mode 100644
index cb453b4..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/data/send_query.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Send sample query to prediction engine
-"""
-
-import predictionio
-engine_client = predictionio.EngineClient(url="http://localhost:8000")
-print engine_client.send_query({"user": "uid1", "num": 4})
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/engine.json b/examples/experimental/scala-parallel-recommendation-entitymap/engine.json
deleted file mode 100644
index 256a153..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/engine.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.template.recommendation.RecommendationEngine",
- "datasource": {
- "params": {
- "appId": 1
- }
- },
- "algorithms": [
- {
- "name": "als",
- "params": {
- "rank": 10,
- "numIterations": 20,
- "lambda": 0.01
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/project/assembly.sbt b/examples/experimental/scala-parallel-recommendation-entitymap/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSAlgorithm.scala b/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSAlgorithm.scala
deleted file mode 100644
index 211ac7d..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSAlgorithm.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PAlgorithm
-import org.apache.predictionio.controller.Params
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.mllib.recommendation.ALS
-import org.apache.spark.mllib.recommendation.{Rating => MLlibRating}
-import org.apache.spark.mllib.recommendation.ALSModel
-
-import grizzled.slf4j.Logger
-
-case class ALSAlgorithmParams(
- rank: Int,
- numIterations: Int,
- lambda: Double) extends Params
-
-class ALSAlgorithm(val ap: ALSAlgorithmParams)
- extends PAlgorithm[PreparedData, ALSModel, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: PreparedData): ALSModel = {
- // Convert user and item String IDs to Int index for MLlib
- val mllibRatings = data.ratings.map( r =>
- // MLlibRating requires integer index for user and item
- MLlibRating(data.users(r.user).toInt,
- data.items(r.item).toInt, r.rating)
- )
- val m = ALS.train(mllibRatings, ap.rank, ap.numIterations, ap.lambda)
- new ALSModel(
- rank = m.rank,
- userFeatures = m.userFeatures,
- productFeatures = m.productFeatures,
- users = data.users,
- items = data.items)
- }
-
- def predict(model: ALSModel, query: Query): PredictedResult = {
- // Convert String ID to Int index for Mllib
- model.users.get(query.user).map { userInt =>
- // recommendProducts() returns Array[MLlibRating], which uses item Int
- // index. Convert it to String ID for returning PredictedResult
- val itemScores = model.recommendProducts(userInt.toInt, query.num)
- .map (r => ItemScore(model.items(r.product.toLong), r.rating))
- new PredictedResult(itemScores)
- }.getOrElse{
- logger.info(s"No prediction for unknown user ${query.user}.")
- new PredictedResult(Array.empty)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSModel.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSModel.scala b/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSModel.scala
deleted file mode 100644
index 856f735..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/ALSModel.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.mllib.recommendation
-// This must be the same package as Spark's MatrixFactorizationModel because
-// MatrixFactorizationModel's constructor is private and we are using
-// its constructor in order to save and load the model
-
-import org.template.recommendation.ALSAlgorithmParams
-
-import org.template.recommendation.User
-import org.template.recommendation.Item
-
-import org.apache.predictionio.controller.IPersistentModel
-import org.apache.predictionio.controller.IPersistentModelLoader
-import org.apache.predictionio.data.storage.EntityMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class ALSModel(
- override val rank: Int,
- override val userFeatures: RDD[(Int, Array[Double])],
- override val productFeatures: RDD[(Int, Array[Double])],
- val users: EntityMap[User],
- val items: EntityMap[Item])
- extends MatrixFactorizationModel(rank, userFeatures, productFeatures)
- with IPersistentModel[ALSAlgorithmParams] {
-
- def save(id: String, params: ALSAlgorithmParams,
- sc: SparkContext): Boolean = {
-
- sc.parallelize(Seq(rank)).saveAsObjectFile(s"/tmp/${id}/rank")
- userFeatures.saveAsObjectFile(s"/tmp/${id}/userFeatures")
- productFeatures.saveAsObjectFile(s"/tmp/${id}/productFeatures")
- sc.parallelize(Seq(users))
- .saveAsObjectFile(s"/tmp/${id}/users")
- sc.parallelize(Seq(items))
- .saveAsObjectFile(s"/tmp/${id}/items")
- true
- }
-
- override def toString = {
- s"userFeatures: [${userFeatures.count()}]" +
- s"(${userFeatures.take(2).toList}...)" +
- s" productFeatures: [${productFeatures.count()}]" +
- s"(${productFeatures.take(2).toList}...)" +
- s" users: [${users.size}]" +
- s"(${users.take(2)}...)" +
- s" items: [${items.size}]" +
- s"(${items.take(2)}...)"
- }
-}
-
-object ALSModel
- extends IPersistentModelLoader[ALSAlgorithmParams, ALSModel] {
- def apply(id: String, params: ALSAlgorithmParams,
- sc: Option[SparkContext]) = {
- new ALSModel(
- rank = sc.get.objectFile[Int](s"/tmp/${id}/rank").first,
- userFeatures = sc.get.objectFile(s"/tmp/${id}/userFeatures"),
- productFeatures = sc.get.objectFile(s"/tmp/${id}/productFeatures"),
- users = sc.get
- .objectFile[EntityMap[User]](s"/tmp/${id}/users").first,
- items = sc.get
- .objectFile[EntityMap[Item]](s"/tmp/${id}/items").first)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/DataSource.scala
deleted file mode 100644
index 72dcbf9..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-import org.apache.predictionio.data.storage.EntityMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-case class DataSourceParams(appId: Int) extends Params
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- val eventsDb = Storage.getPEvents()
-
- val users: EntityMap[User] = eventsDb.extractEntityMap[User](
- appId = dsp.appId,
- entityType = "user",
- required = Some(Seq("attr0", "attr1", "attr2"))
- )(sc) { dm =>
- User(
- attr0 = dm.get[Double]("attr0"),
- attr1 = dm.get[Int]("attr1"),
- attr2 = dm.get[Int]("attr2")
- )
- }
-
- val items: EntityMap[Item] = eventsDb.extractEntityMap[Item](
- appId = dsp.appId,
- entityType = "item",
- required = Some(Seq("attrA", "attrB", "attrC"))
- )(sc) { dm =>
- Item(
- attrA = dm.get[String]("attrA"),
- attrB = dm.get[Int]("attrB"),
- attrC = dm.get[Boolean]("attrC")
- )
- }
-
- val eventsRDD: RDD[Event] = eventsDb.find(
- appId = dsp.appId,
- entityType = Some("user"),
- eventNames = Some(List("rate", "buy")), // read "rate" and "buy" event
- // targetEntityType is optional field of an event.
- targetEntityType = Some(Some("item")))(sc)
-
- val ratingsRDD: RDD[Rating] = eventsRDD.map { event =>
- val rating = try {
- val ratingValue: Double = event.event match {
- case "rate" => event.properties.get[Double]("rating")
- case "buy" => 4.0 // map buy event to rating value of 4
- case _ => throw new Exception(s"Unexpected event ${event} is read.")
- }
- // entityId and targetEntityId is String
- Rating(event.entityId,
- event.targetEntityId.get,
- ratingValue)
- } catch {
- case e: Exception => {
- logger.error(s"Cannot convert ${event} to Rating. Exception: ${e}.")
- throw e
- }
- }
- rating
- }
- new TrainingData(users, items, ratingsRDD)
- }
-}
-
-case class User(
- attr0: Double,
- attr1: Int,
- attr2: Int
-)
-
-case class Item(
- attrA: String,
- attrB: Int,
- attrC: Boolean
-)
-
-case class Rating(
- user: String,
- item: String,
- rating: Double
-)
-
-class TrainingData(
- val users: EntityMap[User],
- val items: EntityMap[Item],
- val ratings: RDD[Rating]
-) extends Serializable {
- override def toString = {
- s"users: [${users.size} (${users.take(2).toString}...)]" +
- s"items: [${items.size} (${items.take(2).toString}...)]" +
- s"ratings: [${ratings.count()}] (${ratings.take(2).toList}...)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Engine.scala
deleted file mode 100644
index 1446ca4..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-
-case class Query(
- user: String,
- num: Int
-)
-
-case class PredictedResult(
- itemScores: Array[ItemScore]
-)
-
-case class ItemScore(
- item: String,
- score: Double
-)
-
-object RecommendationEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- classOf[Preparator],
- Map("als" -> classOf[ALSAlgorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Preparator.scala
deleted file mode 100644
index 968b526..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PPreparator
-import org.apache.predictionio.data.storage.EntityMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(
- users = trainingData.users,
- items = trainingData.items,
- ratings = trainingData.ratings)
- }
-}
-
-class PreparedData(
- val users: EntityMap[User],
- val items: EntityMap[Item],
- val ratings: RDD[Rating]
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Serving.scala
deleted file mode 100644
index 02eb0ec..0000000
--- a/examples/experimental/scala-parallel-recommendation-entitymap/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override
- def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/.gitignore b/examples/experimental/scala-parallel-recommendation-mongo-datasource/.gitignore
deleted file mode 100644
index ea4e89d..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-data/sample_movielens_data.txt
-manifest.json
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/build.sbt b/examples/experimental/scala-parallel-recommendation-mongo-datasource/build.sbt
deleted file mode 100644
index c2b4242..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/build.sbt
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-recommendation"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
-
-// ADDED FOR READING FROM MONGO IN DATASOURCE
-libraryDependencies ++= Seq(
- "org.mongodb" % "mongo-hadoop-core" % "1.3.0"
- exclude("org.apache.hadoop", "hadoop-yarn-api")
- exclude("org.apache.hadoop", "hadoop-common"))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/insert_sample_ratings_mongo.js
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/insert_sample_ratings_mongo.js b/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/insert_sample_ratings_mongo.js
deleted file mode 100644
index 39edb3a..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/insert_sample_ratings_mongo.js
+++ /dev/null
@@ -1,24 +0,0 @@
-// MongoDB script to insert sample random ratings data into MongoDB
-db = connect("localhost:27017/test");
-
-print("Remove old data in test.sample_ratings collection...")
-db.sample_ratings.remove();
-
-// min <= x < max
-function getRandomInt(min, max) {
- return Math.floor(Math.random() * (max - min)) + min;
-}
-
-print("Insert random movie rating data into test.sample_ratings collection...")
-// for eah user 0 to 10, randomly view 10 items between 0 to 49
-for (var uid = 0; uid < 10; uid++) {
- for (var n = 0; n < 10; n++) {
- db.sample_ratings.insert( {
- "uid" : uid.toString(),
- "iid" : getRandomInt(0, 50).toString(), // 0 <= iid < 50
- "rating" : getRandomInt(1, 6) // 1 <= rating < 6
- })
- }
-}
-
-print("done.")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/send_query.py
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/send_query.py b/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/send_query.py
deleted file mode 100644
index ca19dc5..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/data/send_query.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Send sample query to prediction engine
-"""
-
-import predictionio
-engine_client = predictionio.EngineClient(url="http://localhost:8000")
-print engine_client.send_query({"user": "1", "num": 4})
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/engine.json b/examples/experimental/scala-parallel-recommendation-mongo-datasource/engine.json
deleted file mode 100644
index a60eefa..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/engine.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.template.recommendation.RecommendationEngine",
- "datasource": {
- "params": {
- "host": "127.0.0.1",
- "port": 27017,
- "db": "test",
- "collection": "sample_ratings"
- }
- },
- "algorithms": [
- {
- "name": "als",
- "params": {
- "rank": 10,
- "numIterations": 20,
- "lambda": 0.01
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/project/assembly.sbt b/examples/experimental/scala-parallel-recommendation-mongo-datasource/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSAlgorithm.scala b/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSAlgorithm.scala
deleted file mode 100644
index fd93407..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSAlgorithm.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PAlgorithm
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.BiMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.mllib.recommendation.ALS
-import org.apache.spark.mllib.recommendation.{Rating => MLlibRating}
-import org.apache.spark.mllib.recommendation.ALSModel
-
-import grizzled.slf4j.Logger
-
-case class ALSAlgorithmParams(
- rank: Int,
- numIterations: Int,
- lambda: Double) extends Params
-
-class ALSAlgorithm(val ap: ALSAlgorithmParams)
- extends PAlgorithm[PreparedData, ALSModel, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: PreparedData): ALSModel = {
- // Convert user and item String IDs to Int index for MLlib
- val userStringIntMap = BiMap.stringInt(data.ratings.map(_.user))
- val itemStringIntMap = BiMap.stringInt(data.ratings.map(_.item))
- val mllibRatings = data.ratings.map( r =>
- // MLlibRating requires integer index for user and item
- MLlibRating(userStringIntMap(r.user), itemStringIntMap(r.item), r.rating)
- )
- val m = ALS.train(mllibRatings, ap.rank, ap.numIterations, ap.lambda)
- new ALSModel(
- rank = m.rank,
- userFeatures = m.userFeatures,
- productFeatures = m.productFeatures,
- userStringIntMap = userStringIntMap,
- itemStringIntMap = itemStringIntMap)
- }
-
- def predict(model: ALSModel, query: Query): PredictedResult = {
- // Convert String ID to Int index for Mllib
- model.userStringIntMap.get(query.user).map { userInt =>
- // create inverse view of itemStringIntMap
- val itemIntStringMap = model.itemStringIntMap.inverse
- // recommendProducts() returns Array[MLlibRating], which uses item Int
- // index. Convert it to String ID for returning PredictedResult
- val itemScores = model.recommendProducts(userInt, query.num)
- .map (r => ItemScore(itemIntStringMap(r.product), r.rating))
- new PredictedResult(itemScores)
- }.getOrElse{
- logger.info(s"No prediction for unknown user ${query.user}.")
- new PredictedResult(Array.empty)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSModel.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSModel.scala b/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSModel.scala
deleted file mode 100644
index 4697732..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/ALSModel.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.mllib.recommendation
-// This must be the same package as Spark's MatrixFactorizationModel because
-// MatrixFactorizationModel's constructor is private and we are using
-// its constructor in order to save and load the model
-
-import org.template.recommendation.ALSAlgorithmParams
-
-import org.apache.predictionio.controller.IPersistentModel
-import org.apache.predictionio.controller.IPersistentModelLoader
-import org.apache.predictionio.data.storage.BiMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class ALSModel(
- override val rank: Int,
- override val userFeatures: RDD[(Int, Array[Double])],
- override val productFeatures: RDD[(Int, Array[Double])],
- val userStringIntMap: BiMap[String, Int],
- val itemStringIntMap: BiMap[String, Int])
- extends MatrixFactorizationModel(rank, userFeatures, productFeatures)
- with IPersistentModel[ALSAlgorithmParams] {
-
- def save(id: String, params: ALSAlgorithmParams,
- sc: SparkContext): Boolean = {
-
- sc.parallelize(Seq(rank)).saveAsObjectFile(s"/tmp/${id}/rank")
- userFeatures.saveAsObjectFile(s"/tmp/${id}/userFeatures")
- productFeatures.saveAsObjectFile(s"/tmp/${id}/productFeatures")
- sc.parallelize(Seq(userStringIntMap))
- .saveAsObjectFile(s"/tmp/${id}/userStringIntMap")
- sc.parallelize(Seq(itemStringIntMap))
- .saveAsObjectFile(s"/tmp/${id}/itemStringIntMap")
- true
- }
-
- override def toString = {
- s"userFeatures: [${userFeatures.count()}]" +
- s"(${userFeatures.take(2).toList}...)" +
- s" productFeatures: [${productFeatures.count()}]" +
- s"(${productFeatures.take(2).toList}...)" +
- s" userStringIntMap: [${userStringIntMap.size}]" +
- s"(${userStringIntMap.take(2)}...)" +
- s" itemStringIntMap: [${itemStringIntMap.size}]" +
- s"(${itemStringIntMap.take(2)}...)"
- }
-}
-
-object ALSModel
- extends IPersistentModelLoader[ALSAlgorithmParams, ALSModel] {
- def apply(id: String, params: ALSAlgorithmParams,
- sc: Option[SparkContext]) = {
- new ALSModel(
- rank = sc.get.objectFile[Int](s"/tmp/${id}/rank").first,
- userFeatures = sc.get.objectFile(s"/tmp/${id}/userFeatures"),
- productFeatures = sc.get.objectFile(s"/tmp/${id}/productFeatures"),
- userStringIntMap = sc.get
- .objectFile[BiMap[String, Int]](s"/tmp/${id}/userStringIntMap").first,
- itemStringIntMap = sc.get
- .objectFile[BiMap[String, Int]](s"/tmp/${id}/itemStringIntMap").first)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/DataSource.scala
deleted file mode 100644
index a891cc5..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-import org.apache.hadoop.conf.Configuration // ADDED
-import org.bson.BSONObject // ADDED
-import com.mongodb.hadoop.MongoInputFormat // ADDED
-
-case class DataSourceParams( // CHANGED
- host: String,
- port: Int,
- db: String, // DB name
- collection: String // collection name
-) extends Params
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- // CHANGED
- val config = new Configuration()
- config.set("mongo.input.uri",
- s"mongodb://${dsp.host}:${dsp.port}/${dsp.db}.${dsp.collection}")
-
- val mongoRDD = sc.newAPIHadoopRDD(config,
- classOf[MongoInputFormat],
- classOf[Object],
- classOf[BSONObject])
-
- // mongoRDD contains tuples of (ObjectId, BSONObject)
- val ratings = mongoRDD.map { case (id, bson) =>
- Rating(bson.get("uid").asInstanceOf[String],
- bson.get("iid").asInstanceOf[String],
- bson.get("rating").asInstanceOf[Double])
- }
- new TrainingData(ratings)
- }
-}
-
-case class Rating(
- user: String,
- item: String,
- rating: Double
-)
-
-class TrainingData(
- val ratings: RDD[Rating]
-) extends Serializable {
- override def toString = {
- s"ratings: [${ratings.count()}] (${ratings.take(2).toList}...)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Engine.scala
deleted file mode 100644
index 1446ca4..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-
-case class Query(
- user: String,
- num: Int
-)
-
-case class PredictedResult(
- itemScores: Array[ItemScore]
-)
-
-case class ItemScore(
- item: String,
- score: Double
-)
-
-object RecommendationEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- classOf[Preparator],
- Map("als" -> classOf[ALSAlgorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Preparator.scala
deleted file mode 100644
index 0bab35b..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.PPreparator
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(ratings = trainingData.ratings)
- }
-}
-
-class PreparedData(
- val ratings: RDD[Rating]
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Serving.scala
deleted file mode 100644
index 48fda35..0000000
--- a/examples/experimental/scala-parallel-recommendation-mongo-datasource/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.recommendation
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-regression/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-regression/README.md b/examples/experimental/scala-parallel-regression/README.md
deleted file mode 100644
index cfb8c60..0000000
--- a/examples/experimental/scala-parallel-regression/README.md
+++ /dev/null
@@ -1,56 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Parallel Regression Engine
-
-## Configuration
-
-This sample regression engine reads data from file system.
-
-Edit the file path in `engine.json`, change `filepath` of `datasource` to an absolute path that points to
-[lr_data.py](../data/lr_data.txt)
-
-```
-$ cat engine.json
-...
-"datasource": {
- "filepath": <absolute_path_to_lr_data.txt>,
- "k": 3,
- "seed": 9527
-}
-...
-
-```
-
-## Register engine, train, and deploy.
-
-```
-$ pio build
-$ pio train
-$ pio deploy --port 9998
-```
-
-## Query the Engine Instance
-
-```
-$ curl -X POST http://localhost:9998/queries.json -d \
- '[1.80,0.87,2.41,0.35,-0.21,1.35,0.51,1.55,-0.20,1.32]'
-
-0.8912731719174509
-```
-
-0.89... is the prediction result.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-regression/Run.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-regression/Run.scala b/examples/experimental/scala-parallel-regression/Run.scala
deleted file mode 100644
index 232e61f..0000000
--- a/examples/experimental/scala-parallel-regression/Run.scala
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.regression.parallel
-
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.P2LAlgorithm
-import org.apache.predictionio.controller.IdentityPreparator
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.LAverageServing
-import org.apache.predictionio.controller.MeanSquareError
-import org.apache.predictionio.controller.Utils
-import org.apache.predictionio.controller.Workflow
-import org.apache.predictionio.controller.WorkflowParams
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.mllib.linalg.DenseVector
-import org.apache.spark.mllib.linalg.Vector
-import org.apache.spark.mllib.linalg.Vectors
-import org.apache.spark.mllib.regression.LabeledPoint
-import org.apache.spark.mllib.regression.LinearRegressionWithSGD
-import org.apache.spark.mllib.regression.RegressionModel
-import org.apache.spark.mllib.util.MLUtils
-import org.apache.spark.rdd.RDD
-import org.json4s._
-import java.io.File
-
-case class DataSourceParams(
- val filepath: String, val k: Int = 3, val seed: Int = 9527)
-extends Params
-
-case class ParallelDataSource(val dsp: DataSourceParams)
- extends PDataSource[
- DataSourceParams, Integer,
- RDD[LabeledPoint], Vector, Double] {
- override
- def read(sc: SparkContext)
- : Seq[(Integer, RDD[LabeledPoint], RDD[(Vector, Double)])] = {
- val input = sc.textFile(dsp.filepath)
- val points = input.map { line =>
- val parts = line.split(' ').map(_.toDouble)
- LabeledPoint(parts(0), Vectors.dense(parts.drop(1)))
- }
-
- MLUtils.kFold(points, dsp.k, dsp.seed)
- .zipWithIndex
- .map { case (dataSet, index) =>
- (Int.box(index), dataSet._1, dataSet._2.map(p => (p.features, p.label)))
- }
- }
-}
-
-case class AlgorithmParams(
- val numIterations: Int = 200, val stepSize: Double = 0.1) extends Params
-
-case class ParallelSGDAlgorithm(val ap: AlgorithmParams)
- extends P2LAlgorithm[
- AlgorithmParams, RDD[LabeledPoint], RegressionModel, Vector, Double] {
-
- def train(data: RDD[LabeledPoint]): RegressionModel = {
- LinearRegressionWithSGD.train(data, ap.numIterations, ap.stepSize)
- }
-
- def predict(model: RegressionModel, feature: Vector): Double = {
- model.predict(feature)
- }
-
- @transient override lazy val querySerializer =
- Utils.json4sDefaultFormats + new VectorSerializer
-}
-
-object RegressionEngineFactory extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[ParallelDataSource],
- classOf[IdentityPreparator[RDD[LabeledPoint]]],
- Map("SGD" -> classOf[ParallelSGDAlgorithm]),
- LAverageServing(classOf[ParallelSGDAlgorithm]))
- }
-}
-
-object Run {
- def main(args: Array[String]) {
- val filepath = new File("../data/lr_data.txt").getCanonicalPath
- val dataSourceParams = DataSourceParams(filepath, 3)
- val SGD = "SGD"
- val algorithmParamsList = Seq(
- (SGD, AlgorithmParams(stepSize = 0.1)),
- (SGD, AlgorithmParams(stepSize = 0.2)),
- (SGD, AlgorithmParams(stepSize = 0.4)))
-
- Workflow.run(
- dataSourceClassOpt = Some(classOf[ParallelDataSource]),
- dataSourceParams = dataSourceParams,
- preparatorClassOpt =
- Some(classOf[IdentityPreparator[RDD[LabeledPoint]]]),
- algorithmClassMapOpt = Some(Map(SGD -> classOf[ParallelSGDAlgorithm])),
- algorithmParamsList = algorithmParamsList,
- servingClassOpt = Some(LAverageServing(classOf[ParallelSGDAlgorithm])),
- evaluatorClassOpt = Some(classOf[MeanSquareError]),
- params = WorkflowParams(
- batch = "Imagine: Parallel Regression"))
- }
-}
-
-class VectorSerializer extends CustomSerializer[Vector](format => (
- {
- case JArray(x) =>
- val v = x.toArray.map { y =>
- y match {
- case JDouble(z) => z
- }
- }
- new DenseVector(v)
- },
- {
- case x: Vector =>
- JArray(x.toArray.toList.map(d => JDouble(d)))
- }
-))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-regression/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-regression/build.sbt b/examples/experimental/scala-parallel-regression/build.sbt
deleted file mode 100644
index e053ba8..0000000
--- a/examples/experimental/scala-parallel-regression/build.sbt
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "scala-parallel-regression"
-
-organization := "myorg"
-
-version := "0.0.1-SNAPSHOT"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0"
- exclude("org.apache.spark", "spark-core_2.10")
- exclude("org.eclipse.jetty", "jetty-server"))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-regression/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-regression/engine.json b/examples/experimental/scala-parallel-regression/engine.json
deleted file mode 100644
index e078fad..0000000
--- a/examples/experimental/scala-parallel-regression/engine.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "id": "scala-parallel-regression",
- "description": "scala-parallel-regression example",
- "engineFactory": "org.apache.predictionio.examples.regression.parallel.RegressionEngineFactory",
- "datasource": {
- "params": {
- "filepath": "../data/lr_data.txt",
- "k": 3,
- "seed": 9527
- }
- },
- "algorithms": [
- {
- "name": "SGD",
- "params": {
- "numIterations": 800,
- "stepSize": 0.1
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-regression/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-regression/project/assembly.sbt b/examples/experimental/scala-parallel-regression/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-regression/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/.gitignore b/examples/experimental/scala-parallel-similarproduct-dimsum/.gitignore
deleted file mode 100644
index ea4e89d..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-data/sample_movielens_data.txt
-manifest.json
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/build.sbt b/examples/experimental/scala-parallel-similarproduct-dimsum/build.sbt
deleted file mode 100644
index 5ef8f87..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-similarproduct-dimsum"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/engine.json b/examples/experimental/scala-parallel-similarproduct-dimsum/engine.json
deleted file mode 100644
index 312501f..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/engine.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.template.similarproduct.SimilarProductEngine",
- "datasource": {
- "params" : {
- "appId": 9
- }
- },
- "algorithms": [
- {
- "name": "dimsum",
- "params": {
- "threshold": 1
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/project/assembly.sbt b/examples/experimental/scala-parallel-similarproduct-dimsum/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DIMSUMAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DIMSUMAlgorithm.scala b/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DIMSUMAlgorithm.scala
deleted file mode 100644
index 73e15b8..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DIMSUMAlgorithm.scala
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.PAlgorithm
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.IPersistentModel
-import org.apache.predictionio.controller.IPersistentModelLoader
-import org.apache.predictionio.data.storage.BiMap
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.mllib.linalg.Vectors
-import org.apache.spark.mllib.linalg.Vector
-import org.apache.spark.mllib.linalg.SparseVector
-import org.apache.spark.mllib.linalg.distributed.MatrixEntry
-import org.apache.spark.mllib.linalg.distributed.RowMatrix
-import org.apache.spark.mllib.linalg.distributed.CoordinateMatrix
-
-import grizzled.slf4j.Logger
-
-import scala.collection.mutable.PriorityQueue
-
-case class DIMSUMAlgorithmParams(threshold: Double) extends Params
-
-class DIMSUMModel(
- val similarities: RDD[(Int, SparseVector)],
- val itemStringIntMap: BiMap[String, Int],
- val items: Map[Int, Item]
- ) extends IPersistentModel[DIMSUMAlgorithmParams] {
-
- @transient lazy val itemIntStringMap = itemStringIntMap.inverse
-
- def save(id: String, params: DIMSUMAlgorithmParams,
- sc: SparkContext): Boolean = {
-
- similarities.saveAsObjectFile(s"/tmp/${id}/similarities")
- sc.parallelize(Seq(itemStringIntMap))
- .saveAsObjectFile(s"/tmp/${id}/itemStringIntMap")
- sc.parallelize(Seq(items))
- .saveAsObjectFile(s"/tmp/${id}/items")
- true
- }
-
- override def toString = {
- s"similarities: [${similarities.count()}]" +
- s"(${similarities.take(2).toList}...)" +
- s" itemStringIntMap: [${itemStringIntMap.size}]" +
- s"(${itemStringIntMap.take(2).toString}...)]" +
- s" items: [${items.size}]" +
- s"(${items.take(2).toString}...)]"
- }
-}
-
-object DIMSUMModel
- extends IPersistentModelLoader[DIMSUMAlgorithmParams, DIMSUMModel] {
- def apply(id: String, params: DIMSUMAlgorithmParams,
- sc: Option[SparkContext]) = {
- new DIMSUMModel(
- similarities = sc.get.objectFile(s"/tmp/${id}/similarities"),
- itemStringIntMap = sc.get
- .objectFile[BiMap[String, Int]](s"/tmp/${id}/itemStringIntMap").first,
- items = sc.get
- .objectFile[Map[Int, Item]](s"/tmp/${id}/items").first)
- }
-}
-
-class DIMSUMAlgorithm(val ap: DIMSUMAlgorithmParams)
- extends PAlgorithm[PreparedData, DIMSUMModel, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(data: PreparedData): DIMSUMModel = {
-
- // create User and item's String ID to integer index BiMap
- val userStringIntMap = BiMap.stringInt(data.users.keys)
- val itemStringIntMap = BiMap.stringInt(data.items.keys)
-
- // collect Item as Map and convert ID to Int index
- val items: Map[Int, Item] = data.items.map { case (id, item) =>
- (itemStringIntMap(id), item)
- }.collectAsMap.toMap
- val itemCount = items.size
-
- // each row is a sparse vector of rated items by this user
- val rows: RDD[Vector] = data.viewEvents
- .map { r =>
- // Convert user and item String IDs to Int index for MLlib
- val uindex = userStringIntMap.getOrElse(r.user, -1)
- val iindex = itemStringIntMap.getOrElse(r.item, -1)
-
- if (uindex == -1)
- logger.info(s"Couldn't convert nonexistent user ID ${r.user}"
- + " to Int index.")
-
- if (iindex == -1)
- logger.info(s"Couldn't convert nonexistent item ID ${r.item}"
- + " to Int index.")
-
- (uindex, (iindex, 1.0))
- }.filter { case (uindex, (iindex, v)) =>
- // keep events with valid user and item index
- (uindex != -1) && (iindex != -1)
- }.groupByKey().map { case (u, ir) =>
- // de-duplicate if user has multiple events on same item
- val irDedup: Map[Int, Double] = ir.groupBy(_._1) // group By item index
- .map { case (i, irGroup) =>
- // same item index group of (item index, rating value) tuple
- val r = irGroup.reduce { (a, b) =>
- // Simply keep one copy.
- a
- // You may modify here to reduce same item tuple differently,
- // such as summing all values:
- //(a._1, (a._2 + b._2))
- }
- (i, r._2)
- }
-
- // NOTE: index array must be strictly increasing for Sparse Vector
- val irSorted = irDedup.toArray.sortBy(_._1)
- val indexes = irSorted.map(_._1)
- val values = irSorted.map(_._2)
- Vectors.sparse(itemCount, indexes, values)
- }
-
- val mat = new RowMatrix(rows)
- val scores = mat.columnSimilarities(ap.threshold)
- val reversedEntries: RDD[MatrixEntry] = scores.entries
- .map(e => new MatrixEntry(e.j, e.i, e.value))
- val combined = new CoordinateMatrix(scores.entries.union(reversedEntries))
- val similarities = combined.toIndexedRowMatrix.rows
- .map( row => (row.index.toInt, row.vector.asInstanceOf[SparseVector]))
-
- new DIMSUMModel(
- similarities = similarities,
- itemStringIntMap = itemStringIntMap,
- items = items
- )
- }
-
- def predict(model: DIMSUMModel, query: Query): PredictedResult = {
- // convert the white and black list items to Int index
- val whiteList: Option[Set[Int]] = query.whiteList.map( set =>
- set.map(model.itemStringIntMap.get(_)).flatten
- )
- val blackList: Option[Set[Int]] = query.blackList.map ( set =>
- set.map(model.itemStringIntMap.get(_)).flatten
- )
-
- val queryList: Set[Int] = query.items.map(model.itemStringIntMap.get(_))
- .flatten.toSet
-
- val indexScores = query.items.flatMap { iid =>
- model.itemStringIntMap.get(iid).map { itemInt =>
- val simsSeq = model.similarities.lookup(itemInt)
- if (simsSeq.isEmpty) {
- logger.info(s"No similar items found for ${iid}.")
- Array.empty[(Int, Double)]
- } else {
- val sims = simsSeq.head
- sims.indices.zip(sims.values).filter { case (i, v) =>
- whiteList.map(_.contains(i)).getOrElse(true) &&
- blackList.map(!_.contains(i)).getOrElse(true) &&
- // discard items in query as well
- (!queryList.contains(i)) &&
- // filter categories
- query.categories.map { cat =>
- model.items(i).categories.map { itemCat =>
- // keep this item if has ovelap categories with the query
- !(itemCat.toSet.intersect(cat).isEmpty)
- }.getOrElse(false) // discard this item if it has no categories
- }.getOrElse(true)
- }
- }
- }.getOrElse {
- logger.info(s"No similar items for unknown item ${iid}.")
- Array.empty[(Int, Double)]
- }
- }
-
- val aggregatedScores = indexScores.groupBy(_._1)
- .mapValues(_.foldLeft[Double](0)( (b,a) => b + a._2))
- .toList
-
- val ord = Ordering.by[(Int, Double), Double](_._2).reverse
- val itemScores = getTopN(aggregatedScores, query.num)(ord)
- .map{ case (i, s) =>
- new ItemScore(
- item = model.itemIntStringMap(i),
- score = s
- )
- }.toArray
-
- new PredictedResult(itemScores)
- }
-
- private
- def getTopN[T](s: Seq[T], n: Int)(implicit ord: Ordering[T]): Seq[T] = {
-
- val q = PriorityQueue()
-
- for (x <- s) {
- if (q.size < n)
- q.enqueue(x)
- else {
- // q is full
- if (ord.compare(x, q.head) < 0) {
- q.dequeue()
- q.enqueue(x)
- }
- }
- }
-
- q.dequeueAll.toSeq.reverse
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DataSource.scala b/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DataSource.scala
deleted file mode 100644
index 23d1d2b..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-case class DataSourceParams(appId: Int) extends Params
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- val eventsDb = Storage.getPEvents()
-
- // create a RDD of (entityID, User)
- val usersRDD: RDD[(String, User)] = eventsDb.aggregateProperties(
- appId = dsp.appId,
- entityType = "user"
- )(sc).map { case (entityId, properties) =>
- val user = try {
- User()
- } catch {
- case e: Exception => {
- logger.error(s"Failed to get properties ${properties} of" +
- s" user ${entityId}. Exception: ${e}.")
- throw e
- }
- }
- (entityId, user)
- }
-
- // create a RDD of (entityID, Item)
- val itemsRDD: RDD[(String, Item)] = eventsDb.aggregateProperties(
- appId = dsp.appId,
- entityType = "item"
- )(sc).map { case (entityId, properties) =>
- val item = try {
- // Assume categories is optional property of item.
- Item(categories = properties.getOpt[List[String]]("categories"))
- } catch {
- case e: Exception => {
- logger.error(s"Failed to get properties ${properties} of" +
- s" item ${entityId}. Exception: ${e}.")
- throw e
- }
- }
- (entityId, item)
- }
-
- // get all "user" "view" "item" events
- val eventsRDD: RDD[Event] = eventsDb.find(
- appId = dsp.appId,
- entityType = Some("user"),
- eventNames = Some(List("view")),
- // targetEntityType is optional field of an event.
- targetEntityType = Some(Some("item")))(sc)
-
- val viewEventsRDD: RDD[ViewEvent] = eventsRDD.map { event =>
- val viewEvent = try {
- event.event match {
- case "view" => ViewEvent(
- user = event.entityId,
- item = event.targetEntityId.get,
- t = event.eventTime.getMillis)
- case _ => throw new Exception(s"Unexpected event ${event} is read.")
- }
- } catch {
- case e: Exception => {
- logger.error(s"Cannot convert ${event} to U2IEvent. Exception: ${e}.")
- throw e
- }
- }
- viewEvent
- }
-
- new TrainingData(
- users = usersRDD,
- items = itemsRDD,
- viewEvents = viewEventsRDD
- )
- }
-}
-
-case class User()
-
-case class Item(categories: Option[List[String]])
-
-case class ViewEvent(
- user: String,
- item: String,
- t: Long
-)
-
-class TrainingData(
- val users: RDD[(String, User)],
- val items: RDD[(String, Item)],
- val viewEvents: RDD[ViewEvent]
-) extends Serializable {
- override def toString = {
- s"users: [${users.count()} (${users.take(2).toList}...)]" +
- s"items: [${items.count()} (${items.take(2).toList}...)]" +
- s"ratings: [${viewEvents.count()}] (${viewEvents.take(2).toList}...)"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Engine.scala b/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Engine.scala
deleted file mode 100644
index ba929ad..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-
-case class Query(
- items: List[String],
- num: Int,
- categories: Option[Set[String]],
- whiteList: Option[Set[String]],
- blackList: Option[Set[String]]
-)
-
-case class PredictedResult(
- itemScores: Array[ItemScore]
-)
-
-case class ItemScore(
- item: String,
- score: Double
-)
-
-object SimilarProductEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- classOf[Preparator],
- Map("dimsum" -> classOf[DIMSUMAlgorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Preparator.scala b/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Preparator.scala
deleted file mode 100644
index e3394b0..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.PPreparator
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(
- users = trainingData.users,
- items = trainingData.items,
- viewEvents = trainingData.viewEvents)
- }
-}
-
-class PreparedData(
- val users: RDD[(String, User)],
- val items: RDD[(String, Item)],
- val viewEvents: RDD[ViewEvent]
-) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Serving.scala b/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Serving.scala
deleted file mode 100644
index 3e115d5..0000000
--- a/examples/experimental/scala-parallel-similarproduct-dimsum/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.template.similarproduct
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-parallel-similarproduct-localmodel/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-parallel-similarproduct-localmodel/.gitignore b/examples/experimental/scala-parallel-similarproduct-localmodel/.gitignore
deleted file mode 100644
index ea4e89d..0000000
--- a/examples/experimental/scala-parallel-similarproduct-localmodel/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-data/sample_movielens_data.txt
-manifest.json
[45/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by ch...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/2d953485
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/2d953485
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/2d953485
Branch: refs/heads/master
Commit: 2d9534850cf443da9fc71e8f0fb9261430f69111
Parents: bd034f5 951d7a0
Author: Donald Szeto <do...@apache.org>
Authored: Tue Sep 12 20:39:01 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Sep 12 20:39:01 2017 -0700
----------------------------------------------------------------------
docs/manual/source/datacollection/webhooks.html.md.erb | 2 +-
docs/manual/source/gallery/templates.yaml | 12 ++++++------
.../source/install/install-sourcecode.html.md.erb | 2 +-
3 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[39/50] [abbrv] incubator-predictionio git commit: [PIO-119] Bump up
Elasticsearch to 5.5.2
Posted by ch...@apache.org.
[PIO-119] Bump up Elasticsearch to 5.5.2
Closes #430
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/a0a2c126
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/a0a2c126
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/a0a2c126
Branch: refs/heads/master
Commit: a0a2c1261187451d343d70c4251d8f1775a8b6e8
Parents: df406bf
Author: Shinsuke Sugaya <sh...@apache.org>
Authored: Mon Sep 11 13:22:43 2017 +0900
Committer: Shinsuke Sugaya <sh...@apache.org>
Committed: Mon Sep 11 13:22:43 2017 +0900
----------------------------------------------------------------------
.travis.yml | 10 +++++-----
build.sbt | 2 +-
conf/pio-env.sh.template | 2 +-
conf/pio-vendors.sh | 4 ++--
storage/elasticsearch/build.sbt | 2 +-
5 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a0a2c126/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index e208189..61cc4b8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -57,12 +57,12 @@ env:
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
PIO_SCALA_VERSION=2.10.6
PIO_SPARK_VERSION=1.6.3
- PIO_ELASTICSEARCH_VERSION=5.4.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
PIO_SCALA_VERSION=2.10.6
PIO_SPARK_VERSION=1.6.3
- PIO_ELASTICSEARCH_VERSION=5.4.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
PIO_SCALA_VERSION=2.10.6
@@ -86,12 +86,12 @@ env:
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
PIO_SCALA_VERSION=2.11.8
PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=5.4.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
PIO_SCALA_VERSION=2.11.8
PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=5.4.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
PIO_SCALA_VERSION=2.11.8
@@ -106,7 +106,7 @@ env:
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
PIO_SCALA_VERSION=2.11.8
PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=5.4.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
- BUILD_TYPE=LicenseCheck
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a0a2c126/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 139bf39..1a439c2 100644
--- a/build.sbt
+++ b/build.sbt
@@ -71,7 +71,7 @@ akkaVersion in ThisBuild := sys.props.getOrElse(
"akka.version",
scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("akka"))
-lazy val es = sys.props.getOrElse("elasticsearch.version", "5.4.1")
+lazy val es = sys.props.getOrElse("elasticsearch.version", "5.5.2")
elasticsearchVersion in ThisBuild := es
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a0a2c126/conf/pio-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/pio-env.sh.template b/conf/pio-env.sh.template
index 0b6b5b9..a96e4bd 100644
--- a/conf/pio-env.sh.template
+++ b/conf/pio-env.sh.template
@@ -89,7 +89,7 @@ PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=localhost
# PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
# PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-5.4.1
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-5.5.2
# Optional basic HTTP auth
# PIO_STORAGE_SOURCES_ELASTICSEARCH_USERNAME=my-name
# PIO_STORAGE_SOURCES_ELASTICSEARCH_PASSWORD=my-secret
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a0a2c126/conf/pio-vendors.sh
----------------------------------------------------------------------
diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh
index 2159cf5..6420207 100644
--- a/conf/pio-vendors.sh
+++ b/conf/pio-vendors.sh
@@ -32,7 +32,7 @@ if [ -z "$PIO_HADOOP_VERSION" ]; then
fi
if [ -z "$PIO_ELASTICSEARCH_VERSION" ]; then
- PIO_ELASTICSEARCH_VERSION="5.4.1"
+ PIO_ELASTICSEARCH_VERSION="5.5.2"
fi
ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'`
@@ -42,7 +42,7 @@ if [ "$ES_MAJOR" = "1" ]; then
export ES_TAG="1"
else
export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch"
- export ES_TAG="5.4.1"
+ export ES_TAG="5.5.2"
fi
PGSQL_JAR=postgresql-9.4-1204.jdbc41.jar
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a0a2c126/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index b60e86e..754aefb 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -21,7 +21,7 @@ name := "apache-predictionio-data-elasticsearch"
elasticsearchSparkArtifact := (if (majorVersion(sparkVersion.value) == 2) "elasticsearch-spark-20" else "elasticsearch-spark-13")
-elasticsearchVersion := (if (majorVersion(elasticsearchVersion.value) < 5) "5.4.1" else elasticsearchVersion.value)
+elasticsearchVersion := (if (majorVersion(elasticsearchVersion.value) < 5) "5.5.2" else elasticsearchVersion.value)
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
[02/50] [abbrv] incubator-predictionio git commit: Fix use of case
class in similarproduct examples
Posted by ch...@apache.org.
Fix use of case class in similarproduct examples
Closes #413
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/96d4c54a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/96d4c54a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/96d4c54a
Branch: refs/heads/master
Commit: 96d4c54abca4aee2a7f4c83b8465337f140aa0fd
Parents: 42d6d42
Author: Naoki Takezoe <ta...@apache.org>
Authored: Wed Jul 19 10:24:40 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Wed Jul 19 10:24:40 2017 +0900
----------------------------------------------------------------------
.../multi-events-multi-algos/src/main/scala/ALSAlgorithm.scala | 4 ++--
.../src/main/scala/CooccurrenceAlgorithm.scala | 2 +-
.../multi-events-multi-algos/src/main/scala/Engine.scala | 6 +++---
.../multi-events-multi-algos/src/main/scala/Serving.scala | 2 +-
.../recommended-user/src/main/scala/ALSAlgorithm.scala | 4 ++--
.../return-item-properties/src/main/scala/ALSAlgorithm.scala | 4 ++--
.../src/main/scala/CooccurrenceAlgorithm.scala | 2 +-
.../return-item-properties/src/main/scala/Engine.scala | 6 +++---
.../rid-user-set-event/src/main/scala/ALSAlgorithm.scala | 4 ++--
.../src/main/scala/CooccurrenceAlgorithm.scala | 2 +-
.../rid-user-set-event/src/main/scala/Engine.scala | 6 +++---
.../train-with-rate-event/src/main/scala/ALSAlgorithm.scala | 4 ++--
.../src/main/scala/CooccurrenceAlgorithm.scala | 2 +-
.../train-with-rate-event/src/main/scala/Engine.scala | 6 +++---
14 files changed, 27 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/ALSAlgorithm.scala b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/ALSAlgorithm.scala
index aae8322..64d570c 100644
--- a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/ALSAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/ALSAlgorithm.scala
@@ -185,13 +185,13 @@ class ALSAlgorithm(val ap: ALSAlgorithmParams)
val topScores = getTopN(filteredScore, query.num)(ord).toArray
val itemScores = topScores.map { case (i, s) =>
- new ItemScore(
+ ItemScore(
item = model.itemIntStringMap(i),
score = s
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
private
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/CooccurrenceAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/CooccurrenceAlgorithm.scala b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/CooccurrenceAlgorithm.scala
index 30d0b3e..76307e7 100644
--- a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/CooccurrenceAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/CooccurrenceAlgorithm.scala
@@ -146,7 +146,7 @@ class CooccurrenceAlgorithm(val ap: CooccurrenceAlgorithmParams)
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Engine.scala b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Engine.scala
index dcfee6f..8691e71 100644
--- a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Engine.scala
+++ b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Engine.scala
@@ -27,18 +27,18 @@ case class Query(
categoryBlackList: Option[Set[String]],
whiteList: Option[Set[String]],
blackList: Option[Set[String]]
-) extends Serializable
+)
case class PredictedResult(
itemScores: Array[ItemScore]
-) extends Serializable {
+){
override def toString: String = itemScores.mkString(",")
}
case class ItemScore(
item: String,
score: Double
-) extends Serializable
+)
object SimilarProductEngine extends EngineFactory {
def apply() = {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Serving.scala b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Serving.scala
index 79b33a2..831dea2 100644
--- a/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Serving.scala
+++ b/examples/scala-parallel-similarproduct/multi-events-multi-algos/src/main/scala/Serving.scala
@@ -65,6 +65,6 @@ class Serving
.take(query.num)
.map { case (k,v) => ItemScore(k, v) }
- new PredictedResult(combined)
+ PredictedResult(combined)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/recommended-user/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/recommended-user/src/main/scala/ALSAlgorithm.scala b/examples/scala-parallel-similarproduct/recommended-user/src/main/scala/ALSAlgorithm.scala
index 8e93824..fd84284 100644
--- a/examples/scala-parallel-similarproduct/recommended-user/src/main/scala/ALSAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/recommended-user/src/main/scala/ALSAlgorithm.scala
@@ -175,13 +175,13 @@ class ALSAlgorithm(val ap: ALSAlgorithmParams)
val topScores = getTopN(filteredScore, query.num)(ord).toArray
val similarUserScores = topScores.map { case (i, s) =>
- new SimilarUserScore(
+ SimilarUserScore(
user = model.similarUserIntStringMap(i),
score = s
)
}
- new PredictedResult(similarUserScores)
+ PredictedResult(similarUserScores)
}
private
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/ALSAlgorithm.scala b/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/ALSAlgorithm.scala
index fabb098..3bf3402 100644
--- a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/ALSAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/ALSAlgorithm.scala
@@ -187,7 +187,7 @@ class ALSAlgorithm(val ap: ALSAlgorithmParams)
val itemScores = topScores.map { case (i, s) =>
// MODIFIED
val it = model.items(i)
- new ItemScore(
+ ItemScore(
item = model.itemIntStringMap(i),
title = it.title,
date = it.date,
@@ -196,7 +196,7 @@ class ALSAlgorithm(val ap: ALSAlgorithmParams)
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
private
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/CooccurrenceAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/CooccurrenceAlgorithm.scala b/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/CooccurrenceAlgorithm.scala
index 7a75bf0..470d87d 100644
--- a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/CooccurrenceAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/CooccurrenceAlgorithm.scala
@@ -151,7 +151,7 @@ class CooccurrenceAlgorithm(val ap: CooccurrenceAlgorithmParams)
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/Engine.scala b/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/Engine.scala
index 595383e..bd556f0 100644
--- a/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/Engine.scala
+++ b/examples/scala-parallel-similarproduct/return-item-properties/src/main/scala/Engine.scala
@@ -27,11 +27,11 @@ case class Query(
categoryBlackList: Option[Set[String]],
whiteList: Option[Set[String]],
blackList: Option[Set[String]]
-) extends Serializable
+)
case class PredictedResult(
itemScores: Array[ItemScore]
-) extends Serializable {
+){
override def toString: String = itemScores.mkString(",")
}
@@ -42,7 +42,7 @@ case class ItemScore(
date: String,
imdbUrl: String,
score: Double
-) extends Serializable
+)
object SimilarProductEngine extends EngineFactory {
def apply() = {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/ALSAlgorithm.scala b/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/ALSAlgorithm.scala
index b413c08..50c26b5 100644
--- a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/ALSAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/ALSAlgorithm.scala
@@ -181,13 +181,13 @@ class ALSAlgorithm(val ap: ALSAlgorithmParams)
val topScores = getTopN(filteredScore, query.num)(ord).toArray
val itemScores = topScores.map { case (i, s) =>
- new ItemScore(
+ ItemScore(
item = model.itemIntStringMap(i),
score = s
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
private
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/CooccurrenceAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/CooccurrenceAlgorithm.scala b/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/CooccurrenceAlgorithm.scala
index 30d0b3e..76307e7 100644
--- a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/CooccurrenceAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/CooccurrenceAlgorithm.scala
@@ -146,7 +146,7 @@ class CooccurrenceAlgorithm(val ap: CooccurrenceAlgorithmParams)
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/Engine.scala b/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/Engine.scala
index 2563fdf..1c81b32 100644
--- a/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/Engine.scala
+++ b/examples/scala-parallel-similarproduct/rid-user-set-event/src/main/scala/Engine.scala
@@ -27,18 +27,18 @@ case class Query(
categoryBlackList: Option[Set[String]],
whiteList: Option[Set[String]],
blackList: Option[Set[String]]
-) extends Serializable
+)
case class PredictedResult(
itemScores: Array[ItemScore]
-) extends Serializable {
+){
override def toString: String = itemScores.mkString(",")
}
case class ItemScore(
item: String,
score: Double
-) extends Serializable
+)
object SimilarProductEngine extends EngineFactory {
def apply() = {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/ALSAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/ALSAlgorithm.scala b/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/ALSAlgorithm.scala
index eaefe17..507343e 100644
--- a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/ALSAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/ALSAlgorithm.scala
@@ -193,13 +193,13 @@ class ALSAlgorithm(val ap: ALSAlgorithmParams)
val topScores = getTopN(filteredScore, query.num)(ord).toArray
val itemScores = topScores.map { case (i, s) =>
- new ItemScore(
+ ItemScore(
item = model.itemIntStringMap(i),
score = s
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
private
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/CooccurrenceAlgorithm.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/CooccurrenceAlgorithm.scala b/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/CooccurrenceAlgorithm.scala
index b5035f8..0edc76e 100644
--- a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/CooccurrenceAlgorithm.scala
+++ b/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/CooccurrenceAlgorithm.scala
@@ -147,7 +147,7 @@ class CooccurrenceAlgorithm(val ap: CooccurrenceAlgorithmParams)
)
}
- new PredictedResult(itemScores)
+ PredictedResult(itemScores)
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/96d4c54a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/Engine.scala b/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/Engine.scala
index 2563fdf..1c81b32 100644
--- a/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/Engine.scala
+++ b/examples/scala-parallel-similarproduct/train-with-rate-event/src/main/scala/Engine.scala
@@ -27,18 +27,18 @@ case class Query(
categoryBlackList: Option[Set[String]],
whiteList: Option[Set[String]],
blackList: Option[Set[String]]
-) extends Serializable
+)
case class PredictedResult(
itemScores: Array[ItemScore]
-) extends Serializable {
+){
override def toString: String = itemScores.mkString(",")
}
case class ItemScore(
item: String,
score: Double
-) extends Serializable
+)
object SimilarProductEngine extends EngineFactory {
def apply() = {
[48/50] [abbrv] incubator-predictionio git commit: Update RELEASE.md
Posted by ch...@apache.org.
Update RELEASE.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d2a6aa99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d2a6aa99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d2a6aa99
Branch: refs/heads/master
Commit: d2a6aa99e9bd0d70605b0a60492ec863bf60365f
Parents: b9a9e5e
Author: Chan Lee <ch...@gmail.com>
Authored: Thu Sep 14 13:36:13 2017 -0700
Committer: Chan Lee <ch...@gmail.com>
Committed: Thu Sep 14 13:36:13 2017 -0700
----------------------------------------------------------------------
RELEASE.md | 50 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 49 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d2a6aa99/RELEASE.md
----------------------------------------------------------------------
diff --git a/RELEASE.md b/RELEASE.md
index db3549b..b506981 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -21,9 +21,57 @@ limitations under the License.
## Version History
+### 0.12.0
+
+Sep ?, 2017
+
+#### New Features
+
+- PIO-61: S3 support for model data
+- PIO-69, PIO-91: Binary distribution of PredictionIO
+- PIO-105, PIO-110, PIO-111: Batch predictions
+- PIO-95: Configurable request timeout for REST API
+- PIO-114: Basic HTTP authentication for Elasticsearch 5.x StorageClient
+- PIO-116: PySpark support
+
+#### Behavior Changes
+
+- PIO-59: `pio app new` uses /dev/urandom/ to generate entropy.
+- PIO-72: `pio-shell` properly loads storage dependencies.
+- PIO-83, PIO-119: Default environment changed to Spark 2.1.1, Scala 2.11.8,
+ and Elasticsearch 5.5.2.
+- PIO-99: `pio-build` checks for compilation errors before proceeding
+ to build engine.
+- PIO-100: `pio` commands no longer display SLF4J warning messages.
+
+#### Other Changes
+
+- PIO-56: Core unit tests no longer require meta data setup.
+- PIO-60, PIO-62: Minor fixes in authorship information and license checking.
+- PIO-63: Apache incubator logo and disclaimer is displayed on the website.
+- PIO-65: Integration tests on Travis caches downloaded jars.
+- PIO-66: More detailed documentation regarding release process and adding
+ JIRA tickets.
+- PIO-90: Improved performance for /batch/events.json API call.
+- PIO-94: More detailed stack trace for REST API errors.
+- PIO-97: Update examples in official templates.
+- PIO-102, PIO-106, PIO-117, PIO-118, PIO-120: Bug fixes, refactoring, and
+ improved performance on elasticsearch behavior.
+- PIO-104: Bug fix regarding plugins.
+- PIO-107: Obsolete experimental examples are removed.
+
+#### Credits
+
+The following contributors have spent a great deal of effort to bring to you
+this release:
+
+Aayush Kumar, Chan Lee, Donald Szeto, Hugo Duksis, Juha Syrjälä, Lucas Bonatto,
+Marius Rabenarivo, Mars Hall, Naoki Takezoe, Nilmax Moura, Shinsuke Sugaya,
+Tomasz Stęczniewski, Vaghawan Ojha
+
### 0.11.0
-Apr ?, 2017
+Apr 25, 2017
#### New Features
[28/50] [abbrv] incubator-predictionio git commit: [PIO-110]
Refactoring
Posted by ch...@apache.org.
[PIO-110] Refactoring
Closes #425
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/6789dbeb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/6789dbeb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/6789dbeb
Branch: refs/heads/master
Commit: 6789dbeb71b1cc7f13a385032da6fbc3b8cf7a12
Parents: 3856e5c
Author: Naoki Takezoe <ta...@apache.org>
Authored: Mon Aug 28 19:31:58 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Mon Aug 28 19:31:58 2017 +0900
----------------------------------------------------------------------
.../authentication/KeyAuthentication.scala | 4 +-
.../apache/predictionio/controller/Engine.scala | 41 ++++++-----
.../predictionio/controller/EngineParams.scala | 4 +-
.../controller/FastEvalEngine.scala | 4 +-
.../controller/IdentityPreparator.scala | 2 -
.../predictionio/controller/LAlgorithm.scala | 15 ++--
.../controller/MetricEvaluator.scala | 10 +--
.../predictionio/controller/P2LAlgorithm.scala | 15 ++--
.../predictionio/controller/PAlgorithm.scala | 15 ++--
.../core/SelfCleaningDataSource.scala | 24 +++----
.../predictionio/workflow/BatchPredict.scala | 11 ++-
.../predictionio/workflow/CreateServer.scala | 24 +++----
.../predictionio/workflow/CreateWorkflow.scala | 8 +--
.../workflow/EngineServerPluginContext.scala | 4 +-
.../predictionio/workflow/WorkflowContext.scala | 4 +-
.../predictionio/workflow/WorkflowUtils.scala | 17 ++---
.../predictionio/workflow/index.scala.html | 14 ++--
.../predictionio/controller/EngineTest.scala | 2 +-
.../apache/predictionio/data/api/Common.scala | 1 -
.../predictionio/data/api/EventServer.scala | 4 +-
.../data/api/EventServerPluginContext.scala | 4 +-
.../apache/predictionio/data/api/Stats.scala | 2 +-
.../apache/predictionio/data/api/Webhooks.scala | 27 ++------
.../data/storage/EngineInstances.scala | 4 +-
.../data/storage/EvaluationInstances.scala | 4 +-
.../data/storage/EventJson4sSupport.scala | 3 +-
.../data/storage/PEventAggregator.scala | 2 -
.../predictionio/data/storage/Storage.scala | 4 +-
.../predictionio/data/view/LBatchView.scala | 1 -
.../predictionio/data/view/QuickTest.scala | 4 --
.../e2/engine/BinaryVectorizer.scala | 1 -
.../e2/engine/CategoricalNaiveBayes.scala | 1 -
.../predictionio/e2/engine/MarkovChain.scala | 1 -
.../data/storage/elasticsearch/ESApps.scala | 2 +-
.../storage/elasticsearch/ESEventsUtil.scala | 2 +-
.../data/storage/elasticsearch/ESLEvents.scala | 30 +-------
.../data/storage/elasticsearch/ESPEvents.scala | 3 +-
.../data/storage/elasticsearch/ESUtils.scala | 15 ++--
.../storage/elasticsearch/ESAccessKeys.scala | 2 +-
.../data/storage/elasticsearch/ESApps.scala | 2 +-
.../elasticsearch/ESEngineInstances.scala | 6 +-
.../elasticsearch/ESEvaluationInstances.scala | 4 +-
.../data/storage/hbase/HBEventsUtil.scala | 41 +++++------
.../data/storage/hbase/upgrade/HB_0_8_0.scala | 2 +-
.../predictionio/tools/RunBatchPredict.scala | 4 +-
.../apache/predictionio/tools/RunWorkflow.scala | 20 +++---
.../org/apache/predictionio/tools/Runner.scala | 2 +-
.../tools/admin/CommandClient.scala | 4 +-
.../predictionio/tools/commands/App.scala | 72 ++++++++++----------
.../tools/commands/Management.scala | 2 +-
.../predictionio/tools/console/Console.scala | 8 +--
51 files changed, 207 insertions(+), 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala
----------------------------------------------------------------------
diff --git a/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala b/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala
index 3ebc0b4..fa950aa 100644
--- a/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala
+++ b/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala
@@ -49,12 +49,12 @@ trait KeyAuthentication {
val passedKey = accessKeyParamOpt.getOrElse {
Left(AuthenticationFailedRejection(
- AuthenticationFailedRejection.CredentialsRejected, List()))
+ AuthenticationFailedRejection.CredentialsRejected, Nil))
}
if (!ServerKey.authEnforced || passedKey.equals(ServerKey.get)) Right(ctx.request)
else Left(AuthenticationFailedRejection(
- AuthenticationFailedRejection.CredentialsRejected, List()))
+ AuthenticationFailedRejection.CredentialsRejected, Nil))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/Engine.scala b/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
index 436c542..1f9d0ab 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
@@ -38,7 +38,6 @@ import org.apache.predictionio.workflow.StopAfterReadInterruption
import org.apache.predictionio.workflow.WorkflowParams
import org.apache.predictionio.workflow.WorkflowUtils
import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
import org.apache.spark.rdd.RDD
import org.json4s._
import org.json4s.native.JsonMethods._
@@ -255,13 +254,12 @@ class Engine[TD, EI, PD, Q, P, A](
s"Loaded model ${m.getClass.getName} for algorithm " +
s"${algo.getClass.getName}")
sc.stop
- m
} catch {
case e: NullPointerException =>
logger.warn(
s"Null model detected for algorithm ${algo.getClass.getName}")
- m
}
+ m
}
} // model match
}
@@ -692,7 +690,7 @@ object Engine {
val models: Seq[Any] = algorithmList.map(_.trainBase(sc, pd))
if (!params.skipSanityCheck) {
- models.foreach { model => {
+ models.foreach { model =>
model match {
case sanityCheckable: SanityCheck => {
logger.info(s"${model.getClass.getName} supports data sanity" +
@@ -704,7 +702,7 @@ object Engine {
" data sanity check. Skipping check.")
}
}
- }}
+ }
}
logger.info("EngineWorkflow.train completed")
@@ -758,49 +756,49 @@ object Engine {
.mapValues(_._3)
.mapValues{ _.zipWithUniqueId().map(_.swap) }
- val preparedMap: Map[EX, PD] = evalTrainMap.mapValues { td => {
+ val preparedMap: Map[EX, PD] = evalTrainMap.mapValues { td =>
preparator.prepareBase(sc, td)
- }}
+ }
- val algoModelsMap: Map[EX, Map[AX, Any]] = preparedMap.mapValues { pd => {
+ val algoModelsMap: Map[EX, Map[AX, Any]] = preparedMap.mapValues { pd =>
algoMap.mapValues(_.trainBase(sc,pd))
- }}
+ }
val suppQAsMap: Map[EX, RDD[(QX, (Q, A))]] = evalQAsMap.mapValues { qas =>
qas.map { case (qx, (q, a)) => (qx, (serving.supplementBase(q), a)) }
}
val algoPredictsMap: Map[EX, RDD[(QX, Seq[P])]] = (0 until evalCount)
- .map { ex => {
+ .map { ex =>
val modelMap: Map[AX, Any] = algoModelsMap(ex)
val qs: RDD[(QX, Q)] = suppQAsMap(ex).mapValues(_._1)
val algoPredicts: Seq[RDD[(QX, (AX, P))]] = (0 until algoCount)
- .map { ax => {
+ .map { ax =>
val algo = algoMap(ax)
val model = modelMap(ax)
val rawPredicts: RDD[(QX, P)] = algo.batchPredictBase(sc, model, qs)
- val predicts: RDD[(QX, (AX, P))] = rawPredicts.map { case (qx, p) => {
+ val predicts: RDD[(QX, (AX, P))] = rawPredicts.map { case (qx, p) =>
(qx, (ax, p))
- }}
+ }
predicts
- }}
+ }
val unionAlgoPredicts: RDD[(QX, Seq[P])] = sc.union(algoPredicts)
.groupByKey()
- .mapValues { ps => {
+ .mapValues { ps =>
assert (ps.size == algoCount, "Must have same length as algoCount")
// TODO. Check size == algoCount
ps.toSeq.sortBy(_._1).map(_._2)
- }}
+ }
(ex, unionAlgoPredicts)
- }}
+ }
.toMap
val servingQPAMap: Map[EX, RDD[(Q, P, A)]] = algoPredictsMap
- .map { case (ex, psMap) => {
+ .map { case (ex, psMap) =>
// The query passed to serving.serve is the original one, not
// supplemented.
val qasMap: RDD[(QX, (Q, A))] = evalQAsMap(ex)
@@ -811,12 +809,11 @@ object Engine {
case (qx, q, ps, a) => (q, serving.serveBase(q, ps), a)
}
(ex, qpaMap)
- }}
+ }
- (0 until evalCount).map { ex => {
+ (0 until evalCount).map { ex =>
(evalInfoMap(ex), servingQPAMap(ex))
- }}
- .toSeq
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/EngineParams.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/EngineParams.scala b/core/src/main/scala/org/apache/predictionio/controller/EngineParams.scala
index 6dccd4a..8068eaa 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/EngineParams.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/EngineParams.scala
@@ -35,7 +35,7 @@ import scala.language.implicitConversions
class EngineParams(
val dataSourceParams: (String, Params) = ("", EmptyParams()),
val preparatorParams: (String, Params) = ("", EmptyParams()),
- val algorithmParamsList: Seq[(String, Params)] = Seq(),
+ val algorithmParamsList: Seq[(String, Params)] = Nil,
val servingParams: (String, Params) = ("", EmptyParams()))
extends Serializable {
@@ -102,7 +102,7 @@ object EngineParams {
dataSourceParams: Params = EmptyParams(),
preparatorName: String = "",
preparatorParams: Params = EmptyParams(),
- algorithmParamsList: Seq[(String, Params)] = Seq(),
+ algorithmParamsList: Seq[(String, Params)] = Nil,
servingName: String = "",
servingParams: Params = EmptyParams()): EngineParams = {
new EngineParams(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/FastEvalEngine.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/FastEvalEngine.scala b/core/src/main/scala/org/apache/predictionio/controller/FastEvalEngine.scala
index e046b62..d128776 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/FastEvalEngine.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/FastEvalEngine.scala
@@ -270,10 +270,10 @@ object FastEvalEngineWorkflow {
workflow: FastEvalEngineWorkflow[TD, EI, PD, Q, P, A],
engineParamsList: Seq[EngineParams])
: Seq[(EngineParams, Seq[(EI, RDD[(Q, P, A)])])] = {
- engineParamsList.map { engineParams => {
+ engineParamsList.map { engineParams =>
(engineParams,
getServingResult(workflow, new ServingPrefix(engineParams)))
- }}
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala b/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
index a82f493..8256142 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
@@ -22,8 +22,6 @@ import org.apache.predictionio.core.BaseDataSource
import org.apache.predictionio.core.BasePreparator
import org.apache.spark.SparkContext
-import scala.reflect._
-
/** A helper concrete implementation of [[org.apache.predictionio.core.BasePreparator]]
* that passes training data through without any special preparation. This can
* be used in place for both [[PPreparator]] and [[LPreparator]].
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala b/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
index 9e973e4..27d1d14 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
@@ -119,15 +119,12 @@ abstract class LAlgorithm[PD, M : ClassTag, Q, P]
bm: Any): Any = {
// Check RDD[M].count == 1
val m = bm.asInstanceOf[RDD[M]].first()
- if (m.isInstanceOf[PersistentModel[_]]) {
- if (m.asInstanceOf[PersistentModel[Params]].save(
- modelId, algoParams, sc)) {
- PersistentModelManifest(className = m.getClass.getName)
- } else {
- ()
- }
- } else {
- m
+ m match {
+ case m: PersistentModel[Params] @unchecked =>
+ if(m.save(modelId, algoParams, sc)){
+ PersistentModelManifest(className = m.getClass.getName)
+ } else ()
+ case _ => m
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/MetricEvaluator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/MetricEvaluator.scala b/core/src/main/scala/org/apache/predictionio/controller/MetricEvaluator.scala
index 73ecbe4..fc5ec15 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/MetricEvaluator.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/MetricEvaluator.scala
@@ -222,9 +222,8 @@ class MetricEvaluator[EI, Q, P, A, R] (
params: WorkflowParams): MetricEvaluatorResult[R] = {
val evalResultList: Seq[(EngineParams, MetricScores[R])] = engineEvalDataSet
- .zipWithIndex
.par
- .map { case ((engineParams, evalDataSet), idx) =>
+ .map { case (engineParams, evalDataSet) =>
val metricScores = MetricScores[R](
metric.calculate(sc, evalDataSet),
otherMetrics.map(_.calculate(sc, evalDataSet)))
@@ -235,15 +234,16 @@ class MetricEvaluator[EI, Q, P, A, R] (
implicit lazy val formats = Utils.json4sDefaultFormats +
new NameParamsSerializer
- evalResultList.zipWithIndex.foreach { case ((ep, r), idx) =>
+ val evalResultListWithIndex = evalResultList.zipWithIndex
+
+ evalResultListWithIndex.foreach { case ((ep, r), idx) =>
logger.info(s"Iteration $idx")
logger.info(s"EngineParams: ${JsonExtractor.engineParamsToJson(Both, ep)}")
logger.info(s"Result: $r")
}
// use max. take implicit from Metric.
- val ((bestEngineParams, bestScore), bestIdx) = evalResultList
- .zipWithIndex
+ val ((bestEngineParams, bestScore), bestIdx) = evalResultListWithIndex
.reduce { (x, y) =>
if (metric.compare(x._1._2.score, y._1._2.score) >= 0) x else y
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala b/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
index ede8dc2..c617d2c 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
@@ -110,15 +110,12 @@ abstract class P2LAlgorithm[PD, M: ClassTag, Q: ClassTag, P]
algoParams: Params,
bm: Any): Any = {
val m = bm.asInstanceOf[M]
- if (m.isInstanceOf[PersistentModel[_]]) {
- if (m.asInstanceOf[PersistentModel[Params]].save(
- modelId, algoParams, sc)) {
- PersistentModelManifest(className = m.getClass.getName)
- } else {
- ()
- }
- } else {
- m
+ m match {
+ case m: PersistentModel[Params] @unchecked =>
+ if(m.save(modelId, algoParams, sc)){
+ PersistentModelManifest(className = m.getClass.getName)
+ } else ()
+ case _ => m
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala b/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
index 3419de3..55f8363 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
@@ -115,15 +115,12 @@ abstract class PAlgorithm[PD, M, Q, P]
algoParams: Params,
bm: Any): Any = {
val m = bm.asInstanceOf[M]
- if (m.isInstanceOf[PersistentModel[_]]) {
- if (m.asInstanceOf[PersistentModel[Params]].save(
- modelId, algoParams, sc)) {
- PersistentModelManifest(className = m.getClass.getName)
- } else {
- ()
- }
- } else {
- ()
+ m match {
+ case m: PersistentModel[Params] @unchecked =>
+ if(m.save(modelId, algoParams, sc)){
+ PersistentModelManifest(className = m.getClass.getName)
+ } else ()
+ case _ => ()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala b/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala
index 3520d80..cadf6b8 100644
--- a/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala
+++ b/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala
@@ -93,7 +93,6 @@ trait SelfCleaningDataSource {
*/
@DeveloperApi
def getCleanedLEvents(lEvents: Iterable[Event]): Iterable[Event] = {
-
eventWindow
.flatMap(_.duration)
.map { duration =>
@@ -101,7 +100,7 @@ trait SelfCleaningDataSource {
lEvents.filter(e =>
e.eventTime.isAfter(DateTime.now().minus(fd.toMillis)) || isSetEvent(e)
)
- }.getOrElse(lEvents).toIterable
+ }.getOrElse(lEvents)
}
def compressPProperties(sc: SparkContext, rdd: RDD[Event]): RDD[Event] = {
@@ -117,7 +116,7 @@ trait SelfCleaningDataSource {
}
def compressLProperties(events: Iterable[Event]): Iterable[Event] = {
- events.filter(isSetEvent).toIterable
+ events.filter(isSetEvent)
.groupBy(_.entityType)
.map { pair =>
val (_, ls) = pair
@@ -164,7 +163,7 @@ trait SelfCleaningDataSource {
val result = cleanPEvents(sc)
val originalEvents = PEventStore.find(appName)(sc)
val newEvents = result subtract originalEvents
- val eventsToRemove = (originalEvents subtract result).map { case e =>
+ val eventsToRemove = (originalEvents subtract result).map { e =>
e.eventId.getOrElse("")
}
@@ -187,7 +186,7 @@ trait SelfCleaningDataSource {
def removeEvents(eventsToRemove: Set[String], appId: Int) {
val listOfFuture: List[Future[Boolean]] = eventsToRemove
- .filter(x => x != "").toList.map { case eventId =>
+ .filter(x => x != "").toList.map { eventId =>
lEventsDb.futureDelete(eventId, appId)
}
@@ -202,9 +201,8 @@ trait SelfCleaningDataSource {
/** Replace events in Event Store
*
- * @param events new events
- * @param appId delete all events of appId
- * @param channelId delete all events of channelId
+ * @param newEvents new events
+ * @param eventsToRemove event ids to remove
*/
def wipe(
newEvents: Set[Event],
@@ -212,7 +210,7 @@ trait SelfCleaningDataSource {
): Unit = {
val (appId, channelId) = Common.appNameToId(appName, None)
- val listOfFutureNewEvents: List[Future[String]] = newEvents.toList.map { case event =>
+ val listOfFutureNewEvents: List[Future[String]] = newEvents.toList.map { event =>
lEventsDb.futureInsert(recreateEvent(event, None, event.eventTime), appId)
}
@@ -233,10 +231,10 @@ trait SelfCleaningDataSource {
val rdd = eventWindow match {
case Some(ew) =>
- var updated =
+ val updated =
if (ew.compressProperties) compressPProperties(sc, pEvents) else pEvents
- val deduped = if (ew.removeDuplicates) removePDuplicates(sc,updated) else updated
+ val deduped = if (ew.removeDuplicates) removePDuplicates(sc, updated) else updated
deduped
case None =>
pEvents
@@ -258,7 +256,7 @@ trait SelfCleaningDataSource {
val result = cleanLEvents().toSet
val originalEvents = LEventStore.find(appName).toSet
val newEvents = result -- originalEvents
- val eventsToRemove = (originalEvents -- result).map { case e =>
+ val eventsToRemove = (originalEvents -- result).map { e =>
e.eventId.getOrElse("")
}
@@ -278,7 +276,7 @@ trait SelfCleaningDataSource {
val events = eventWindow match {
case Some(ew) =>
- var updated =
+ val updated =
if (ew.compressProperties) compressLProperties(lEvents) else lEvents
val deduped = if (ew.removeDuplicates) removeLDuplicates(updated) else updated
deduped
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala b/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
index 2fb0545..5420638 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
@@ -134,12 +134,11 @@ object BatchPredict extends Logging {
val maybeEngine = engineFactory()
// EngineFactory return a base engine, which may not be deployable.
- if (!maybeEngine.isInstanceOf[Engine[_,_,_,_,_,_]]) {
- throw new NoSuchMethodException(
+ maybeEngine match {
+ case e: Engine[_, _, _, _, _, _] => e
+ case _ => throw new NoSuchMethodException(
s"Engine $maybeEngine cannot be used for batch predict")
}
-
- maybeEngine.asInstanceOf[Engine[_,_,_,_,_,_]]
}
def run[Q, P](
@@ -207,8 +206,8 @@ object BatchPredict extends Logging {
// finally Serving.serve.
val supplementedQuery = serving.supplementBase(query)
// TODO: Parallelize the following.
- val predictions = algorithms.zipWithIndex.map { case (a, ai) =>
- a.predictBase(models(ai), supplementedQuery)
+ val predictions = algorithms.zip(models).map { case (a, m) =>
+ a.predictBase(m, supplementedQuery)
}
// Notice that it is by design to call Serving.serve with the
// *original* query.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
index 8f0aed7..2447682 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
@@ -18,7 +18,7 @@
package org.apache.predictionio.workflow
-import java.io.{PrintWriter, Serializable, StringWriter}
+import java.io.Serializable
import java.util.concurrent.TimeUnit
import akka.actor._
@@ -32,6 +32,7 @@ import com.twitter.chill.{KryoBase, KryoInjection, ScalaKryoInstantiator}
import com.typesafe.config.ConfigFactory
import de.javakaffee.kryoserializers.SynchronizedCollectionsSerializer
import grizzled.slf4j.Logging
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.predictionio.authentication.KeyAuthentication
import org.apache.predictionio.configuration.SSLConfiguration
import org.apache.predictionio.controller.{Engine, Params, Utils, WithPrId}
@@ -309,7 +310,7 @@ class MasterActor (
sprayHttpListener.map { l =>
log.info("Server is shutting down.")
l ! Http.Unbind(5.seconds)
- system.shutdown
+ system.shutdown()
} getOrElse {
log.warning("No active server is running.")
}
@@ -353,7 +354,7 @@ class MasterActor (
}
} else {
log.error("Bind failed. Shutting down.")
- system.shutdown
+ system.shutdown()
}
}
@@ -432,13 +433,6 @@ class ServerActor[Q, P](
}
}
- def getStackTraceString(e: Throwable): String = {
- val writer = new StringWriter()
- val printWriter = new PrintWriter(writer)
- e.printStackTrace(printWriter)
- writer.toString
- }
-
val myRoute =
path("") {
get {
@@ -492,8 +486,8 @@ class ServerActor[Q, P](
// finally Serving.serve.
val supplementedQuery = serving.supplementBase(query)
// TODO: Parallelize the following.
- val predictions = algorithms.zipWithIndex.map { case (a, ai) =>
- a.predictBase(models(ai), supplementedQuery)
+ val predictions = algorithms.zip(models).map { case (a, m) =>
+ a.predictBase(m, supplementedQuery)
}
// Notice that it is by design to call Serving.serve with the
// *original* query.
@@ -533,7 +527,7 @@ class ServerActor[Q, P](
case id: WithPrId =>
Map("prId" -> id.prId)
case _ =>
- Map()
+ Map.empty
}
val data = Map(
// "appId" -> dataSourceParams.asInstanceOf[ParamsWithAppId].appId,
@@ -596,7 +590,7 @@ class ServerActor[Q, P](
} catch {
case e: MappingException =>
val msg = s"Query:\n$queryString\n\nStack Trace:\n" +
- s"${getStackTraceString(e)}\n\n"
+ s"${ExceptionUtils.getStackTrace(e)}\n\n"
log.error(msg)
args.logUrl map { url =>
remoteLog(
@@ -607,7 +601,7 @@ class ServerActor[Q, P](
complete(StatusCodes.BadRequest, e.getMessage)
case e: Throwable =>
val msg = s"Query:\n$queryString\n\nStack Trace:\n" +
- s"${getStackTraceString(e)}\n\n"
+ s"${ExceptionUtils.getStackTrace(e)}\n\n"
log.error(msg)
args.logUrl map { url =>
remoteLog(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala b/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
index 899ace2..303ed06 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CreateWorkflow.scala
@@ -165,14 +165,14 @@ object CreateWorkflow extends Logging {
}
}
- val pioEnvVars = wfc.env.map(e =>
- e.split(',').flatMap(p =>
+ val pioEnvVars = wfc.env.map { e =>
+ e.split(',').flatMap { p =>
p.split('=') match {
case Array(k, v) => List(k -> v)
case _ => Nil
}
- ).toMap
- ).getOrElse(Map())
+ }.toMap
+ }.getOrElse(Map.empty)
if (evaluation.isEmpty) {
val variantJson = parse(stringFromFile(wfc.engineVariant))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPluginContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPluginContext.scala b/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPluginContext.scala
index 78d86ac..cfc83eb 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPluginContext.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/EngineServerPluginContext.scala
@@ -41,9 +41,9 @@ class EngineServerPluginContext(
val pluginParams: mutable.Map[String, JValue],
val log: LoggingAdapter) {
def outputBlockers: Map[String, EngineServerPlugin] =
- plugins.getOrElse(EngineServerPlugin.outputBlocker, Map()).toMap
+ plugins.getOrElse(EngineServerPlugin.outputBlocker, Map.empty).toMap
def outputSniffers: Map[String, EngineServerPlugin] =
- plugins.getOrElse(EngineServerPlugin.outputSniffer, Map()).toMap
+ plugins.getOrElse(EngineServerPlugin.outputSniffer, Map.empty).toMap
}
object EngineServerPluginContext extends Logging {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/workflow/WorkflowContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/WorkflowContext.scala b/core/src/main/scala/org/apache/predictionio/workflow/WorkflowContext.scala
index 7bd9117..c1e6937 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/WorkflowContext.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/WorkflowContext.scala
@@ -28,8 +28,8 @@ import scala.language.existentials
object WorkflowContext extends Logging {
def apply(
batch: String = "",
- executorEnv: Map[String, String] = Map(),
- sparkEnv: Map[String, String] = Map(),
+ executorEnv: Map[String, String] = Map.empty,
+ sparkEnv: Map[String, String] = Map.empty,
mode: String = ""
): SparkContext = {
val conf = new SparkConf()
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala b/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
index 0e578be..9a75415 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
@@ -60,12 +60,11 @@ object WorkflowUtils extends Logging {
engineObject.instance.asInstanceOf[EngineFactory]
)
} catch {
- case e @ (_: NoSuchFieldException | _: ClassNotFoundException) => try {
+ case e @ (_: NoSuchFieldException | _: ClassNotFoundException) =>
(
EngineLanguage.Java,
Class.forName(engine).newInstance.asInstanceOf[EngineFactory]
)
- }
}
}
@@ -80,12 +79,11 @@ object WorkflowUtils extends Logging {
epgObject.instance.asInstanceOf[EngineParamsGenerator]
)
} catch {
- case e @ (_: NoSuchFieldException | _: ClassNotFoundException) => try {
+ case e @ (_: NoSuchFieldException | _: ClassNotFoundException) =>
(
EngineLanguage.Java,
Class.forName(epg).newInstance.asInstanceOf[EngineParamsGenerator]
)
- }
}
}
@@ -99,12 +97,11 @@ object WorkflowUtils extends Logging {
evaluationObject.instance.asInstanceOf[Evaluation]
)
} catch {
- case e @ (_: NoSuchFieldException | _: ClassNotFoundException) => try {
+ case e @ (_: NoSuchFieldException | _: ClassNotFoundException) =>
(
EngineLanguage.Java,
Class.forName(evaluation).newInstance.asInstanceOf[Evaluation]
)
- }
}
}
@@ -265,9 +262,9 @@ object WorkflowUtils extends Logging {
Seq(file.toURI)
} else {
warn(s"Environment variable $p is pointing to a nonexistent file $f. Ignoring.")
- Seq.empty[URI]
+ Seq.empty
}
- } getOrElse Seq.empty[URI]
+ } getOrElse Seq.empty
)
}
@@ -325,8 +322,8 @@ object WorkflowUtils extends Logging {
error("Arrays are not allowed in the sparkConf section of engine.js.")
sys.exit(1)
}
- case JNothing => List()
- case _ => List(List() -> jv.values.toString)
+ case JNothing => Nil
+ case _ => List(Nil -> jv.values.toString)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/main/twirl/org/apache/predictionio/workflow/index.scala.html
----------------------------------------------------------------------
diff --git a/core/src/main/twirl/org/apache/predictionio/workflow/index.scala.html b/core/src/main/twirl/org/apache/predictionio/workflow/index.scala.html
index 62fe5a7..5040796 100644
--- a/core/src/main/twirl/org/apache/predictionio/workflow/index.scala.html
+++ b/core/src/main/twirl/org/apache/predictionio/workflow/index.scala.html
@@ -75,13 +75,13 @@
<h2>Algorithms and Models</h2>
<table class="table table-bordered table-striped">
<tr><th>#</th><th colspan="2">Information</th></tr>
- @for(a <- algorithms.zip(algorithmsParams).zip(models).zipWithIndex) {
- <tr>
- <th rowspan="3">@{a._2 + 1}</th>
- <th>Class</th><td>@{a._1._1._1}</td>
- </tr>
- <tr><th>Parameters</th><td>@{a._1._1._2}</td></tr>
- <tr><th>Model</th><td>@{a._1._2}</td></tr>
+ @for((((algo, param), model), i) <- algorithms.zip(algorithmsParams).zip(models).zipWithIndex) {
+ <tr>
+ <th rowspan="3">@{i + 1}</th>
+ <th>Class</th><td>@{algo}</td>
+ </tr>
+ <tr><th>Parameters</th><td>@{param}</td></tr>
+ <tr><th>Model</th><td>@{model}</td></tr>
}
</table>
<h2>Serving</h2>
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala b/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
index 94879a5..fb10c94 100644
--- a/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
@@ -320,7 +320,7 @@ class EngineTrainSuite extends FunSuite with SharedSparkContext {
sc,
new PDataSource0(0),
new PPreparator0(1),
- Seq(),
+ Nil,
defaultWorkflowParams
)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/api/Common.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/Common.scala b/data/src/main/scala/org/apache/predictionio/data/api/Common.scala
index a579add..60efea2 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/Common.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/Common.scala
@@ -25,7 +25,6 @@ import spray.routing._
import spray.routing.Directives._
import spray.routing.Rejection
import spray.http.StatusCodes
-import spray.http.StatusCode
import spray.httpx.Json4sSupport
import org.json4s.Formats
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
index 75c2227..41dfefb 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
@@ -131,13 +131,13 @@ class EventServiceActor(
private val FailedAuth = Left(
AuthenticationFailedRejection(
- AuthenticationFailedRejection.CredentialsRejected, List()
+ AuthenticationFailedRejection.CredentialsRejected, Nil
)
)
private val MissedAuth = Left(
AuthenticationFailedRejection(
- AuthenticationFailedRejection.CredentialsMissing, List()
+ AuthenticationFailedRejection.CredentialsMissing, Nil
)
)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/api/EventServerPluginContext.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/EventServerPluginContext.scala b/data/src/main/scala/org/apache/predictionio/data/api/EventServerPluginContext.scala
index 36f3f73..cd14cc4 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/EventServerPluginContext.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/EventServerPluginContext.scala
@@ -30,10 +30,10 @@ class EventServerPluginContext(
val plugins: mutable.Map[String, mutable.Map[String, EventServerPlugin]],
val log: LoggingAdapter) {
def inputBlockers: Map[String, EventServerPlugin] =
- plugins.getOrElse(EventServerPlugin.inputBlocker, Map()).toMap
+ plugins.getOrElse(EventServerPlugin.inputBlocker, Map.empty).toMap
def inputSniffers: Map[String, EventServerPlugin] =
- plugins.getOrElse(EventServerPlugin.inputSniffer, Map()).toMap
+ plugins.getOrElse(EventServerPlugin.inputSniffer, Map.empty).toMap
}
object EventServerPluginContext extends Logging {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/api/Stats.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/Stats.scala b/data/src/main/scala/org/apache/predictionio/data/api/Stats.scala
index ba2b575..9bbbc2e 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/Stats.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/Stats.scala
@@ -67,7 +67,7 @@ class Stats(val startTime: DateTime) {
m
.toSeq
.flatMap { case (k, v) =>
- if (k._1 == appId) { Seq(KV(k._2, v)) } else { Seq() }
+ if (k._1 == appId) { Seq(KV(k._2, v)) } else { Nil }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/api/Webhooks.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/Webhooks.scala b/data/src/main/scala/org/apache/predictionio/data/api/Webhooks.scala
index 87a4600..57be037 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/Webhooks.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/Webhooks.scala
@@ -18,22 +18,13 @@
package org.apache.predictionio.data.api
-import org.apache.predictionio.data.webhooks.JsonConnector
-import org.apache.predictionio.data.webhooks.FormConnector
import org.apache.predictionio.data.webhooks.ConnectorUtil
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.EventJson4sSupport
import org.apache.predictionio.data.storage.LEvents
-import spray.routing._
-import spray.routing.Directives._
import spray.http.StatusCodes
import spray.http.StatusCode
import spray.http.FormData
-import spray.httpx.Json4sSupport
-import org.json4s.Formats
-import org.json4s.DefaultFormats
import org.json4s.JObject
import akka.event.LoggingAdapter
@@ -61,14 +52,13 @@ private[predictionio] object Webhooks {
}
}
- eventFuture.flatMap { eventOpt =>
- if (eventOpt.isEmpty) {
+ eventFuture.flatMap {
+ case None =>
Future successful {
val message = s"webhooks connection for ${web} is not supported."
(StatusCodes.NotFound, Map("message" -> message))
}
- } else {
- val event = eventOpt.get
+ case Some(event) =>
val data = eventClient.futureInsert(event, appId, channelId).map { id =>
val result = (StatusCodes.Created, Map("eventId" -> s"${id}"))
@@ -78,7 +68,6 @@ private[predictionio] object Webhooks {
result
}
data
- }
}
}
@@ -114,14 +103,13 @@ private[predictionio] object Webhooks {
}
}
- eventFuture.flatMap { eventOpt =>
- if (eventOpt.isEmpty) {
- Future {
+ eventFuture.flatMap {
+ case None =>
+ Future successful {
val message = s"webhooks connection for ${web} is not supported."
(StatusCodes.NotFound, Map("message" -> message))
}
- } else {
- val event = eventOpt.get
+ case Some(event) =>
val data = eventClient.futureInsert(event, appId, channelId).map { id =>
val result = (StatusCodes.Created, Map("eventId" -> s"${id}"))
@@ -131,7 +119,6 @@ private[predictionio] object Webhooks {
result
}
data
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/storage/EngineInstances.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/EngineInstances.scala b/data/src/main/scala/org/apache/predictionio/data/storage/EngineInstances.scala
index 7c3aad0..82d62f8 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/EngineInstances.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/EngineInstances.scala
@@ -118,8 +118,8 @@ class EngineInstanceSerializer
engineVariant = "",
engineFactory = "",
batch = "",
- env = Map(),
- sparkConf = Map(),
+ env = Map.empty,
+ sparkConf = Map.empty,
dataSourceParams = "",
preparatorParams = "",
algorithmsParams = "",
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/storage/EvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/EvaluationInstances.scala b/data/src/main/scala/org/apache/predictionio/data/storage/EvaluationInstances.scala
index a40adb3..5714fde 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/EvaluationInstances.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/EvaluationInstances.scala
@@ -47,8 +47,8 @@ case class EvaluationInstance(
evaluationClass: String = "",
engineParamsGeneratorClass: String = "",
batch: String = "",
- env: Map[String, String] = Map(),
- sparkConf: Map[String, String] = Map(),
+ env: Map[String, String] = Map.empty,
+ sparkConf: Map[String, String] = Map.empty,
evaluatorResults: String = "",
evaluatorResultsHTML: String = "",
evaluatorResultsJSON: String = "")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala b/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
index ba20c61..57f0472 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
@@ -56,7 +56,7 @@ object EventJson4sSupport {
val targetEntityType = fields.getOpt[String]("targetEntityType")
val targetEntityId = fields.getOpt[String]("targetEntityId")
val properties = fields.getOrElse[Map[String, JValue]](
- "properties", Map())
+ "properties", Map.empty)
// default currentTime expressed as UTC timezone
lazy val currentTime = DateTime.now(EventValidation.defaultTimeZone)
val eventTime = fields.getOpt[String]("eventTime")
@@ -70,7 +70,6 @@ object EventJson4sSupport {
}.getOrElse(currentTime)
// disable tags from API for now.
- val tags = List()
// val tags = fields.getOpt[Seq[String]]("tags").getOrElse(List())
val prId = fields.getOpt[String]("prId")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/storage/PEventAggregator.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/PEventAggregator.scala b/data/src/main/scala/org/apache/predictionio/data/storage/PEventAggregator.scala
index 8e58384..5cf8ffc 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/PEventAggregator.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/PEventAggregator.scala
@@ -22,8 +22,6 @@ import org.joda.time.DateTime
import org.json4s.JValue
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
import org.apache.spark.rdd.RDD
// each JValue data associated with the time it is set
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
index fd05767..52442a6 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
@@ -18,8 +18,6 @@
package org.apache.predictionio.data.storage
-import java.lang.reflect.InvocationTargetException
-
import grizzled.slf4j.Logging
import org.apache.predictionio.annotation.DeveloperApi
@@ -78,7 +76,7 @@ trait BaseStorageClient {
case class StorageClientConfig(
parallel: Boolean = false, // parallelized access (RDD)?
test: Boolean = false, // test mode config
- properties: Map[String, String] = Map())
+ properties: Map[String, String] = Map.empty)
/** :: DeveloperApi ::
* Thrown when a StorageClient runs into an exceptional condition
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala b/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
index d9c53ef..4f531b6 100644
--- a/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
@@ -136,7 +136,6 @@ class EventSeq(val events: List[Event]) {
events
.groupBy( _.entityId )
.mapValues( _.sortBy(_.eventTime.getMillis).foldLeft[T](init)(op))
- .toMap
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/data/src/main/scala/org/apache/predictionio/data/view/QuickTest.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/view/QuickTest.scala b/data/src/main/scala/org/apache/predictionio/data/view/QuickTest.scala
index 30f9c17..b6b509c 100644
--- a/data/src/main/scala/org/apache/predictionio/data/view/QuickTest.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/view/QuickTest.scala
@@ -18,10 +18,6 @@
package org.apache.predictionio.data.view
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.LEvents
-import org.apache.predictionio.data.storage.EventValidation
-import org.apache.predictionio.data.storage.DataMap
import org.apache.predictionio.data.storage.Storage
import scala.concurrent.ExecutionContext.Implicits.global // TODO
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/e2/src/main/scala/org/apache/predictionio/e2/engine/BinaryVectorizer.scala
----------------------------------------------------------------------
diff --git a/e2/src/main/scala/org/apache/predictionio/e2/engine/BinaryVectorizer.scala b/e2/src/main/scala/org/apache/predictionio/e2/engine/BinaryVectorizer.scala
index 114c6b9..dae9074 100644
--- a/e2/src/main/scala/org/apache/predictionio/e2/engine/BinaryVectorizer.scala
+++ b/e2/src/main/scala/org/apache/predictionio/e2/engine/BinaryVectorizer.scala
@@ -18,7 +18,6 @@
package org.apache.predictionio.e2.engine
import org.apache.spark.rdd.RDD
-import org.apache.spark.SparkContext._
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.linalg.Vector
import scala.collection.immutable.HashMap
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/e2/src/main/scala/org/apache/predictionio/e2/engine/CategoricalNaiveBayes.scala
----------------------------------------------------------------------
diff --git a/e2/src/main/scala/org/apache/predictionio/e2/engine/CategoricalNaiveBayes.scala b/e2/src/main/scala/org/apache/predictionio/e2/engine/CategoricalNaiveBayes.scala
index 9dc6f9d..90b5267 100644
--- a/e2/src/main/scala/org/apache/predictionio/e2/engine/CategoricalNaiveBayes.scala
+++ b/e2/src/main/scala/org/apache/predictionio/e2/engine/CategoricalNaiveBayes.scala
@@ -17,7 +17,6 @@
package org.apache.predictionio.e2.engine
-import org.apache.spark.SparkContext._
import org.apache.spark.rdd.RDD
/** Class for training a naive Bayes model with categorical variables */
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/e2/src/main/scala/org/apache/predictionio/e2/engine/MarkovChain.scala
----------------------------------------------------------------------
diff --git a/e2/src/main/scala/org/apache/predictionio/e2/engine/MarkovChain.scala b/e2/src/main/scala/org/apache/predictionio/e2/engine/MarkovChain.scala
index 3c3ac34..130c53a 100644
--- a/e2/src/main/scala/org/apache/predictionio/e2/engine/MarkovChain.scala
+++ b/e2/src/main/scala/org/apache/predictionio/e2/engine/MarkovChain.scala
@@ -17,7 +17,6 @@
package org.apache.predictionio.e2.engine
-import org.apache.spark.SparkContext._
import org.apache.spark.mllib.linalg.distributed.CoordinateMatrix
import org.apache.spark.mllib.linalg.{SparseVector, Vectors}
import org.apache.spark.rdd.RDD
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
index 0b319ab..6afed12 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
@@ -141,7 +141,7 @@ class ESApps(client: ESClient, config: StorageClientConfig, index: String)
try {
val json =
("query" ->
- ("match_all" -> List.empty))
+ ("match_all" -> Nil))
ESUtils.getAll[App](restClient, index, estype, compact(render(json)))
} catch {
case e: IOException =>
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
index ec72a49..749ab49 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
@@ -80,7 +80,7 @@ object ESEventsUtil {
targetEntityId = targetEntityId,
properties = properties,
eventTime = eventTime,
- tags = Seq(),
+ tags = Nil,
prId = prId,
creationTime = creationTime
)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
index 6240059..6c0c4a7 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
@@ -28,7 +28,6 @@ import org.apache.http.util.EntityUtils
import org.apache.predictionio.data.storage.Event
import org.apache.predictionio.data.storage.LEvents
import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.client.RestClient
import org.joda.time.DateTime
import org.json4s._
import org.json4s.JsonDSL._
@@ -36,7 +35,6 @@ import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.write
import org.json4s.ext.JodaTimeSerializers
import grizzled.slf4j.Logging
-import org.elasticsearch.client.ResponseException
import org.apache.http.message.BasicHeader
class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: String)
@@ -217,29 +215,6 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
}
}
- private def exists(restClient: RestClient, estype: String, id: Int): Boolean = {
- try {
- restClient.performRequest(
- "GET",
- s"/$index/$estype/$id",
- Map.empty[String, String].asJava).getStatusLine.getStatusCode match {
- case 200 => true
- case _ => false
- }
- } catch {
- case e: ResponseException =>
- e.getResponse.getStatusLine.getStatusCode match {
- case 404 => false
- case _ =>
- error(s"Failed to access to /$index/$estype/$id", e)
- false
- }
- case e: IOException =>
- error(s"Failed to access to $index/$estype/$id", e)
- false
- }
- }
-
override def futureGet(
eventId: String,
appId: Int,
@@ -288,7 +263,8 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
val response = restClient.performRequest(
"POST",
s"/$index/$estype/_delete_by_query",
- Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava)
+ Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
+ entity)
val jsonResponse = parse(EntityUtils.toString(response.getEntity))
val result = (jsonResponse \ "result").extract[String]
result match {
@@ -331,7 +307,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
} catch {
case e: IOException =>
error(e.getMessage)
- Iterator[Event]()
+ Iterator.empty
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
index b9ad8bb..9f0a188 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
@@ -120,7 +120,8 @@ class ESPEvents(client: ESClient, config: StorageClientConfig, index: String)
val response = restClient.performRequest(
"POST",
s"/$index/$estype/_delete_by_query",
- Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava)
+ Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
+ entity)
val jsonResponse = parse(EntityUtils.toString(response.getEntity))
val result = (jsonResponse \ "result").extract[String]
result match {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
index 3f7b058..cd9aa53 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
@@ -44,16 +44,13 @@ object ESUtils {
implicit formats: Formats): Event = {
def getString(s: String): String = {
(value \ s) match {
- case x if x == JNothing => null
+ case JNothing => null
case x => x.extract[String]
}
}
def getOptString(s: String): Option[String] = {
- getString(s) match {
- case null => None
- case x => Some(x)
- }
+ Option(getString(s))
}
val properties: DataMap = getOptString("properties")
@@ -233,14 +230,14 @@ object ESUtils {
targetEntityId: Option[Option[String]] = None,
reversed: Option[Boolean] = None): String = {
val mustQueries = Seq(
- startTime.map(x => {
+ startTime.map { x =>
val v = formatUTCDateTime(x)
s"""{"range":{"eventTime":{"gte":"${v}"}}}"""
- }),
- untilTime.map(x => {
+ },
+ untilTime.map { x =>
val v = formatUTCDateTime(x)
s"""{"range":{"eventTime":{"lt":"${v}"}}}"""
- }),
+ },
entityType.map(x => s"""{"term":{"entityType":"${x}"}}"""),
entityId.map(x => s"""{"term":{"entityId":"${x}"}}"""),
targetEntityType.flatMap(xx => xx.map(x => s"""{"term":{"targetEntityType":"${x}"}}""")),
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
index 077168a..5e3abe2 100644
--- a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
@@ -95,7 +95,7 @@ class ESAccessKeys(client: Client, config: StorageClientConfig, index: String)
} catch {
case e: ElasticsearchException =>
error(e.getMessage)
- Seq[AccessKey]()
+ Nil
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
index 3781a4b..270af0e 100644
--- a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
@@ -105,7 +105,7 @@ class ESApps(client: Client, config: StorageClientConfig, index: String)
} catch {
case e: ElasticsearchException =>
error(e.getMessage)
- Seq[App]()
+ Nil
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
index 21690bf..2d6056b 100644
--- a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
@@ -107,7 +107,7 @@ class ESEngineInstances(client: Client, config: StorageClientConfig, index: Stri
} catch {
case e: ElasticsearchException =>
error(e.getMessage)
- Seq()
+ Nil
}
}
@@ -127,7 +127,7 @@ class ESEngineInstances(client: Client, config: StorageClientConfig, index: Stri
} catch {
case e: ElasticsearchException =>
error(e.getMessage)
- Seq()
+ Nil
}
}
@@ -150,7 +150,7 @@ class ESEngineInstances(client: Client, config: StorageClientConfig, index: Stri
def delete(id: String): Unit = {
try {
- val response = client.prepareDelete(index, estype, id).get
+ client.prepareDelete(index, estype, id).get
} catch {
case e: ElasticsearchException => error(e.getMessage)
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
index 85bf820..68c5a74 100644
--- a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
@@ -101,7 +101,7 @@ class ESEvaluationInstances(client: Client, config: StorageClientConfig, index:
} catch {
case e: ElasticsearchException =>
error(e.getMessage)
- Seq()
+ Nil
}
}
@@ -114,7 +114,7 @@ class ESEvaluationInstances(client: Client, config: StorageClientConfig, index:
} catch {
case e: ElasticsearchException =>
error(e.getMessage)
- Seq()
+ Nil
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
index 2cdb734..64487fb 100644
--- a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
@@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.client.Scan
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.filter.FilterList
-import org.apache.hadoop.hbase.filter.RegexStringComparator
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp
import org.apache.hadoop.hbase.filter.BinaryComparator
@@ -275,7 +274,7 @@ object HBEventsUtil {
targetEntityId = targetEntityId,
properties = properties,
eventTime = eventTime,
- tags = Seq(),
+ tags = Nil,
prId = prId,
creationTime = creationTime
)
@@ -375,34 +374,28 @@ object HBEventsUtil {
}
}
- targetEntityType.foreach { tetOpt =>
- if (tetOpt.isEmpty) {
+ targetEntityType.foreach {
+ case None =>
val filter = createSkipRowIfColumnExistFilter("targetEntityType")
filters.addFilter(filter)
- } else {
- tetOpt.foreach { tet =>
- val filter = createBinaryFilter(
- "targetEntityType", Bytes.toBytes(tet))
- // the entire row will be skipped if the column is not found.
- filter.setFilterIfMissing(true)
- filters.addFilter(filter)
- }
- }
+ case Some(tet) =>
+ val filter = createBinaryFilter(
+ "targetEntityType", Bytes.toBytes(tet))
+ // the entire row will be skipped if the column is not found.
+ filter.setFilterIfMissing(true)
+ filters.addFilter(filter)
}
- targetEntityId.foreach { teidOpt =>
- if (teidOpt.isEmpty) {
+ targetEntityId.foreach {
+ case None =>
val filter = createSkipRowIfColumnExistFilter("targetEntityId")
filters.addFilter(filter)
- } else {
- teidOpt.foreach { teid =>
- val filter = createBinaryFilter(
- "targetEntityId", Bytes.toBytes(teid))
- // the entire row will be skipped if the column is not found.
- filter.setFilterIfMissing(true)
- filters.addFilter(filter)
- }
- }
+ case Some(teid) =>
+ val filter = createBinaryFilter(
+ "targetEntityId", Bytes.toBytes(teid))
+ // the entire row will be skipped if the column is not found.
+ filter.setFilterIfMissing(true)
+ filters.addFilter(filter)
}
if (!filters.getFilters().isEmpty) {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
index cc07fa4..795cf7e 100644
--- a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
@@ -185,7 +185,7 @@ object HB_0_8_0 {
targetEntityId = targetEntityId,
properties = properties,
eventTime = new DateTime(rowKey.millis, eventTimeZone),
- tags = Seq(),
+ tags = Nil,
prId = prId,
creationTime = creationTime
)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala
index 35572c9..c76d203 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala
@@ -59,10 +59,10 @@ object RunBatchPredict extends Logging {
"--engine-variant",
batchPredictArgs.variantJson.getOrElse(
new File(engineDirPath, "engine.json")).getCanonicalPath) ++
- (if (batchPredictArgs.queryPartitions.isEmpty) Seq()
+ (if (batchPredictArgs.queryPartitions.isEmpty) Nil
else Seq("--query-partitions",
batchPredictArgs.queryPartitions.get.toString)) ++
- (if (verbose) Seq("--verbose") else Seq()) ++
+ (if (verbose) Seq("--verbose") else Nil) ++
Seq("--json-extractor", batchPredictArgs.jsonExtractor.toString)
Runner.runOnSpark(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
index 6df0750..a25f4e0 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
@@ -64,25 +64,25 @@ object RunWorkflow extends Logging {
"--verbosity",
wa.verbosity.toString) ++
wa.engineFactory.map(
- x => Seq("--engine-factory", x)).getOrElse(Seq()) ++
+ x => Seq("--engine-factory", x)).getOrElse(Nil) ++
wa.engineParamsKey.map(
- x => Seq("--engine-params-key", x)).getOrElse(Seq()) ++
- (if (wa.batch != "") Seq("--batch", wa.batch) else Seq()) ++
- (if (verbose) Seq("--verbose") else Seq()) ++
- (if (wa.skipSanityCheck) Seq("--skip-sanity-check") else Seq()) ++
- (if (wa.stopAfterRead) Seq("--stop-after-read") else Seq()) ++
+ x => Seq("--engine-params-key", x)).getOrElse(Nil) ++
+ (if (wa.batch != "") Seq("--batch", wa.batch) else Nil) ++
+ (if (verbose) Seq("--verbose") else Nil) ++
+ (if (wa.skipSanityCheck) Seq("--skip-sanity-check") else Nil) ++
+ (if (wa.stopAfterRead) Seq("--stop-after-read") else Nil) ++
(if (wa.stopAfterPrepare) {
Seq("--stop-after-prepare")
} else {
- Seq()
+ Nil
}) ++
wa.evaluation.map(x => Seq("--evaluation-class", x)).
- getOrElse(Seq()) ++
+ getOrElse(Nil) ++
// If engineParamsGenerator is specified, it overrides the evaluation.
wa.engineParamsGenerator.orElse(wa.evaluation)
.map(x => Seq("--engine-params-generator-class", x))
- .getOrElse(Seq()) ++
- (if (wa.batch != "") Seq("--batch", wa.batch) else Seq()) ++
+ .getOrElse(Nil) ++
+ (if (wa.batch != "") Seq("--batch", wa.batch) else Nil) ++
Seq("--json-extractor", wa.jsonExtractor.toString)
Runner.runOnSpark(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index 4a721be..4e266c8 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -30,7 +30,7 @@ import scala.sys.process._
case class SparkArgs(
sparkHome: Option[String] = None,
- sparkPassThrough: Seq[String] = Seq(),
+ sparkPassThrough: Seq[String] = Nil,
sparkKryo: Boolean = false,
scratchUri: Option[URI] = None)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/tools/src/main/scala/org/apache/predictionio/tools/admin/CommandClient.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/admin/CommandClient.scala b/tools/src/main/scala/org/apache/predictionio/tools/admin/CommandClient.scala
index 94a1a03..710c23a 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/admin/CommandClient.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/admin/CommandClient.scala
@@ -84,11 +84,11 @@ class CommandClient(
val accessKey = AccessKey(
key = "",
appid = id,
- events = Seq())
+ events = Nil)
val accessKey2 = accessKeyClient.insert(AccessKey(
key = "",
appid = id,
- events = Seq()))
+ events = Nil))
accessKey2 map { k =>
new AppNewResponse(1,"App created successfully.",id, req.name, k)
} getOrElse {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
index 44fa667..5884ebd 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
@@ -67,7 +67,7 @@ object App extends EitherLogging {
val newKey = storage.AccessKey(
key = accessKey,
appid = id,
- events = Seq())
+ events = Nil)
accessKeys.insert(newKey)
.map { k =>
Right(AppDescription(
@@ -85,7 +85,7 @@ object App extends EitherLogging {
errStr += s"""
|Failed to revert back the App meta-data change.
|The app ${name} CANNOT be used!
- |Please run 'pio app delete ${name}' to delete this app!"""
+ |Please run 'pio app delete ${name}' to delete this app!""".stripMargin
}
logAndFail(errStr)
}
@@ -209,12 +209,12 @@ object App extends EitherLogging {
channels.find(ch => ch.name == chName) match {
case None =>
return logAndFail(s"""Unable to delete data for channel.
- |Channel ${chName} doesn't exist.""")
+ |Channel ${chName} doesn't exist.""".stripMargin)
case Some(ch) => Seq(Some(ch.id))
}
- } getOrElse {
- Seq(None) // for default channel
- }
+ } getOrElse {
+ Seq(None) // for default channel
+ }
}
chanIdsToRemove.map { chId: Option[Int] =>
@@ -246,8 +246,7 @@ object App extends EitherLogging {
errStr =
if (chId.isDefined) {
s"Unable to initialize Event Store for the channel ID: ${chId.get}."
- }
- else {
+ } else {
s"Unable to initialize Event tore for the app ID: ${appDesc.app.id}."
}
error(errStr)
@@ -272,11 +271,11 @@ object App extends EitherLogging {
show(appName).right flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
if (channels.find(ch => ch.name == newChannel).isDefined) {
logAndFail(s"""Channel ${newChannel} already exists.
- |Unable to create new channel.""")
+ |Unable to create new channel.""".stripMargin)
} else if (!storage.Channel.isValidName(newChannel)) {
logAndFail(s"""Unable to create new channel.
|The channel name ${newChannel} is invalid.
- |${storage.Channel.nameConstraint}""")
+ |${storage.Channel.nameConstraint}""".stripMargin)
} else {
val channel = Channel(
@@ -299,7 +298,7 @@ object App extends EitherLogging {
Right(channel.copy(id = chanId))
} else {
errStr = s"""Unable to create new channel.
- |Failed to initalize Event Store."""
+ |Failed to initalize Event Store.""".stripMargin
error(errStr)
// reverted back the meta data
try {
@@ -307,17 +306,17 @@ object App extends EitherLogging {
Left(errStr)
} catch {
case e: Exception =>
- val nextErrStr = s"""
+ val nextErrStr = (s"""
|Failed to revert back the Channel meta-data change.
|The channel ${newChannel} CANNOT be used!
|Please run 'pio app channel-delete ${appName} ${newChannel}'""" +
- " to delete this channel!"
+ " to delete this channel!").stripMargin
logAndFail(errStr + nextErrStr)
}
}
} getOrElse {
logAndFail(s"""Unable to create new channel.
- |Failed to update Channel meta-data.""")
+ |Failed to update Channel meta-data.""".stripMargin)
}
}
}
@@ -329,33 +328,32 @@ object App extends EitherLogging {
def channelDelete(appName: String, deleteChannel: String): MaybeError = {
val chanStorage = storage.Storage.getMetaDataChannels
val events = storage.Storage.getLEvents()
- var errStr = ""
try {
show(appName).right.flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
val foundChannel = channels.find(ch => ch.name == deleteChannel)
- if (foundChannel.isEmpty) {
- logAndFail(s"""Unable to delete channel
- |Channel ${deleteChannel} doesn't exists.""")
- } else {
- val chId = foundChannel.get.id
- val dbRemoved = events.remove(appDesc.app.id, Some(chId))
- if (dbRemoved) {
- info(s"Removed Event Store for this channel: ${deleteChannel}")
- try {
- chanStorage.delete(chId)
- logAndSucceed(s"Deleted channel: ${deleteChannel}.")
- } catch {
- case e: Exception =>
- logAndFail(s"""Unable to delete channel.
- |Failed to update Channel meta-data.
- |The channel ${deleteChannel} CANNOT be used!
- |Please run 'pio app channel-delete ${appDesc.app.name} ${deleteChannel}'""" +
- " to delete this channel again!")
+ foundChannel match {
+ case None =>
+ logAndFail(s"""Unable to delete channel
+ |Channel ${deleteChannel} doesn't exists.""".stripMargin)
+ case Some(channel) =>
+ val dbRemoved = events.remove(appDesc.app.id, Some(channel.id))
+ if (dbRemoved) {
+ info(s"Removed Event Store for this channel: ${deleteChannel}")
+ try {
+ chanStorage.delete(channel.id)
+ logAndSucceed(s"Deleted channel: ${deleteChannel}.")
+ } catch {
+ case e: Exception =>
+ logAndFail((s"""Unable to delete channel.
+ |Failed to update Channel meta-data.
+ |The channel ${deleteChannel} CANNOT be used!
+ |Please run 'pio app channel-delete ${appDesc.app.name} ${deleteChannel}'""" +
+ " to delete this channel again!").stripMargin)
+ }
+ } else {
+ logAndFail(s"""Unable to delete channel.
+ |Error removing Event Store for this channel.""".stripMargin)
}
- } else {
- logAndFail(s"""Unable to delete channel.
- |Error removing Event Store for this channel.""")
- }
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
index 30c249b..ee8cd50 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -49,7 +49,7 @@ case class PioStatus(
sparkHome: String = "",
sparkVersion: String = "",
sparkMinVersion: String = "",
- warnings: Seq[String] = Seq())
+ warnings: Seq[String] = Nil)
object Management extends EitherLogging {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6789dbeb/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
index 4a72635..acd7598 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
@@ -49,12 +49,12 @@ case class ConsoleArgs(
dashboard: DashboardArgs = DashboardArgs(),
export: ExportArgs = ExportArgs(),
imprt: ImportArgs = ImportArgs(),
- commands: Seq[String] = Seq(),
+ commands: Seq[String] = Nil,
metricsParamsJsonPath: Option[String] = None,
paramsPath: String = "params",
engineInstanceId: Option[String] = None,
mainClass: Option[String] = None,
- driverPassThrough: Seq[String] = Seq(),
+ driverPassThrough: Seq[String] = Nil,
pioHome: Option[String] = None,
verbose: Boolean = false)
@@ -69,7 +69,7 @@ case class AppArgs(
case class AccessKeyArgs(
accessKey: String = "",
- events: Seq[String] = Seq())
+ events: Seq[String] = Nil)
case class EngineInfo(
engineId: String,
@@ -763,7 +763,7 @@ object Console extends Logging {
}
}
- def help(commands: Seq[String] = Seq()): String = {
+ def help(commands: Seq[String] = Nil): String = {
if (commands.isEmpty) {
mainHelp
} else {
[47/50] [abbrv] incubator-predictionio git commit: Update versions in
examples and install script
Posted by ch...@apache.org.
Update versions in examples and install script
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/b9a9e5e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/b9a9e5e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/b9a9e5e1
Branch: refs/heads/master
Commit: b9a9e5e13a12832e3868cc4272b706d1dff12c6c
Parents: 954f234
Author: Chan Lee <ch...@gmail.com>
Authored: Wed Sep 13 10:30:55 2017 -0700
Committer: Chan Lee <ch...@gmail.com>
Committed: Wed Sep 13 12:02:32 2017 -0700
----------------------------------------------------------------------
bin/install.sh | 6 +++---
build.sbt | 2 ++
docs/manual/data/versions.yml | 8 ++++----
examples/scala-parallel-classification/README.md | 2 +-
.../scala-parallel-classification/add-algorithm/build.sbt | 2 +-
.../reading-custom-properties/build.sbt | 2 +-
examples/scala-parallel-ecommercerecommendation/README.md | 2 +-
.../adjust-score/build.sbt | 2 +-
.../train-with-rate-event/build.sbt | 2 +-
examples/scala-parallel-recommendation/README.md | 2 +-
.../scala-parallel-recommendation/blacklist-items/build.sbt | 2 +-
.../customize-data-prep/build.sbt | 2 +-
.../customize-serving/build.sbt | 2 +-
.../reading-custom-events/build.sbt | 2 +-
.../train-with-view-event/build.sbt | 2 +-
examples/scala-parallel-similarproduct/README.md | 2 +-
.../multi-events-multi-algos/build.sbt | 2 +-
.../scala-parallel-similarproduct/recommended-user/build.sbt | 2 +-
.../return-item-properties/build.sbt | 2 +-
.../rid-user-set-event/build.sbt | 2 +-
.../train-with-rate-event/build.sbt | 2 +-
21 files changed, 27 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/bin/install.sh
----------------------------------------------------------------------
diff --git a/bin/install.sh b/bin/install.sh
index ca7fcac..8277b57 100755
--- a/bin/install.sh
+++ b/bin/install.sh
@@ -18,10 +18,10 @@
#
OS=`uname`
-SPARK_VERSION=1.6.3
+SPARK_VERSION=2.1.1
# Looks like support for Elasticsearch 2.0 will require 2.0 so deferring
-ELASTICSEARCH_VERSION=1.7.5
-HBASE_VERSION=1.2.2
+ELASTICSEARCH_VERSION=5.5.2
+HBASE_VERSION=1.2.6
POSTGRES_VERSION=42.0.0
MYSQL_VERSION=5.1.41
PIO_DIR=$HOME/PredictionIO
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 5a38e6f..a02bdd9 100644
--- a/build.sbt
+++ b/build.sbt
@@ -52,6 +52,8 @@ organization in ThisBuild := "org.apache.predictionio"
scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.11.8")
+scalaBinaryVersion in ThisBuild := binaryVersion(scalaVersion.value)
+
crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8")
scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/docs/manual/data/versions.yml
----------------------------------------------------------------------
diff --git a/docs/manual/data/versions.yml b/docs/manual/data/versions.yml
index 7cbccc8..3fba629 100644
--- a/docs/manual/data/versions.yml
+++ b/docs/manual/data/versions.yml
@@ -1,7 +1,7 @@
-pio: 0.11.0-incubating
-spark: 1.6.3
-spark_download_filename: spark-1.6.3-bin-hadoop2.6
-elasticsearch_download_filename: elasticsearch-1.7.6
+pio: 0.12.0-incubating
+spark: 2.1.1
+spark_download_filename: spark-2.1.1-bin-hadoop2.6
+elasticsearch_download_filename: elasticsearch-5.5.2
hbase_version: 1.2.6
hbase_basename: hbase-1.2.6
hbase_variant: bin
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-classification/README.md
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-classification/README.md b/examples/scala-parallel-classification/README.md
index 2ec7d22..6b0d137 100644
--- a/examples/scala-parallel-classification/README.md
+++ b/examples/scala-parallel-classification/README.md
@@ -15,6 +15,6 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-This is based on Classification Engine Template v0.11.0-incubating.
+This is based on Classification Engine Template v0.12.0-incubating.
Please refer to http://predictionio.incubator.apache.org/templates/classification/how-to/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-classification/add-algorithm/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-classification/add-algorithm/build.sbt b/examples/scala-parallel-classification/add-algorithm/build.sbt
index f5fb250..aef4495 100644
--- a/examples/scala-parallel-classification/add-algorithm/build.sbt
+++ b/examples/scala-parallel-classification/add-algorithm/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-classification"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-classification/reading-custom-properties/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-classification/reading-custom-properties/build.sbt b/examples/scala-parallel-classification/reading-custom-properties/build.sbt
index f5fb250..aef4495 100644
--- a/examples/scala-parallel-classification/reading-custom-properties/build.sbt
+++ b/examples/scala-parallel-classification/reading-custom-properties/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-classification"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-ecommercerecommendation/README.md
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-ecommercerecommendation/README.md b/examples/scala-parallel-ecommercerecommendation/README.md
index c750458..27a13ce 100644
--- a/examples/scala-parallel-ecommercerecommendation/README.md
+++ b/examples/scala-parallel-ecommercerecommendation/README.md
@@ -15,6 +15,6 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-This is based on E-Commerce Recommendation Template v0.11.0-incubating.
+This is based on E-Commerce Recommendation Template v0.12.0-incubating.
Please refer to http://predictionio.incubator.apache.org/templates/ecommercerecommendation/how-to/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt b/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt
index 85b21d6..f5dc7e5 100644
--- a/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt
+++ b/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-ecommercerecommendation"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt b/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt
index 85b21d6..f5dc7e5 100644
--- a/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt
+++ b/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-ecommercerecommendation"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-recommendation/README.md
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-recommendation/README.md b/examples/scala-parallel-recommendation/README.md
index 0be8090..4b0e2ca 100644
--- a/examples/scala-parallel-recommendation/README.md
+++ b/examples/scala-parallel-recommendation/README.md
@@ -15,6 +15,6 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-This is based on Recommendation Template v0.11.0-incubating.
+This is based on Recommendation Template v0.12.0-incubating.
Please refer to http://predictionio.incubator.apache.org/templates/recommendation/how-to/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-recommendation/blacklist-items/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-recommendation/blacklist-items/build.sbt b/examples/scala-parallel-recommendation/blacklist-items/build.sbt
index 3d25df1..1c38229 100644
--- a/examples/scala-parallel-recommendation/blacklist-items/build.sbt
+++ b/examples/scala-parallel-recommendation/blacklist-items/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-recommendation/customize-data-prep/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-recommendation/customize-data-prep/build.sbt b/examples/scala-parallel-recommendation/customize-data-prep/build.sbt
index 3d25df1..1c38229 100644
--- a/examples/scala-parallel-recommendation/customize-data-prep/build.sbt
+++ b/examples/scala-parallel-recommendation/customize-data-prep/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-recommendation/customize-serving/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-recommendation/customize-serving/build.sbt b/examples/scala-parallel-recommendation/customize-serving/build.sbt
index 3d25df1..1c38229 100644
--- a/examples/scala-parallel-recommendation/customize-serving/build.sbt
+++ b/examples/scala-parallel-recommendation/customize-serving/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-recommendation/reading-custom-events/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-recommendation/reading-custom-events/build.sbt b/examples/scala-parallel-recommendation/reading-custom-events/build.sbt
index 3d25df1..1c38229 100644
--- a/examples/scala-parallel-recommendation/reading-custom-events/build.sbt
+++ b/examples/scala-parallel-recommendation/reading-custom-events/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-recommendation/train-with-view-event/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-recommendation/train-with-view-event/build.sbt b/examples/scala-parallel-recommendation/train-with-view-event/build.sbt
index 3d25df1..1c38229 100644
--- a/examples/scala-parallel-recommendation/train-with-view-event/build.sbt
+++ b/examples/scala-parallel-recommendation/train-with-view-event/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-similarproduct/README.md
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/README.md b/examples/scala-parallel-similarproduct/README.md
index 0a56220..343afc6 100644
--- a/examples/scala-parallel-similarproduct/README.md
+++ b/examples/scala-parallel-similarproduct/README.md
@@ -15,6 +15,6 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-This is based on Similar Product Template v0.11.0-incubating.
+This is based on Similar Product Template v0.12.0-incubating.
Please refer to http://predictionio.incubator.apache.org/templates/similarproduct/how-to/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt b/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt
index 1daded6..9d7c345 100644
--- a/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt
+++ b/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-similarproduct"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-similarproduct/recommended-user/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/recommended-user/build.sbt b/examples/scala-parallel-similarproduct/recommended-user/build.sbt
index 7409ab6..a233b8b 100644
--- a/examples/scala-parallel-similarproduct/recommended-user/build.sbt
+++ b/examples/scala-parallel-similarproduct/recommended-user/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-recommendeduser"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-similarproduct/return-item-properties/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/return-item-properties/build.sbt b/examples/scala-parallel-similarproduct/return-item-properties/build.sbt
index 1daded6..9d7c345 100644
--- a/examples/scala-parallel-similarproduct/return-item-properties/build.sbt
+++ b/examples/scala-parallel-similarproduct/return-item-properties/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-similarproduct"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt b/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt
index 1daded6..9d7c345 100644
--- a/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt
+++ b/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-similarproduct"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b9a9e5e1/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt
----------------------------------------------------------------------
diff --git a/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt b/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt
index 1daded6..9d7c345 100644
--- a/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt
+++ b/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt
@@ -20,5 +20,5 @@ name := "template-scala-parallel-similarproduct"
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.0-incubating" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
[19/50] [abbrv] incubator-predictionio git commit: [PIO-102] Fix ES5
scroll order
Posted by ch...@apache.org.
[PIO-102] Fix ES5 scroll order
Closes #406
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/5813b80d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/5813b80d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/5813b80d
Branch: refs/heads/master
Commit: 5813b80dc79c7a727bbcc73329135c385b71900d
Parents: a813021
Author: Mars Hall <ma...@heroku.com>
Authored: Fri Jul 28 11:52:16 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Fri Jul 28 11:52:16 2017 -0700
----------------------------------------------------------------------
.../apache/predictionio/data/storage/elasticsearch/ESUtils.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/5813b80d/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
index 2bf18ef..3f7b058 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
@@ -150,7 +150,7 @@ object ESUtils {
val responseJValue = parse(EntityUtils.toString(response.getEntity))
scroll((responseJValue \ "_scroll_id").extract[String],
(responseJValue \ "hits" \ "hits").extract[Seq[JValue]],
- hits.map(h => (h \ "_source").extract[JValue]) ++ results)
+ results ++ hits.map(h => (h \ "_source").extract[JValue]))
}
}
[13/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/data/lr_data.txt
----------------------------------------------------------------------
diff --git a/examples/experimental/data/lr_data.txt b/examples/experimental/data/lr_data.txt
deleted file mode 100644
index d4df063..0000000
--- a/examples/experimental/data/lr_data.txt
+++ /dev/null
@@ -1,1000 +0,0 @@
-1 2.1419053154730548 1.919407948982788 0.0501333631091041 -0.10699028639933772 1.2809776380727795 1.6846227956326554 0.18277859260127316 -0.39664340267804343 0.8090554869291249 2.48621339239065
-1 1.8023071496873626 0.8784870753345065 2.4105062239438624 0.3597672177864262 -0.20964445925329134 1.3537576978720287 0.5096503508009924 1.5507215382743629 -0.20355100196508347 1.3210160806416416
-1 2.5511476388671834 1.438530286247105 1.481598060824539 2.519631078968068 0.7231682708126751 0.9160610215051366 2.255833005788796 0.6747272061334229 0.8267096669389163 -0.8585851445864527
-1 2.4238069456328435 -0.3637260240750231 -0.964666098753878 0.08140515606581078 -1.5488873933848062 -0.6309606578419305 0.8779952253801084 2.289159071801577 0.7308611443440066 1.257491408509089
-1 0.6800856239954673 -0.7684998592513064 0.5165496871407542 0.4900095346106301 2.116673376966199 0.9590527984827171 -0.10767151692007948 2.8623214176471947 2.1457411377091526 -0.05867720489309214
-1 2.0725991339400673 -0.9317441520296659 1.30102521611535 1.2475231582804265 2.4061568492490872 -0.5202207203569256 1.2709294126920896 1.5612492848137771 0.4701704219631393 1.5390221914988276
-1 3.2123402141787243 0.36706643122715576 -0.8831759122084633 1.3865659853763344 1.3258292709064945 0.09869568049999977 0.9973196910923824 0.5260407450146751 0.4520218452340974 0.9808998515280365
-1 2.6468163882596327 -0.10706259221579106 1.5938103926672538 0.8443353789148835 1.6632872929286855 2.2267933606886228 1.8839698437730905 1.2217245467021294 1.9197020859698617 0.2606241814111323
-1 1.803517749531419 0.7460582552369641 0.23616113949394446 -0.8645567427274516 -0.861306200027518 0.423400118883695 0.5910061937877524 1.2484609376165419 0.5190870450972256 1.4462120573539101
-1 0.5534111111196087 1.0456386878650537 1.704566327313564 0.7281759816328417 1.0807487791523882 2.2590964696340183 1.7635098382407333 2.7220810801509723 1.1459500540537249 0.005336987537813309
-1 1.2007496259633872 1.8962364439355677 2.5117192131332224 -0.40347372807487814 -0.9069696484274985 2.3685654487373133 0.44032696763461554 1.7446081536741977 2.5736655956810672 2.128043441818191
-1 0.8079184133027463 -1.2544936618345086 1.439851862908128 1.6568003265998676 0.2550498385706287 2.1994753269490133 2.7797467521986703 1.0674041520757056 2.2950640220107115 0.4173234715497547
-1 1.7688682382458407 1.4176645501737688 0.5309077640093247 1.4141481732625842 1.663022727536151 1.8671946375362718 1.2967008778056806 1.3215230565153893 3.2242953580982188 1.8358482078498959
-1 -0.1933022979733765 1.1188051459900596 1.5580410346433533 -0.9527104650970353 2.4960553383489517 0.2374178113187807 1.8951776489120973 0.817329097076558 1.9297634639960395 0.5625196401726915
-1 0.8950890609697704 0.3885617561119906 1.3527646644845603 -0.14451661079866773 0.34616820106951784 3.677097108514281 1.1513217164424643 2.8470372001182738 1.440743314981174 1.8773090852445982
-1 1.946980694388772 0.3002263539854614 -1.315207227451069 1.0948002011749645 1.1920371028231238 -0.008130832288609113 -1.150717205632501 2.6170416083849215 1.5473509656354905 2.6230096333098776
-1 1.369669298870147 2.2240526315272633 1.8751209163514155 0.7099955723660032 1.4333345396190893 2.0069743967645715 2.783008145523796 2.356870316505785 1.4459302415658664 2.3915127940536753
-1 1.0329554152547427 0.19817512014940342 0.9828173667832262 -0.3164854365297216 0.9721814447840595 2.9719833390831583 2.3758681039407463 -0.2706898498985282 1.2920337802284907 2.533319271731563
-1 1.1046204258897305 -0.31316036717589113 2.779996494431689 1.3952547694086233 0.49953716767570155 -1.0407393926238933 2.0869289165797924 -0.04084913117769684 2.9616582572418197 1.9258632212977318
-1 2.361656934659277 3.8896525506477344 0.5089863292545287 0.28980141682319804 2.570466720662197 0.15759150270048905 0.6680692313979322 -0.698847669879108 0.4688584882078929 -1.5875629832762232
-1 1.301564524776174 -0.15280528962364026 -0.7133285086762593 1.081319758035075 -0.3278612176303164 1.6965862080356764 -0.28767133135763223 2.2509059068665724 1.0125522002674598 1.6566974914450203
-1 -0.3213530059013969 1.8149172295041944 1.6110409277400992 1.1234808948785417 1.3884025750196511 0.41787276194289835 1.4334356888417783 0.20395689549800888 1.0639952991231423 0.25788892433087685
-1 2.1806635961066307 1.9198186083780135 2.238005178835123 0.9291144984960873 0.4341039397491093 2.050821228244721 1.9441165305261188 0.30883909322226666 1.8859638093504212 -1.533371339542391
-1 1.4163203752064484 1.4062903984061705 1.8418616457792907 0.6519263935739821 2.0703545150299583 0.7652230912847241 1.1557263986072353 1.6683095785190067 1.3685121432402299 1.0970993371965074
-1 -0.23885375176985146 0.7346703244086044 0.39686127458413645 0.8536167113915564 2.8821103658250253 2.843586967989016 0.2256284103968883 0.8466499260789964 1.1372088070346282 0.0880674005359322
-1 1.190682102191321 1.7232172113039872 0.5636637342794258 0.8190845829178903 1.803778929309528 2.386253140767585 0.651507090146642 2.053713849719438 1.049889279545437 2.367448527229836
-1 1.2667391586127408 1.0272601665986936 0.1694838905810353 1.3980698432838456 1.2347363543406824 1.519978239538835 0.7755635065536938 1.9518789476720877 0.8463891970929239 -0.1594658182609312
-1 1.9177143967118988 0.1062210539075672 1.0776111251281053 1.969732837479783 0.5806581670596382 0.9622645870604398 0.5267699759271061 0.14462924425226986 3.205183137564584 0.3349768610796714
-1 2.8022977941941876 1.7233623251887376 1.8343656581164236 2.5078868235362135 2.8732773429688496 1.175657348763883 1.8230498418068863 -0.06420099579179217 -0.31850161026000223 1.3953402446037735
-1 1.293815946466546 1.9082454404595959 1.0390424276302468 1.4123446397119441 0.14272371474828127 0.5954644427489499 1.9311182993772318 1.4425836945233532 0.23593915711070867 -0.0046799615367818514
-1 2.1489058966224226 1.5823735498702165 0.47984538863958215 0.05725411130294378 -0.19205537448285037 2.578016006340281 2.635623602110286 1.9829002135878433 0.19799288106884738 1.7028918814014005
-1 1.5672862680104924 -0.0987393491518127 0.7244061201774454 -0.41182579172916434 1.1979110917942835 -0.12481753033835274 0.5630131395041615 1.385537735117697 -0.8919101455344216 2.7424648070251116
-1 0.6879772771184975 1.582111812261079 0.3665634721723976 0.850798208790375 0.9426300131823666 1.983603842699607 0.8130990941989288 -1.0826899070777283 0.7979163057567745 -0.12841040130621417
-1 0.49726755658797983 1.1012109678729847 0.27184530927569217 0.09590187123183869 2.7114680848906723 1.0712539490680686 0.4661357697833658 1.1666136730805596 1.0060435328852553 1.3752864302671253
-1 1.5705074035386362 2.5388314004618415 3.705325086899449 1.7253747699098896 0.2905920924621258 2.2062201954483274 1.7686772759307146 -0.14389818761776474 1.317117811881067 1.960659458484061
-1 -0.6097266693243066 1.5050792404611277 1.5597531261282835 1.801921952517151 1.021637610172004 1.0147308245966982 0.496200008835183 1.2470065877402576 1.09033470655824 2.154244343371553
-1 1.7311626690342417 -0.7981106861881657 1.576306673263288 2.0139307462486293 0.9669340713114077 2.6079849454993758 2.4417756902619443 0.97773788498047 -0.02280274021786477 1.9625031913007136
-1 0.034608060780454086 0.43324370378601906 0.6464567365972307 0.16942820411876358 2.773634414356671 0.950387120399953 0.20399015246948005 2.45383876915324 1.4728192154140967 0.27665303590986445
-1 0.669423341908155 2.753528514524716 -0.3114457433066151 0.42623362468295967 0.17585723777040074 0.3896466198418058 3.382230016050147 0.5628980580934769 0.1855399231085304 -1.0368812374682252
-1 1.1578929223859837 -0.9772673038070927 1.628472811304047 0.1706064825334408 -0.4368078914563116 1.3238749660151412 -0.6328206376503045 -0.1268798336415804 1.4614917163766068 0.05098215234403425
-1 1.9810025566400666 1.076214892921874 -1.1668914854936587 1.6219892570599912 0.5991126181156119 1.0668387700181805 -0.38561466584746307 -0.3346008538706646 -0.13693208851002447 1.082271823637847
-1 1.6753996221697711 -0.2204800911406224 1.3643600908733924 1.3667965239511641 1.4202494777278367 0.1990171616310349 1.3814657607888683 1.0156848718344853 1.1547747341458854 1.919747223811457
-1 2.306325804101286 2.013331566156439 1.1223877708770225 -0.06481662603037197 1.7942868367810174 0.7587370182842376 0.8698939230717255 0.37170451929485726 1.353135265304875 -0.013085996169272862
-1 0.20271462066175472 1.8670116701629946 0.1618067461065149 -0.2974653145373134 2.0274885311314446 1.7489571027636028 2.991328245656333 2.3823300780216257 2.078511519846326 1.97782037580114
-1 2.2596721244733233 1.006588878797566 2.2453074888557705 0.4245510909203909 1.557587461354759 1.7728855159117356 1.0648265192392103 1.1365923061997036 0.5379050122382909 0.9997617294083609
-1 2.414464891572643 0.30469754105126257 2.1935238570960616 2.587308021245376 1.5756963983924648 1.9319407933274975 0.8074477639415376 1.7357619185236388 0.23815230672958865 -0.4761137753554259
-1 1.3855245092290591 1.955100157523304 1.4341819377958671 0.28696565179644584 1.7291061523286055 1.714048489489178 1.164672495926134 1.6545959369641716 1.9496841789853843 2.5374349926535062
-1 1.1158271727931894 2.213425162173939 1.36638012222097 -0.023757883337165886 2.406876786398608 1.1126742159637397 0.12318438504039564 2.8153485847571273 0.15506376286728374 0.33355971489136393
-1 1.7297171728443748 0.6719390218027237 1.3753247894650051 -0.10182607341800742 1.7453755134851177 1.0960805604241037 0.40205225932790567 1.6103118877057256 -1.03955805358224 -0.3213966754338211
-1 1.316257046547979 1.2853238426515166 2.0480481778475728 0.6602539720919305 0.7379613133231193 2.0626091656565495 1.4509651703701687 1.864003948893211 2.2982171285406796 0.9359019132591221
-1 1.6046620370312947 2.321499271109006 2.2161407602345786 0.5862066390480085 -1.06591519642831 0.4488708706540525 0.9764088582932869 -0.17539686817265143 1.0261570987217379 1.8924236336247766
-1 -0.013917852015644883 0.4901030850643481 0.574360829130456 0.08844371614484736 1.3233068279136773 0.7589759244353294 1.7201737182853447 0.517426440952053 2.7274693051068777 0.036397493927961544
-1 1.2232096749473036 1.4768480172452538 1.5300887552091489 1.8810354040615782 -0.6436862913845212 1.5878631039716906 0.09394891272528805 1.7766036014727926 -0.08618397395873112 1.5926757324414604
-1 -0.006190798924250895 -1.1803586949394225 2.237721401521945 0.7324966516613158 1.4038442669165114 -0.06019103023815764 -0.7655029652453154 -0.3991986433215591 2.3296187529650685 0.38065062537135896
-1 1.0869918851572522 -0.37412852726006984 0.27965894114884915 -0.0733849426330444 0.7458288899809582 0.38504406064556884 1.3823407462352355 1.0530056181901168 -0.10908828320629294 -0.3163748213825457
-1 2.0800232080218937 0.6793681518120379 1.0126904247021766 0.5099365686965533 1.4765728601491988 -0.90922098444035 0.01578092821031385 2.531202299543557 1.3694116442965245 0.03526109196146243
-1 2.52004533036052 -0.11716335755537322 2.043801269881338 -0.4889959907470973 1.3717334116816158 -0.5907796618760839 2.9080140714861864 2.3969176626246114 0.9445325920064912 0.9620736405334235
-1 0.8261430232725533 0.9003472941846893 1.2648199316806048 1.3110765897825498 0.9484044458467761 1.5971370020069537 1.89838012162931 0.5844972943740565 2.1114035373528974 2.8066708339226407
-1 1.7131825192258492 0.5164803724034563 1.3400031460569826 1.159025272879641 -0.6475319792487726 0.7895415906096561 0.3591049378091684 0.3507368152114154 0.46463582975963413 1.2784917703092404
-1 0.9196047831077019 0.6917912743533342 1.7505158395265692 2.275307243506136 2.9871554281485713 0.584299496238456 1.2741949422522685 0.42838234246585094 2.613957509033075 1.479280190769243
-1 0.6865489083893408 1.6888181847006614 1.5612615114298305 0.28075030293939784 0.7611637101018122 0.17543992215891036 0.8532136322118986 1.6171101997247541 2.487562859731773 2.1695780390240165
-1 3.746488178488735 0.5902211931946351 1.4116785188193897 -0.302213259977852 1.3900348431280398 1.8058092139513118 1.9063920023065686 -0.6748417828946516 1.2856680423450677 1.4181322176013937
-1 1.3957855809267268 0.6788775338735233 1.2694449274462256 0.7739220722195589 1.6662774494836934 0.2263815064326532 0.3746198256735065 0.6981525121209534 0.6659194682736781 2.34383566814983
-1 0.3820962920141968 -0.11474969137094182 1.4456430767826618 1.7541264342573286 0.5841263905944027 0.3310478153678522 0.1361074962599954 2.1517668203954323 2.1312973802189523 0.08816171787088545
-1 0.44857483955792765 -1.3332507048491813 0.5685902212376108 1.1213432607484823 2.634120632788485 0.7837711869120604 1.0078687896423884 1.8982652887205418 1.1818816137394528 1.2876714951624808
-1 1.1951146419526084 0.9947742549449248 0.19840725400812698 2.48569644222758 1.7391898607628944 2.40036741337463 2.0600530189294144 -0.5340832975220873 2.0467391216154094 1.1908285513553203
-1 0.9918935330929904 -0.3542942677260328 1.3105513869382395 1.1904643448960697 -0.3602658438636872 0.6816024636806379 1.9768303812038046 0.4000132856795251 0.09352911692893684 1.9754791705404877
-1 1.0081698742896188 0.8916746417259931 1.496601632133103 1.8174757593692714 0.49297596177715564 1.828839820849067 1.662627028300793 1.2253219256823615 -1.6200329115107013 1.051770724619957
-1 0.9867026242209636 2.0915066394830326 0.2608828095090572 1.5275154403994393 0.3157310747415396 -0.7181525036523673 1.281115387917441 2.286539214837881 0.5653973688805878 3.0047565660570132
-1 0.9224469399191068 1.2533868053906783 -0.10077556308999824 0.06127395021274762 -0.18013801007271568 0.8043572428627129 -0.3236336059948026 1.6130489732175104 3.313472221318618 -0.15122165909659913
-1 0.7882345197971014 1.141304212890955 0.9030550623054504 2.543084656196279 0.7468302223968317 1.6832418500477586 0.10324287869065907 0.8952909318554702 1.7968146536867757 1.8337447891715968
-1 1.5801885793428398 2.438564562880532 1.346652611597816 2.013682644266395 0.5423884037920474 1.5509096942566918 -0.09721979565291483 0.7802050454421068 -0.07405588910002847 1.1020403166091144
-1 0.03083257777543913 0.09561020933135189 2.783828684436811 0.6702011711663662 1.1177709598763554 1.507733845629784 0.7190681946142053 0.4421675532332505 2.0062047937031338 1.3078544626787887
-1 0.029946310071738202 2.9974008035637247 1.2712685297793174 1.564287715942167 0.9318120646963208 1.9611220391387494 0.6955370789941844 2.8474941997466665 1.7216550057775473 1.033229285227095
-1 1.7919476706914224 2.674070943673579 1.0707436458201804 -1.2652465769212773 0.13786669485292458 -0.9521873641153344 -0.5112273884476357 1.8041566655420045 2.0489287678822823 1.4526766050251194
-1 2.1567394248692624 0.2787475011337476 1.2693515582998967 2.141920061908346 -0.311063434715769 2.7871358520284515 0.4011362416354143 1.2240722802790835 2.0224267357566696 0.6055884380482317
-1 1.2810578825169523 -0.06149076783837382 -0.3631214532063931 1.8242040060835376 0.936708636871513 0.9599645524867305 -0.2864664075189678 1.4575636141356014 -0.6521604857506678 1.4782024605158144
-1 1.922007864215502 0.41092515579085087 1.3614694131826193 1.2516141141035275 1.1032104604396404 1.5618738178080496 0.22277705609915832 -0.10552941002887595 0.8187789394182741 1.1899147160759034
-1 -1.101159111435701 2.0868811582857676 2.061754901850132 0.831389858205579 1.1022205058106118 -0.15327367461990105 3.263172683870654 -0.13185404063281925 0.4215198415563227 0.5983645772645423
-1 0.9017414538285525 1.5815719854072032 -0.33621575096987555 0.7353127316624433 2.000881249246564 1.752079037914068 2.188342812418916 2.464770657128536 1.9873120348231552 2.5280681270799197
-1 0.36229490936502484 0.9764447193507352 0.5513927408959507 1.2450834166369436 1.0347591040069144 0.23319917869834939 2.9368656872660264 1.3867291773435497 2.0279815142744324 1.3025138236731233
-1 0.12338005279277287 -0.11881556712737162 1.0293241194113785 2.053803566510112 1.694932390223226 1.2851644900727108 -0.09123042470171838 1.4542526750729492 0.9314422039244139 1.484525799738803
-1 2.2791038050359416 0.13652686573061323 0.34425341235820794 0.5134789845294401 1.199131994695721 1.285766903846671 1.6396476063943415 0.37354865288496775 -0.9325874103952065 1.9432993173271385
-1 0.3187247126988978 -0.23565755255952947 1.4653008405179144 1.4073930754043715 1.86867235923796 -0.8601040662125556 0.17314198154775828 1.359209951341465 1.8780560671833557 1.0497896254122507
-1 -0.35095212337482606 2.1382594819736456 0.21582557882234288 1.563987660659988 0.8742557302587846 2.7376537243676307 1.1089682445267717 0.3906567030119056 0.90272045105723 0.3199475930277361
-1 -1.0755666969659972 2.587500753780116 0.43523091172933415 1.9715380667335656 -1.206591074948113 2.3082117218149953 2.9003512906773183 1.8894617822889117 0.2612428397679113 2.3034517860165904
-1 1.2752641746970284 -0.8368104009920136 0.03573979915049008 0.9337645939367554 1.8180936927791564 0.35607066313035163 0.9553794086170463 2.3774664468818862 0.27151841486690464 0.5861688049602704
-1 1.3242463950740633 1.5079874960068127 2.2093340505083026 1.2611978264745287 1.7161846809846164 -0.49880331209390905 2.2386520558115137 1.259321190419847 1.3434715137362212 2.044909528652566
-1 0.8795598947051465 1.8282710612070696 0.8010144751459073 0.6664561865521288 0.4104626238753195 0.23255356821870798 0.33916496869925716 -0.2708146821069548 0.9241466333878707 -0.450452229744047
-1 1.9192448235188513 0.4969214523219533 2.4011260745046066 1.1346909629811026 -0.6596351603517379 -0.5351409933958904 0.02441943738258512 2.288141877404522 1.2367780341721122 1.584102117316426
-1 0.9682490849657925 -1.8650300168768377 0.8811925017526988 1.1594483122156354 1.121203677520715 0.9099984493527551 0.08826662255652562 -0.7539889420899628 0.4595729579317809 -0.7165782835963082
-1 1.5995281560764565 0.20521558652985616 -1.1164794717138746 1.5074668507140967 0.7877952768927691 0.902667397635835 1.6081861816054732 1.3133186016363785 1.5296162271430345 1.0712740040810271
-1 0.42211731340992986 0.502442828209289 0.3565737103297629 0.4478456815580649 1.617182070323055 0.9823042873485613 1.0704168281976632 -0.26776498356102985 1.8711459938723063 0.791693835933734
-1 0.23896637909254625 0.6184009702378752 1.484473242669571 -2.0960256478350034 1.007509277044258 1.4880525091303394 0.14825818901395527 2.918617492389175 2.7162682081607343 1.2852769131414254
-1 0.09951845043296148 0.10778080557671554 1.6153805572528395 0.21496629935184874 0.5695206599630613 0.5995686906470605 1.6226444344121718 1.400956890784598 2.5804792645155237 1.8818183326984712
-1 1.5660653841435699 1.9424448683907583 -0.5018032946330131 0.38813943551967744 0.21678795998247846 0.4592981799067166 0.3853775631077989 0.782922855791653 2.9697907962454226 2.0478747128589188
-1 0.5992085726320009 0.8326763829762222 1.0404230260991942 1.3571653199047529 0.05351664648320875 -1.8860610207228041 -0.5191719995314692 1.4226132032544871 1.6669779033604124 0.3253081253110943
-1 1.5903828533545434 1.894569333674546 1.5910544740636994 -1.6611392075582438 0.23842067636563624 -0.5406681576023691 1.7385589161163928 0.08969602776306584 1.4276561463432735 2.1566164427616634
-1 1.1913811808857528 0.32434695668325997 1.323498708189486 1.3596937187302878 3.4642496063989223 1.2876491657559253 -0.6543683402478666 1.4762502189363769 1.7353590098925795 2.8134629202660317
-1 3.123286693375267 1.877368736310955 0.9503145430714942 0.5342686470311402 0.3451961663217381 0.23995547380392213 0.5196925578399603 1.3087329089934692 0.5609549451755507 2.0018380155694433
-1 -0.70471754448335 0.396960196596961 2.8076920787881408 1.0486680479609312 0.1272088037522776 0.46477225522402743 1.0400518017377827 1.724354900707523 0.5172234824476354 0.70073364273413
-1 -0.04890176228714482 1.183623201015611 0.31679837772569197 2.442803942979677 2.475613952046278 1.316874640917748 2.1326668609632957 -1.1984022921949467 1.6326265827096553 0.13549684503148585
-1 1.532730344901386 1.8862673099243719 0.8433953501998975 0.9617349215859397 0.9632178266458564 1.7656392455188015 0.6166388141868028 0.36673723822668447 1.6148100615636092 1.9120508667715108
-1 1.8531415713908175 1.9856258806463458 0.8742545608077308 0.01891740612207793 0.754430421572012 1.2629533382356322 2.5668913595968625 0.7074626529557771 1.471180058040478 0.14210105766798764
-1 0.2946588114247314 1.7385325023150382 2.05805803890677 1.1285587768294627 0.30443899971020716 0.17710198470084348 -0.5876955744308521 1.6684452883987464 0.7429316176330647 0.24223269345723197
-1 0.12828383509135766 2.8251621371579123 -0.8683350630211126 1.3881503321455106 -0.9269673097143274 1.1340435175521124 1.1482061370168226 0.9886836766952749 1.3639211879675324 2.221424872356976
-1 1.6230819590031813 2.1140726634236273 0.8803195980146348 0.6957671564440406 1.3391648515238626 3.3118192086623672 1.206763244141946 0.5724427229085818 2.3692467877986934 1.2731917884083277
-1 0.6095837137279339 2.0886462170941087 1.5293277948541921 0.875698342933093 0.9739071638488416 -0.6284005601740021 0.7080909588024915 1.2483475820206364 0.39878604428574227 0.45167768471833614
-1 0.6622065044914254 0.7302732598978321 1.5839711558395906 0.33559568645900273 1.3094508963156517 1.5256964735790022 -0.2606881050391294 -0.13646086393521872 0.858395568393544 0.7983659548572369
-1 1.6030491170288057 0.8411660994073609 2.2968025114870225 0.7039288437264786 2.8125132767337133 0.23511452019598467 1.1415093151481583 -0.5416578453683565 2.121640334408583 -0.29666850192733474
-1 2.0779652161151883 1.0668503227493862 -0.3461938034511103 -1.9467096604673708 -0.4997902436835773 0.3419044702794434 0.8098524987621489 0.8131208951963917 1.3237950963836287 1.0429693266336961
-1 0.37001171609371697 0.29180348786692334 -0.2507809978364861 1.152821888667346 3.0890087304413267 1.215489406549123 1.199447470435283 0.789305354976556 0.8365245923088752 0.9787024262828808
-1 0.9296046114728362 2.19739063739452 1.533572358281578 0.7759925327491899 1.557482584766074 1.7151021392829757 0.9544359521103486 0.20077841759520276 1.59524901629763 2.175430873131662
-1 0.8112131582336873 0.2864940430793351 0.5833958780431041 1.7741485867050852 0.7779977372833543 1.8236769123328878 1.9278891617195901 -1.0188957672300982 0.9197794797358201 0.045052296436480455
-1 1.3702354298117274 0.5815346064645623 -0.04109583670633299 2.5064872968829004 1.206757887015013 0.2506549572813025 0.655306538898329 -0.3438030831151808 0.36458112520078056 0.8710435445702591
-1 1.4561762683494108 0.9681359328856552 3.136045420267423 0.7520560598452287 1.6528697058481434 0.9607920473099414 0.7156379077840067 1.857016542269911 -0.16277187766324142 0.4874157744630184
-1 1.2664980583047298 0.4023544599875911 0.9080313985150303 0.6549364577494126 2.738329489381062 2.3768996789882744 1.3393128915299277 -1.0430311123744418 0.8323494096430804 -0.12738742588819885
-1 0.8365391310807251 2.2822870725882503 2.6266615690102215 0.004265515881109128 2.4879345431323623 0.4875299849317022 1.351118317094851 1.245328886439785 0.8575534087593427 0.669435902035294
-1 0.8058511262644885 0.7473099050414014 2.303189816277799 1.2225351585963724 1.8247316651754097 -0.30810342366775534 0.2821704820687452 -1.6099991877186302 0.8406234201201898 2.0583805330826985
-1 2.250164789914201 1.7436544269774978 2.947667398091067 1.4771471077132423 -1.586188610201127 2.320910876555482 1.636258094383067 1.2987326716659215 -1.311058489828028 -0.011700890501986194
-1 0.8080250762510234 1.6440873832130936 0.8879459460961949 1.2082440017762488 -0.3984868670511643 -1.6750959916314896 0.9349087046999264 0.7232463907082566 2.2386173679423806 -0.017579999213251485
-1 1.0323998857804233 -0.7718677431568479 1.776325436331275 0.5932669960371175 1.7054720461060777 1.709001306281528 2.088236771173788 -0.13891858312535765 2.4540464522669634 2.581504187930639
-1 -0.36589663467243794 0.9800989499410697 1.512657907848574 2.481982348891716 1.879063921040467 1.6783314697156686 2.519822194339233 1.5139378983098026 1.4765499639533166 -0.4586543768759259
-1 1.031519656541507 0.37677631561513636 1.215439603971527 -0.8333793025092529 1.2297449965589116 0.7309661122339723 0.2233308234176088 1.8978096741161727 1.0017178523256016 1.540799199113878
-1 0.37535440891823324 1.05838458440246 1.7478919610180488 1.4358567778260587 2.634621031491021 2.6733943020176536 1.4038023921761382 2.09456237109269 0.18751380927669214 0.9030253353081665
-1 0.6050644162204089 0.42475868702885367 0.67729642342563 0.9159762799821485 0.9966211703282338 1.0325406378266162 -0.31600956837305927 1.1275195620810772 0.7550807758634188 2.0556587502944152
-1 0.9639628237078233 1.6612996949785008 0.15018611313458818 3.079012778712338 1.6765505664424296 -0.3164200745592767 1.180094372490766 0.16048718182365862 2.6754833932699764 0.2861554471536204
-1 -0.4733123063374025 2.215557819873761 1.4809169546161616 0.5331014736871407 0.509471219211528 -0.5366908461365221 2.5757870803346328 1.3082491695854135 1.3064213366309576 0.9305958816930349
-1 3.0207863567912003 0.23781737522480972 0.07878478120317567 1.6302281378682424 0.5980775385393649 1.5928976343724883 0.3212142395168056 1.7151012207401586 1.593816382695755 0.7481118256003316
-1 -0.5298380895168147 -0.34947847130115894 1.259810473989246 1.907798036285846 0.35944121815361163 0.6444888816334708 0.34377708875002244 0.6836686767703974 1.2932110945792579 -0.458790316071632
-1 1.8401629428690227 2.259471445176863 -0.3223229794980764 0.7728238347557039 1.5724556976510322 1.3274646917002721 1.6717333483877963 0.03745904530831912 2.6550649930379056 0.9705596819145808
-1 0.12431297464461755 1.7563279244667416 0.7774986621540451 0.5111136337905993 0.6433978537639469 1.8971862751406254 0.45959793718271824 1.781102107071228 1.4062626338777793 0.6234780410061468
-1 0.8407772366817298 0.35964705320370294 -0.9623019831100632 0.44149536693473657 2.074342161562674 0.9904199365414913 3.2137011456900098 1.0337076328449122 2.0693337269664083 1.8277506449533987
-1 1.0113056814830639 0.9851992899356764 0.873659978134487 1.0421853488103219 2.299837087915077 0.8071982744117732 -0.1096427502124051 2.5599638730556995 2.3458120257795656 1.9104294240298325
-1 -0.2652413955956079 0.2771478177147122 -1.7578972328231406 0.5091791920398325 1.3694768197526315 0.5806835043255031 -0.0948278795711135 3.822899721567823 0.5484905756054144 -0.25075975842777454
-1 0.6859095316452635 0.791069272223955 1.2193553385123195 0.7291514560030636 1.3876944292574216 0.8892463484292987 3.4273502454413576 0.6580296103521155 0.3238972925695067 -0.6496800158558074
-1 -1.5436851049150522 1.956099227374563 0.2779057405377705 0.7339456639197723 0.014024861431684466 2.6630936618511405 0.7161890905680435 0.5077767425517368 1.3259571967911001 0.9137278907925384
-1 -0.292961767713223 1.3071340106236198 -0.7017668375142168 1.2860358231830809 -0.8122076288210658 1.7211614223707081 1.8304680327555625 0.16021436599026517 0.19612682942548998 1.2082198804992264
-1 1.5187520786413158 0.1828654866775874 0.7328431724966722 1.7953629646772824 0.8216669452081463 -0.4014319711127199 0.23334012012093153 1.534537449937785 1.3889014942993092 -0.8511049828025341
-1 0.8451858363611996 1.3418063089585763 -0.8238999092902703 -1.575942571644518 2.0750484405729095 2.033997248128906 1.4449221159961598 2.0253497341487448 2.2283973766958023 2.404323890979427
-1 1.6107433076928133 0.5404780687423208 0.7937155331805563 -0.6077722620726684 0.21332376555661758 -0.9993545668337882 0.31523750335957845 0.5473005319402997 0.960730821903916 -0.28012631768751084
-1 1.9389616507358387 1.9532576203532324 1.2153193637879869 -1.4069714611803268 0.4662801445447652 -0.6193751496277011 -0.028999422131398056 1.3038353983411688 1.4946684162238129 -0.7409848880778342
-1 0.9021404373434705 1.5851981284549943 0.6057610277009148 1.1112421784262574 1.413214054275196 1.9417673251914613 1.634690668060366 -0.08301380649683576 2.1711500689414116 2.99282324374365
-1 0.1637260233089869 0.49637480750763263 -0.5285944959659445 1.5681001289396956 1.6803958442936107 1.2246294425310562 2.5669221884551776 0.7567621149423418 1.5037234063128802 0.3463214960951032
-1 1.5723472760593176 0.6432239887651015 1.804758599642208 1.2176050861917662 1.8717138471483157 4.077916319312581 1.5133550052844793 1.3823856879297753 2.6113216067389695 -1.1093237177115047
-1 0.8602744779765249 2.178619602525301 2.453544172271271 1.0510379811276036 1.8409684994496875 0.11803069280172118 0.3230760986621918 2.259943083391159 0.6024489055423363 1.1990484290135006
-1 1.649184578143986 1.616265278882509 2.2742015008761607 2.626169250389406 -1.1492939072912116 1.0408825980561895 0.4369989721349081 0.9034290059197084 -0.11385932074779648 1.0982078408810698
-1 0.6341310783502718 -0.9708605273806881 -0.017201345919524602 0.8926037502408949 0.22822364223265212 0.9096851395074563 2.0473818885200648 -0.7848615761262032 1.4441059896043467 -0.24922705201528594
-1 1.4520344107406407 1.2639986753730716 -0.8513007095320302 1.6293092619132934 0.7394579998929112 1.3445648999777857 1.5178679268046242 0.9933053628903701 -0.9336323582033459 -1.6920287783811307
-1 -0.584837407411567 0.9604177163540187 -0.003828672372695019 0.1731711935522725 3.512170380159825 0.4926659491064572 1.1587769448255618 0.6600987191801231 0.9926496119226857 1.9870269736899853
-1 0.40697221517240734 0.7915676379059069 1.4331616842644888 1.6198603975182355 1.6417243704332136 1.6270560025018783 1.6799759614717393 1.700588227134973 1.8464436799312134 -0.9250687955521861
-1 0.04736288349237683 1.5587027295355322 0.12163352594242882 1.124943757807633 0.2850023846865297 -0.07621319541134719 0.6373292813835088 2.5571634870370934 1.905346123931221 0.30969838202705213
-1 0.23757107697869606 0.7009274223790678 -0.6005151170274707 0.46131870148693055 0.694253134444586 1.8704279215134783 1.9559864883094595 1.5475302665627626 0.902775266852526 2.253986651760284
-1 0.0931484209802732 -1.0536269817119295 0.7832662454709735 1.3370869763110287 1.8021230335269156 1.0422523333084228 0.5539002500282262 1.1402739247006104 1.3778884263982012 0.9839666885480669
-1 1.4022006973888672 0.3301442305911556 1.4159864215392552 1.0753881627418582 -0.2194812627814522 1.576874528728394 0.351144790840509 2.9042579131410218 0.33439079197692423 -0.21115533384764373
-1 0.9200624394093888 1.9601307267236312 1.3048792499777433 1.044019487533702 1.295476599028682 1.06479650163913 -0.8347875409017176 0.8767774440123639 0.1631761919249426 0.962325538273012
-1 0.4606387639284839 1.93128591538725 3.2494332751166293 0.4217241090513292 0.5940126704202255 0.12271071800591238 0.009005952876745105 0.0631236875750606 1.2229161931162333 2.3879030147755866
-1 3.2172098250997503 -0.021922357496697797 1.1859662862492402 1.2154601324678136 -0.3071029158823224 2.1738376762747613 2.2872633132290443 0.954809047991948 1.901337785669559 1.3011976479019711
-1 1.1885608047442375 2.721310638802292 0.9617587859607313 0.12651320336878014 0.12567757686210834 1.887061564570169 0.8860616196551063 0.6430168020234137 -0.030733700547949327 1.0564998980605065
-1 1.352748382066948 0.5202126729710697 0.14331687879826782 0.40785023484169414 1.9641960196192663 2.7910712640458297 0.7740423932819342 1.52559135640059 0.3239548613578228 2.31826432040899
-1 0.5203741956670356 0.884417958844451 1.3777220780800918 -0.4643847508675174 -0.37572084642581793 0.1262513952897556 1.5518202424896383 3.3877379158242378 -1.403581970685686 0.1009940122529609
-1 0.9894392616099077 -0.0034178714976433877 0.689046476206714 1.4208906847616534 1.5473446325066496 0.44218920279820595 0.24101228948954234 1.1801070630847152 0.8039116009276253 -0.46102470089902536
-1 0.6361572167176843 1.5563186537784683 0.8983823810124998 1.0798802186419254 -0.038600239378366874 1.6649842223710727 1.6378836320811345 0.3059309271799856 0.8901320418030211 0.10914549884068314
-1 -0.18003932381317478 1.5693004310535423 1.8013396839368538 1.7544292528839476 2.460230078664536 0.8072540575395855 0.8326108318826944 1.5006349728524033 0.7460792678168342 2.6820859579435474
-1 1.8960169042497794 2.1576293718618 2.424978645426269 0.6268556772800932 4.221588312115547 1.1780884004744951 1.5616604868899797 1.8886529082537074 1.6168854045075025 2.7308325759110224
-1 0.12878554700508837 2.1150328351027246 0.5356772045785253 0.8698163232516893 2.3406750293658183 0.6627125907242539 2.4239833684636736 -0.17649747406412253 0.34655417092691454 0.37167266730649473
-1 0.7700976682797439 1.2052165149892542 2.0323449543315446 1.8093079753157488 2.677682507242789 1.2230772168351174 0.10002304289163721 0.38829774391404126 0.7382541961293962 1.4604650485834432
-1 1.2304476527122155 1.5911723818857464 -0.6663405193368004 1.9423332506900772 1.4218831147452045 0.7172255125851585 -0.12990659585261488 0.9108053409327858 0.11424096453618027 1.1083558363715305
-1 0.5195105474968298 0.5710613703505523 2.2928613438234455 0.021245928903329103 2.1269497746764197 0.8932419976165424 0.9360795887134954 0.4206153958722527 -0.013928240567511851 1.9267860815714657
-1 -0.27500090463981786 1.163598213361118 2.396756337306596 0.7166497755216299 0.5087064238485857 1.2644991273445112 2.207063036182604 1.511076159763578 0.7514616147389759 -0.386653321343986
-1 1.275981257794266 0.28386450023604437 2.0468065778588445 0.3368819014778913 0.7803798072812063 -0.11268418399709335 1.0692622536985994 0.7450466892913328 0.6521234033954817 0.3533878920228143
-1 -0.26632749480506046 0.09964814030131464 -0.14774546592772242 -0.44102911713759774 -0.8175624623446118 0.5982737657645009 1.8018589102471618 1.0206495963947055 2.1703414097910376 2.509625756793014
-1 -1.084176873793715 0.003374206020577475 1.0490056163609893 0.7413062315194299 0.5457392593753987 0.47876209776833123 2.7997789450020427 0.8473717379952329 0.07511100942298876 2.342980564354181
-1 -0.6060249411337237 0.3100831921729499 2.5027389254157533 0.4950992021162349 -0.7743243396300394 2.254986439984994 1.524435417647438 1.5581584085809914 0.7613263552054441 0.7313335506205685
-1 1.252570109684499 -0.2259101116089468 2.02870927406763 -0.1982100935627482 -1.0747860634656639 0.5696675160105826 2.0536113238469964 2.436984468208358 1.087350912351074 1.6355207346806782
-1 0.08793454138157841 -0.7701820062667433 1.6526323582054276 2.648211639393969 1.5418579075681154 0.9489571984728947 0.05918410476639424 -0.9099915058439798 1.4346179896632103 -0.7890540352574975
-1 0.3047705090908783 -0.041817851700766795 1.864590556312606 2.2126512576725283 0.850687528022706 1.1516079924281961 0.7160824885255048 0.23428914563411007 1.5892718454214458 2.0304685172157515
-1 1.8541494516233115 0.4996871983195521 0.9048408243621995 0.7096255802229431 0.33910504796127783 1.3134581495613444 -0.2753494959695286 2.3289922141730686 0.7323942203055318 -0.274626661821493
-1 -1.338544772611924 1.2944523849511644 1.821257734737301 1.6793492696385324 1.5967736493283293 1.712864874826922 1.5745612820947925 0.4891550646810052 0.47846091208172825 -0.1743221254069207
-1 2.131766719148957 0.7608227099296399 1.0630568268599263 -1.1476984731054647 2.3867190880037636 1.130561984384332 0.9131559753959471 0.2973457770910879 1.3007036631285942 0.4372322143839449
-1 0.7708567792295566 0.580257476003238 1.5887140302216574 1.0413330688401965 0.7733129718389264 -0.5163740146933058 0.07497254374425988 0.28623086041167667 1.5489309172205683 0.8551008347224718
-1 3.4595137256272586 1.1532560360380666 1.588361571148596 1.3802224477267615 -0.7001860654912402 1.8740796848274577 0.14520299815591176 2.5193824279795254 0.03909705046483791 0.7357475729770275
-1 -0.6544136676184351 2.8745518291193553 2.1515280898247315 2.757731240766754 2.429606589051394 2.330014751072225 0.9115033589433934 2.6873787753182583 1.2992135444029829 2.3920287356459284
-1 1.885270281917602 1.858016821901751 -0.06157363620807099 0.308401967243883 -0.31307820201782555 1.461038889339163 1.6128329392090914 1.5772000116247265 2.710615509497419 0.8050419240018178
-1 1.405879563380197 0.659914831493603 1.912269260893395 0.529404740699135 1.4277377811246783 1.2913475473601614 1.7339294107927208 0.5215235778431477 1.7550541630505698 1.4400196124978555
-1 0.3245588747842635 0.42197424404348816 3.6539265313256526 1.2857918279043645 -0.03655209163203632 1.2407043968389915 0.4433829786888507 -0.07023065483472712 -0.6733771504197963 1.4798448078129154
-1 0.9085359200450331 -0.009624824747410887 1.0280527195285618 2.14148134591638 1.0562537066073983 0.8809817771790907 1.4071063563557673 -0.6597423723027149 1.5583011903165707 2.3154204049509683
-1 1.8050769097358077 1.7786869407899135 2.6495184641125515 1.158177494691216 1.1671375960394383 -0.45722370125523115 0.9835693406300088 1.6357021360875077 -0.16826461081967703 1.1932740024664812
-1 0.576688853348233 2.151495453088904 0.8572555252181385 3.405728819429614 2.101231270195057 1.6771308649271772 1.2637521672030567 3.1154229758040874 2.485850964748577 1.7694224707976827
-1 -0.22806118428106337 -0.9061154967479863 0.8964938904788088 0.6816585601664856 2.013761003670729 1.0313228363661557 0.9260597798962866 -0.18946147062989205 0.28527619220858247 0.8963510651947846
-1 0.3148947081465582 2.161975824817249 2.609645991041186 0.959492387316128 2.397824851151471 0.6697921252418206 2.313069590047294 0.8776639563036727 1.0599994333376752 2.8237989480782524
-1 2.652125755323301 1.8602107889115338 0.7683127593190835 2.2682293581606165 -0.6222001971107851 1.7327348607601576 1.7973442155328485 2.3026732779864645 1.6376913865909977 1.4336254291699817
-1 -0.033946588281949186 2.300669560977641 1.160077113314741 -1.035089589522486 -0.3088401922649133 2.2246952213732962 1.5263288862385613 1.2041606436782568 0.6360015906365958 -0.46568448099058934
-1 -0.8340563619947565 1.4168203411347104 -0.5724699864440952 -0.5633561206742383 1.454288263940742 2.091140792301254 -0.9346927324544323 0.0969827614306541 0.9901527415253794 2.0293060494871034
-1 2.1766440722293696 2.1765927443625097 -0.9288701141928257 -0.4887885438886057 1.415145042839749 0.7869820800801398 1.3531410283773004 0.38467574204818133 1.265876278197796 -0.2027790078386682
-1 0.8270879503594885 2.371236015912422 1.8437897438725939 1.7890683065643116 0.7718878947557098 0.1132854516378462 2.6937038226634122 1.34827091113804 1.8024405913978527 0.9733403683960185
-1 2.4175771508586754 0.8851307536623965 0.965109486208773 2.4006169759083864 1.1967556814639715 1.2950307543358157 1.9415648218013744 0.35864528885541735 0.40940436545238557 0.7868294504129988
-1 2.2098184536505663 0.889100413360103 2.1851586347238285 0.13494389682652308 -1.1445348600024268 0.8595807349607005 0.46845661480480505 0.07882338616350792 0.222858479263641 1.6187566311742603
-1 1.5395105587908753 1.5090442727804423 0.8644957394514675 1.2222062988283733 -0.657302278508328 -0.8584774737648058 0.7847354502810749 1.066321874171543 0.6763302367935397 -0.3056807220148554
-1 1.3241371059217268 1.1998033042587848 1.6413385242724854 1.2616652980595755 0.8214439629174916 0.7323804916810981 1.446327599557899 2.1344373550969333 0.5323048652541784 1.325312471981157
-1 0.44793596733276986 3.5291804831601397 2.304481907075438 1.7159536021092872 0.49378464200637107 0.529685187245525 -0.19498379135409039 0.6257392880667672 -0.5922944256976155 0.9677085580549932
-1 1.6001908684230077 0.8441053959985582 2.191005295444758 1.8601204690315698 1.4231646338661619 0.7172326899436327 1.3685291716454426 1.7459708463423858 -0.20021564447567597 0.7886037237104406
-1 -0.832715908403886 0.9821249159854097 1.9340136298649147 2.0863867471576207 0.8588263222826337 0.3940359686539505 0.5667076617327207 0.6813674534100007 1.0601080933156564 0.9940095449693623
-1 0.5362749326926859 1.3784556073957994 0.7830926551836939 0.7926130115032175 -0.45867401264881047 0.7649235836439627 1.9252198419840811 -0.5932278037833087 -0.20495235948345436 0.8228620061430476
-1 -0.5026862346261936 0.32379950915933053 0.4877018370232078 1.848487603750593 2.5612814512394575 2.6996258863788105 0.15501963775759875 1.779188209155349 -1.1587607119995043 0.5286988956500273
-1 0.03890979688369878 2.5700833608321876 -0.41167989902736224 0.4405078623025871 0.11339883057634925 1.2618969624421223 0.5661859841701755 0.4450152294875418 0.06553355298472463 2.9653045304903003
-1 1.2066695218108954 -1.135846422758188 1.3472000646449644 1.995247004371493 0.4067019132360835 0.6014718489518214 1.1945804244235247 2.563237911092928 -0.30000446942459824 0.6782859264246553
-1 0.43145271645135497 -0.15638436316804127 1.806542814206817 2.509982504123812 0.2908319784765735 1.093034072836503 1.8310934308417324 -0.428111571478186 1.0227258944948991 1.3181088073443865
-1 0.6593145377977876 0.5513227059953492 0.08971356052593105 0.6997087344297779 0.3547337578286779 2.044316172416025 1.7054002807979272 1.177077903869836 1.6118683425448608 1.3817764734854732
-1 3.26027582916473 1.922453791560931 1.5445220345277253 -0.3361563876793128 -0.20451311346146506 -0.02755370253733158 0.2523835913052155 1.8457060509750052 0.7729749699076125 1.2691512131543639
-1 0.7853510230572176 1.92550267228468 1.3840760296517856 1.019170128522936 1.257277800158144 0.2954835667658987 -0.02339082355482236 2.344976472145047 0.8650491281625572 1.6705466337391612
-1 1.0256022223771357 1.2521800754728607 2.5454645690960165 1.519642791108941 0.8120657189050374 1.395012570155324 1.0067859707833062 1.6154722360698295 -0.1911479039843622 0.3192273565677406
-1 0.9212215747887599 1.614097542109768 2.153211482594465 0.25851295883461667 0.015421396864703008 2.910093225363264 1.180736322866857 -0.024920942327103957 2.669708944799861 -0.4455433802815518
-1 1.5936186055028179 2.948335176521773 -0.9304959929630894 -0.25674218734698395 0.856450569458336 2.2464434469263295 2.2695814273033834 0.9023024874886443 0.1998192758289271 0.9614747140727596
-1 0.4171564598259989 1.2341430652292795 0.7613883447910024 1.4327906124857261 0.8248656963940865 -0.09370178940656282 0.5302446693348143 0.5977304498921516 1.9672679105851836 1.8549778581991436
-1 1.9988876732611685 1.7067688718725715 0.709840257121064 1.8195818549115197 -0.196218309209645 2.158975719537872 -0.387052375493828 0.2684905146219133 1.1751943798566946 -0.08233263071043195
-1 -0.004588558850024516 1.280146957738293 2.2274500380613915 2.068436441505224 2.4406629422607455 -0.020552259353522784 -1.9306504989533266 1.606929445859563 0.12204039563080737 1.554314194847439
-1 0.04312231827054913 2.293183585915505 0.5515907062418919 2.0319631309075303 0.2043494544647857 2.163212294566986 0.24687989300151647 2.1776229267798914 1.1368594510956058 1.1067868768921156
-1 0.8380882562583268 2.7318988397710573 1.4749062376973399 2.3244811915569885 1.498055997999189 1.4901966783173328 0.9547300656875682 1.2938212544822327 0.920830744648933 0.7960603079946061
-1 1.1730459404168871 2.4157763285361744 2.2769114804572554 1.781254882347914 1.8939310535271043 1.8204037399884672 1.2330253630970833 0.24898375343327694 1.4526754173493885 1.2327670337378527
-1 0.7828957363283248 1.961806185656672 1.0945811949626496 0.6471160715303457 1.2988151512993327 0.9231258952067597 1.7059995140840485 1.582221842249981 0.5731086038064922 2.929881320548402
-1 0.4240209410200867 2.0612687767691504 1.4013347045251126 1.0775762488985852 -0.5648359238473468 1.5394818276041304 0.5250719203859092 0.3867254288273827 1.836032841951298 -0.02644684457005053
-1 0.12838309666764036 -0.2524433635395231 0.14063539701460914 -0.8169781441139783 2.638413098813798 1.5872934688325704 1.343252734685199 1.1584200404773857 0.6163819194666804 0.6654328763469552
-1 -0.26416941528334714 0.32620704315453675 -0.7502936599619701 0.8401389782535786 0.09753988131424873 1.796236698582462 1.5877879186693455 0.9856032545638709 1.2072784259771 2.4653229099496707
-1 -0.6337999979940661 0.8076685452502981 1.2207084350653477 0.9123689527781019 1.838283774286254 2.2836210170990996 1.7394640050289512 0.6351189156017663 0.9629884451362287 1.7680252591425618
-1 1.8654459163757884 0.06089772776268909 0.9679374944456427 0.8889470807355174 -0.08754935246071827 -0.12680613988340284 -1.0637769092192588 1.512338996915241 1.9515416090320272 0.5015769881603198
-1 1.7247706923845918 0.360222898716523 0.18071931378959916 2.0371848423820293 1.5266006033053001 1.353704597154892 -0.2696414308039541 1.343721201156886 0.46275842064535144 2.3294944321291413
-1 2.1105081742950267 0.5116093610246693 2.2446634834462875 0.658957834299546 0.34134432630789047 0.4247161540652681 0.3292829996171407 -0.19362053618697583 2.62788746256027 1.3966627696966927
-1 1.8475295891856125 1.3887694988244523 0.6817244598020126 2.5809988844215908 0.32696789850689245 1.081015261872673 0.2386938164664013 1.0118382786145506 2.209217716205016 0.7574090447478952
-1 1.082260517720307 -0.6266070913930977 0.6832252128874979 1.2966340694320664 2.324615742379285 2.5627557774177543 1.72092865539378 0.15590225454118978 -0.2816198860581334 -0.5099568334403046
-1 1.6725629461607472 1.0353690658867798 -0.8225360006266837 2.1324720159286894 1.9885924374595836 2.537256632003289 0.9677496818620155 1.454681559021501 1.3029797950165192 0.26385709812366753
-1 0.31156560050102955 2.1652814753810112 2.0058163682540036 -0.04562872657851469 2.724179402266973 0.6222125728521903 0.42811650448637917 1.0387953213300416 1.8914700820960233 -0.5893540202775569
-1 0.2578251741975023 0.11378011266272059 2.797638612913183 0.13983902653928637 -0.03255261699221346 1.2576586825716858 -0.6642415184742925 1.2799765368331657 2.3385679931813983 1.8159437052025178
-1 0.33578001261352897 2.0063591095825952 1.0807987120174516 0.3543665780473314 -0.4202063816731054 2.113462588586846 2.306817160855979 0.9446592793327631 -0.6774687350899611 1.6189786930902486
-1 0.8614448755152566 0.27807051666810034 1.490952308696544 0.42812809570277155 -0.6130395196516234 0.23931476380563366 1.3454272824526288 1.8553493467683078 0.7262585485463864 0.8060386596767135
-1 1.509477780297391 3.879562737499862 0.5886532526077162 1.2655619776606024 1.3990929522583664 -0.34170560649024506 1.7418923966881366 1.629417743427085 1.7445593580979215 0.5930685838392928
-1 -0.17633273947080386 1.8278089865738787 1.6079874279761104 2.0641657251872525 0.0013949787963080107 0.9779219807727019 -0.9229761793545943 -1.0291570090345807 1.3628786284816425 0.5752391889181461
-1 -1.0143862085431188 1.1194733654329676 0.372026303777525 0.4779765819717211 0.873963169712578 0.8031044909741862 1.438202993892749 1.483386025663741 0.39707846786644874 -0.5347159094832814
-1 0.11016676987687668 1.44535659616203 0.47296285732106014 0.9569700223555272 0.22754986353621043 1.1107842631735818 -0.20365888995072612 1.7095423750241086 -0.848293390426655 0.857847169492578
-1 0.7508129008937717 2.8747883333024182 0.8289112296791319 1.5951701814113632 0.7420525998761323 1.9537834679324622 0.5603407250007024 0.6017647337718439 0.6431621236261322 1.7673108381156395
-1 -0.1852593368859976 2.2089214215364246 0.17988209448256942 1.720553251777205 1.2120857158218548 1.296273725719677 -0.25129199617788966 2.0013217992492613 0.5065314908683332 0.4536706566267381
-1 0.3257759973178981 0.17932720424930182 1.2245897173975124 1.4392674655132107 -0.19990974032801478 1.616015721370362 1.0976249377861196 2.286751487136163 0.5998423893372578 -0.10744364268832474
-1 -0.18860318421456523 0.6481395082246904 0.8471055242008172 0.8364035710726628 0.5027181893375049 -0.04737632027053729 0.6081198234429218 1.8117061812925739 0.7882062608326725 0.501707612022315
-1 1.4843082385614745 1.1158750459458913 -1.4894665738544455 0.25826376510509763 0.8737547870296022 0.6842381688703825 1.5781821909490459 -0.8859809290045597 2.6448010296898516 1.0451355125183155
-1 1.7920903749688475 2.181377042700981 -0.2580670741698272 0.835878310743556 0.8282113555574907 1.2918481880236576 1.2845735763240005 -0.6226879211726246 1.7452863581983848 0.35415213876681106
-1 1.6059906951044978 0.5477408796911678 2.033456301629621 -0.6056116844976043 2.3157299435817342 1.0282347361444912 -0.37895653151562936 0.9752299146785057 -0.41816188526715736 0.9125445080555991
-1 0.36434340752558814 0.6902917518300258 0.9253611225661063 -0.42114130346772227 2.0970094095591443 2.7085188507498557 1.4289293922116237 0.9542757519821615 1.0546374187652479 1.3258156303811686
-1 1.4902539943349453 1.6573630488454014 -0.3809764834643814 0.9358657723296077 2.7348124001551435 0.9897672456356681 2.560439397267852 2.494870519932018 1.6580041060544213 0.276867359286432
-1 1.1191344811462158 -0.6181668923123884 1.5490411146166472 1.8183809809806493 1.3028570357467482 1.486951380254144 1.1831247980434945 1.780974941037947 -1.827510680099897 2.305550677513012
-1 0.849190160180726 0.927714888220189 0.4152982301284849 1.7201547897444616 1.0010482110516308 0.47888318535920815 1.7303425098316922 1.5212540746719077 1.2164640343110604 0.8672666819224022
-1 1.1818789164071632 2.3299574339825355 -0.2238086965126307 1.0866668603828966 1.777789469252217 -0.2473412361708398 2.4917056426594892 1.0985567817486692 0.8205900594343175 -0.4507497282180284
-1 0.4806312370873962 0.768849921524061 2.2816919830317324 1.8888027374056304 1.3666588628364746 0.313010983641146 -0.9582374160527103 1.7350822166838902 -1.0292285073997203 0.6398099597089605
-1 2.387963695369674 -0.5899448356258876 0.21621305588176487 0.9380272998222627 0.6981388782356867 -0.4629800914467903 0.7722932223610299 1.5585013561079406 0.39398387576565874 1.605900840338324
-1 1.2715952476157897 1.439635629557708 1.0983640636833376 0.9812043919910073 1.5353214720014243 1.0984936772644822 1.1502708274998623 -1.295397653899192 0.2861064908535764 -0.9932837563816654
-1 1.3012696782417956 0.7849306120035814 0.5043907367704977 1.317902271109904 1.2355512152607722 1.7921035283313613 1.3780045579049331 -1.1334086181295735 0.7594490553748667 1.2920327236325173
-1 0.7390703584602525 2.457743695195635 0.3128347254263576 3.2777913748283356 -0.3729594628152144 2.2165912805252592 -0.3208945778133039 0.25945266028499947 0.12129953303222862 0.9577961880424101
-1 0.8445123778336028 1.4240300974070288 0.1873583546229668 0.4955218063785525 0.9094332296150236 1.3540661068354631 0.9171697258910753 0.41888437045897486 2.9462218414395487 0.6502477720645555
-1 1.3877586550503413 0.987611562870769 1.2584972385417663 -0.31990526604547664 1.8690834901315843 1.7043650395994414 -0.9964092334530854 1.1408598689320075 1.4213381391949258 1.3073798077919028
-1 0.06076427697113995 0.42120236957849067 0.592901981159774 1.3720471193027384 0.9036775292098581 0.8953372123185973 1.5452404312257344 2.0708178196722606 -0.8979750106430204 1.6853058787444881
-1 1.1694470503331111 -0.7289698765725721 -0.3241777565346444 -0.02733490335945188 1.8863228847530946 0.8073024667207529 -0.9818689747023401 -0.4283553318571569 0.9994871828689351 0.07075638531545037
-1 1.1047596078086386 1.7708874592017232 -0.1612806069289101 0.08556210685307786 1.8572899576629136 0.7200423074285855 1.2170692625583286 2.0347880443589847 2.7432017121214005 1.3957939162622077
-1 1.197861378414133 1.556444574585297 0.629813576730021 2.4550574210435823 1.9226732616821978 1.9859797173418605 2.186728551603152 2.221928254196631 0.8555508774400884 1.723787004755138
-1 1.161571044817612 0.07979292393847359 0.473025751301427 1.205676831999432 -0.5466232243147817 0.8191419439472176 1.0060075056738604 0.785322530707329 0.22058837011880694 2.6154680787761726
-1 0.17077134170060482 1.1137337091671946 2.318497500926356 0.3973424625226393 1.461779582118195 1.9295571893710908 0.7785519323891255 1.0672230065462434 2.1223852587473258 1.5460766694219767
-1 1.1564652200933274 2.510183232201066 1.6891434345580443 0.13174662119947889 0.8871123877951895 1.4958243544578553 2.9794729912305575 0.901901296036228 1.3871706497633103 2.8969924652525334
-1 -1.0521680406383696 -0.0031861766791221324 -0.10915897400357322 -0.1303567225640898 -0.09337344840645234 0.7148597244723245 1.2180327568998717 3.4184983500514545 1.697740318234704 2.002711960184084
-1 2.376709016910577 0.958001009693663 -0.1081121213002203 1.327468223880286 -0.41205779656829145 1.4289978911250902 0.9819807423748184 2.3188491121493113 0.8657078618437748 0.9391669120890416
-1 0.9776980417955967 -0.6674206197457981 -1.5563935251898675 1.5446269906729104 3.047754956305709 0.3970621484971374 2.7173431471851766 1.7243005353672034 1.9755492634674017 -0.7077753665556163
-1 1.1671355902086602 -0.8193057764678835 1.410567460875851 1.7497653081783076 0.6901637048786208 1.2119799048759736 1.3226344341934888 2.2695811100443404 0.9907324730003678 0.5558635315480431
-1 2.4336171222847973 -0.73180099697987 0.110963544711143 0.2466617891220264 -0.8154643837784403 1.7051343160057892 0.4485983625979719 2.319215306602568 -0.5223921322733727 -0.05099278306658839
-1 1.901698041087508 0.8988295187852892 0.6511477798135669 3.0420349436695076 1.3810269156306683 -0.24628147854970273 0.5188524250377791 1.4141097609090438 0.24777660167964255 1.535797527794107
-1 1.7629403294957187 -0.13022007315691875 1.1647647804960592 0.5890754693324485 2.06533631915097 2.21452694737647 0.673652898562904 2.2005666335367784 1.5261645592168471 0.9017580067794544
-1 1.7376137405520378 1.227528622148764 2.1537333953075093 -0.7244714994487282 0.9737436380972475 1.1956909226237713 2.612848244020281 0.30122025453481716 2.973720741303093 1.8186667174448368
-1 -0.2742361456988558 2.1098716503801613 2.953664212753427 1.574905508426148 1.8552665501344494 1.321110382365208 1.7445198966258182 2.471288236145563 -0.11919705782427648 1.8624551969544791
-1 1.5436386497853212 1.8153339598609863 1.363613793156124 3.0510249899073756 0.5489376037189108 0.007578350689908864 -1.1820947864458877 1.3011272158310803 0.07518458687451968 1.5312667541972245
-1 0.3224512020283108 -0.2209974586026877 2.042104637824572 -0.37728305633852743 -0.5498729693279798 0.7193283373851307 1.2590924907118073 -0.3944236589332939 1.1250230341812884 1.4070211742408931
-1 1.1444341603579156 1.3629504333367566 1.6939924628296188 1.9479380654467797 0.7894876586788064 1.049604859005768 0.3408015558912614 0.6014994900100508 1.4716224256141708 1.185118554114717
-1 1.5859690594959832 0.30570898129196966 0.7464020043785254 2.2285474871009723 2.412881908798376 0.6904305558007539 1.6192643153889568 0.5920043651364744 0.7807197394828229 -0.20297994754139137
-1 1.2950387623080977 1.0916188301034222 0.6600573067651259 1.862615598644322 0.6876153259228353 1.1481594206078056 0.8784422750187779 0.24715809175194348 0.7857238169348668 2.1619479520100247
-1 3.0828763562487733 1.7362496731683166 -0.20896157853930264 1.5332869652046193 -0.21794910668079526 0.9202735211245334 2.574049390833994 1.5268503392385662 -0.38999953644207186 0.22479935308805854
-1 1.7627009184421887 2.2255381870678437 -1.016295091642716 0.6254801643275638 0.6618861479958897 0.9047308122786223 0.852721929456685 -0.7505113940627413 1.7250343985280407 1.8166918481323084
-1 -0.5022420621997736 2.733043970376204 1.5120949360070959 1.9428063677250476 1.3780749670748853 2.2350181236519657 0.8716131236741619 0.2782380235553522 -0.297799811324456 0.16653587974789763
-1 -0.2981918597327633 2.860715416679886 2.1275708273598566 -0.29508534819399324 0.846188811185981 1.8713251354650118 1.0723090993878512 0.4374636574396571 2.210140762205574 0.6809712558014431
-1 1.5619715587750584 1.2704149431309402 1.9712386149819312 0.026280766936758293 0.8206955786918028 1.6318403698412411 -0.5566358146889887 1.7571793612461013 -0.5366638533754291 -0.040269040641153
-1 1.2643496455778207 2.038185139306229 0.6395741359412223 0.27135915089505125 1.4201127961240902 1.5041067668659303 -0.09091064494863543 1.109133071144227 -0.4794905621068224 1.3208155875591663
-1 -0.02895244930542762 -0.49403509214487396 0.712435362084801 2.5460059356446374 0.9396714328426592 -0.7949960754019478 1.6183020075071732 -0.38577084963397135 1.6991710568290967 2.786233832662353
-1 1.261753017958196 1.0918709535770748 1.1265646053317926 0.9867326079450506 0.8288572122803143 2.4418772115091816 1.0454798487585901 -0.19993011811143235 0.14523995518141886 0.866687319252661
-1 1.6985511320556277 0.795437122527888 1.556653786587669 2.1174479278276426 0.3999172845317358 -0.5010796653100276 -0.08438438589923591 1.1138001295987414 -0.30602571964029956 1.4972214829613484
-1 0.41786595805108906 0.6459011706826348 3.657046684462284 0.8222874793996409 0.050062147599186035 0.23963259661744873 3.98442324525362 0.28119552752146837 0.8964441562070578 -0.253526879649719
-1 1.4488020919552733 0.8929138056330631 0.3161270487767218 0.7331766954467245 2.3366307109566495 0.6815405492334983 1.5281435010244593 1.6431760386153362 0.5321346633571438 0.34130859830303917
-1 1.2748486181912866 0.33303368481427886 1.2151848478627916 1.0756517104783787 1.2083219051593854 0.8277625946461055 1.9666455377419778 0.6651325140447175 0.16327294989918317 0.8603717402697098
-1 1.5090300715612457 1.5180463731650495 0.6972598598076571 1.3556192196865902 0.9126434148820246 0.8127664907242128 1.3311309435526322 1.279157714746425 1.7829837559894246 2.988071791570289
-1 0.2727158735259818 1.2998080669104182 1.5121347623238246 -1.5679984907159152 1.515508708019623 -0.15391403969184858 3.1311081089984323 1.847318459389865 1.3425374198002933 1.296082544224974
-1 2.408189206457478 1.2760154921881726 2.1197548437178906 0.05936234352435599 0.19907763560203529 1.5479638808770004 2.471816233765586 2.4680208521093805 1.4113824572688618 0.383801428379995
-1 -0.17965112079351564 -0.3404976625536871 2.7837262771738205 2.6881515223765398 -0.30847324983815394 0.9993265400000024 1.1374605736665502 2.2049953998249694 -0.2513007616550551 0.448830380725894
-1 1.3443693966742452 -0.025711889743784466 2.2443775230207503 0.14834884628873723 0.7271367845373308 2.4714407353590957 2.562158361402452 1.7047011572226343 1.6769293581505482 -7.308081317807247E-4
-1 -0.41870353312467423 1.2877545442386 -0.3164789161896502 1.803839696410392 1.008076378658354 0.10616668976164723 0.4098865481816575 1.146539676959654 1.1538344544688937 0.05907242504921317
-1 1.7936911543812046 1.485342520804878 0.31800311694795325 1.9199555201066274 1.9312631279902837 1.362366670774782 2.6306006265218365 0.133055817623004 2.5078649689837027 1.2068433004457952
-1 -0.1411582634165307 -1.0426813196108524 1.434523926692467 -0.25113509019608093 0.507539296016366 0.23168671363927917 1.1893212121098466 0.8304584451378183 1.4556473134325054 0.6534542423873613
-1 0.6079927716629916 0.09194609771904183 1.6120179701101955 -0.5022953903177365 1.2170945269028797 2.100831302657739 0.8386155807612904 1.5684558466558434 0.27605209581418555 1.5594274213225667
-1 0.07428493649230228 2.293483112741116 0.9708779280979398 -0.45177079067335923 -0.057110219872378076 0.015433876379835065 1.0794154562045615 2.105620271870406 0.9395998613200235 1.2851835351116119
-1 1.578883010870155 1.5609283984502076 1.8223960032380064 2.2142614021520837 0.7130462722633009 0.9252426132551667 2.868560600039225 1.6968141988566166 1.9976720397763048 1.6813323051682774
-1 0.5016495406992045 1.04908195692884 -0.07722896372502253 1.330713406245241 1.1267715047602667 1.6360574586472572 1.2420706446269942 1.9672850660325922 1.054929403781838 1.6077148722801038
-1 2.0538334867970534 1.9213949071716163 1.8934373144800345 1.2381794078176593 0.9175279056098742 0.8206265873347616 -0.8312726444851357 -0.5131966390183769 2.567300850622103 1.6719008505918898
-1 1.2689208746241893 1.4402293624087208 2.7176532271741003 0.01336457957384174 0.1702333910599565 2.3778902914738547 1.7217780353501682 0.7054536312666535 0.3361164972231122 1.1589949811743772
-1 -0.5767062059491888 1.7138887496399136 -1.1154021033816348 0.7168636442060621 2.217046440509127 -0.8161420769580656 1.6271150941587713 -0.09702287214964955 0.22946937882986906 2.7922011937600097
-1 0.9710624979613078 1.5610147329117985 -1.5053608758479413 0.9711728502628203 -0.5150150692664308 0.49562546380947603 1.7163450863443273 1.306018285087743 0.5473958850146698 1.8540315462762198
-1 0.6425941154359618 -0.31480994520520533 -0.056642174933536404 2.2269443093694914 0.6505566385114631 -0.3709635056159635 1.8873810442041976 0.5119563367121428 1.291713540770698 -0.6943082761794022
-1 0.5927308007246384 0.8464951673655936 0.18447571041818456 -0.006190250203252257 -0.012631850494107644 0.81828806055344 0.03231106794400085 2.0927752513240994 -0.12600012916564518 1.9639580630933335
-1 -0.34831756463523855 1.623268907572022 2.1594197097470325 1.0562200902265129 0.9414684460546705 1.4340305236290405 0.7654931413466368 0.01719894816346723 1.5959585538584955 0.2885792827923064
-1 2.2697657120238466 3.1420889453091094 -0.8210208940698709 0.2035264954846796 0.34878833066083437 1.3187569677046596 1.0219701238612262 -0.1213159939916395 1.0802611304225862 1.3078831016284853
-1 1.2480724077104584 1.9077146304274128 0.702946174596962 2.3286147355852034 1.0071749708265634 2.5149204905160154 1.349779745606328 1.044016863507004 0.365723895391459 0.6519926945711725
-1 -0.8985903846454402 -0.5021240182148043 -0.01073065243449256 2.290069714856683 1.9819036535789476 0.03105672582226615 1.339000036426309 0.3323749578280565 0.8021635756060409 1.195220952578341
-1 3.008655872898343 1.0129636641232918 -1.5088469891308582 -0.6947292093040875 1.2487527838514174 0.9032973743393249 1.9979774814850564 0.0435076158833696 0.8478193472405138 0.5026222405279126
-1 -1.0608662183020523 1.511703517053053 0.4555272804535656 2.076056547724862 1.754307244984986 1.3854010129660659 1.8247443481696117 -0.0246162652477655 0.24988078939072067 0.9872960257572898
-1 0.8740725946015646 1.7804072513374016 1.9060935705517543 1.8265003967793456 0.91953745409342 1.3629234354248754 -0.2803757506365385 -1.0129022749852892 2.5019279152710756 1.5245757538298341
-1 0.32688805354617134 1.6000098575767967 -0.1786618864414944 2.3806085458526325 2.3338676324290164 0.7609884113833272 0.1498428862635196 -0.25090796239660373 2.3770456932981814 1.6131488558961797
-1 2.290620763512112 1.3541047134925366 1.2421787622602398 0.8804930591189608 0.6595899728536196 1.6277353547734075 0.18759874372088237 -1.1351531086694964 0.18251082831485133 -0.5713204010530248
-1 -0.22047844715313447 0.8310592465340738 1.7892315227363613 1.1470591393757708 1.0726224455927464 -0.10592031044447459 1.9817888345656018 2.432077040490821 2.2450973493606203 1.3210707817547482
-1 2.070368262568201 2.3671178117141207 0.8627035047548697 1.366475314693422 -0.8331190909005985 0.7551440285820138 2.178737629795865 1.0323167492638525 -0.3148106607913368 0.50662477745953
-1 0.8604853943488086 -0.09592589897715587 2.600032474430587 0.9839706092809413 1.519739305696014 2.1260793286184008 0.03744939964524108 1.2611070446598698 -0.511324151442442 0.5454482162340912
-1 1.8946369523511708 3.362602104881858 1.8838436706953976 1.2491758602363099 0.0054680988441749845 2.651799339501261 0.6411444300353089 1.1035969889037076 0.8324869555591509 1.3031776807447846
-1 2.5154071822014554 1.6803408091264473 0.37434333648729623 2.496324926040323 -0.16401882096773224 -0.5744479735763091 0.9352239350517153 2.442683227544391 -0.5264039462194898 3.015307788051603
-1 1.5111987262832436 0.6410066045062515 1.0002585904405568 -0.8894537972030532 2.8014684904508944 -0.5393437655384221 1.1524079090931012 0.021728095470450404 2.1130698813482622 0.9468113077109184
-1 2.246571391447209 1.2010599601897547 1.234941576895316 -1.7706644509786722 1.471058855485551 0.8939500026890757 3.0844244960496563 0.3937694347012187 2.4529138646148967 1.1858907139355346
-1 2.4615314217465514 2.138799653615231 0.6155097299332213 -0.26863064780465895 1.4804373561575783 1.9409343558847068 0.44935568187190045 1.4016783544796323 0.5844124030092861 3.560614430022461
-1 2.170074376135311 -0.044012090187616204 0.4876588954783079 2.3603606696538524 2.125197091710744 2.4134190214591262 0.41472234938098607 1.9434029103795312 0.10273955644383004 1.235145974467383
-1 1.2969727061242051 3.098685038424812 0.9785969987985332 0.5224703037252412 2.5948178849934393 1.9056896554251344 2.1303162130115787 1.6936027246350522 1.591959269634407 1.3287905654720076
-1 -0.015989877059035873 1.5072072218307366 0.08389293810681375 0.9234581285114085 0.4320229724446347 -0.17718855392460764 0.7238001450159828 1.8397437251675461 0.9523656518925097 2.513817935317845
-1 3.7089889925376345 1.6027646547595036 0.30439608816889874 1.325556017740845 1.5649758448214102 2.0480467830712694 1.4268815678658604 -0.08232989657136769 2.0319641149268852 0.4859663282113227
-1 2.9299411753408178 0.6939333819644463 0.5980477746930858 1.1544643358350055 0.5988463132053894 0.8004691945155193 -0.7969681294710653 -1.246477065340748 0.7551153563842066 2.2320600943025157
-1 1.5618544649786017 -1.2039729275512823 1.9863936078958404 -0.7698679015907834 0.6433908271785455 1.7173978058694828 0.8771509209324759 2.664740793299653 -0.6994627263844606 0.6322436483068374
-1 1.187061394437512 -0.6451485516060627 2.476357446033039 1.7693108617562059 1.3697550089364834 0.40908284287939223 -0.5656163253633264 3.468763307766636 1.617455962016709 0.4894706139195705
-1 -0.4273229723387111 -0.26809867009452515 1.3843160982545846 0.8212240154930317 1.1784396971750364 1.872828424638627 1.3779623371802083 1.1888620042820783 -0.10589695125965615 1.4199981576509952
-1 0.12193951392066005 2.616540426567961 -1.337357835943099 -0.10743949585791679 0.3939788495591735 -0.02266440276523496 2.766246408329433 1.779318925725903 1.1626163281228863 1.1568240129972165
-1 1.4669291522156196 -0.8005956562590923 -0.6879775244399986 3.461310058748968 1.1339641121124138 3.0998254868058384 0.245952923446367 0.7214863675143265 1.0108020940282363 1.8538791497646767
-1 0.37376581529952313 0.3065031814805871 1.3343221577395563 -0.36245405167755473 -0.7157134718616156 0.9091314241626773 0.6213443407765016 -0.3159031135243049 1.0607486905684709 -0.2566933833287508
-1 2.0069622762472235 1.3555276909717138 1.3738458420384927 1.3307981771643953 1.1352058939547374 1.1872314739705727 2.0206074946330155 2.6193996043859977 0.9754506254457527 2.4788773949517737
-1 1.6559576152851871 1.5613387714537157 0.9820632656447196 0.24990370738791912 0.6790482468297928 0.7177001456270966 1.2177661518329543 -0.010128389509312274 0.9949778601566439 0.2730735896651332
-1 3.3541347870312084 1.8903267206950842 1.6609607533550115 0.6313086218186583 1.0174443932043256 2.1002778641752133 -0.7433879263515524 3.6635365130163358 -0.12072379016630852 1.2613991803119946
-1 0.741882011562536 -0.33389745909875646 0.49850980476986007 0.6209294892871532 -0.9345674636388526 1.0706987501267613 0.17174378573602178 1.4966350235504806 1.7786390376763213 1.6231643119303771
-1 0.737851271176944 3.1107332677301804 0.5595554860713969 0.03240910648046724 0.7418890189368929 2.5744268937009354 0.08490736311553437 0.9454019320976027 2.3004255005209213 2.673423266074501
-1 0.9964678056269282 -0.4050367214023043 0.7634512054670727 0.6104047048598984 -0.18420038230329872 2.8225484519075694 -0.17480506682904684 1.188578222519793 2.3609744942610704 2.0104954250932927
-1 0.8561825142599002 1.4715100244558175 1.1551932439330008 -0.866432954658839 0.06672467583391328 0.6567191940892094 2.1238239921343776 1.9236498444842514 1.774783717232303 2.1705643226440356
-1 2.1686685144492652 -0.46548035607855187 1.7905868508290022 1.7291739618095732 1.8420059988367683 1.2812869543894454 0.7094922226284579 4.578093325453002 2.159649972834322 -0.703298751877151
-1 0.01038121312435214 2.041036231629956 1.406313867978486 1.3944476209150578 -0.7450794741024422 0.36098991012411563 -0.8145936978526842 1.0085439903773337 0.6693692426324003 0.6121851518794861
-1 1.8571542967953807 1.4070713551879899 0.5321067816124654 0.6429601839486434 0.9165980917544774 1.071305634192637 -0.06040670535870918 2.5384035240078604 -0.21377477606093764 0.3369977088082866
-1 2.405103563655566 -0.4546855764355364 -0.24489042907792635 1.3318409806777944 1.2523408877207844 0.9313587923017596 1.2089956458520745 3.0921428523894092 1.956850142357836 0.7702767453893322
-1 0.9086347130699683 1.2100828227228213 0.5327052367165771 -0.6550532780225489 2.5505664076947587 1.4300751019325881 -0.9806442677198526 1.9110672232516768 1.956204319904626 -0.6406447989012172
-1 1.750246620105648 1.3081292130126525 1.4716986993259968 -0.3042704857661218 0.2354470475646966 -0.6074481355981227 0.9333801721029178 1.3220227127047701 2.0998355566318203 3.340047345554312
-1 0.8132766080998793 0.345182592805539 -0.08434230880799043 0.371975995128044 1.030128701009812 -0.0838490306566615 1.891400724652641 2.133657072232741 2.4719821498192935 0.9603084853474415
-1 1.426463569977554 2.123479869287884 1.8449734404123337 0.8841571967965259 1.3206820715765568 2.414835584218742 1.129163483268984 -0.8781190476518506 1.5162895167347454 -0.6528866908043633
-1 1.2017423534681941 1.9686754970835203 1.3014044708959847 -1.0240935923675734 0.7502387139905979 0.8253575777839712 1.224646644221756 1.480689489076607 1.7640815996729344 0.2056821278829375
-1 2.7250146939462083 2.227656483011149 2.84947399343455 2.451014425645574 -0.3739053762247364 1.1582450151950303 1.741290414111453 1.376435447217923 0.35033655530431784 0.4806336989868223
-1 1.3542581369916695 0.415546436380271 0.6688613033041042 0.9102881456111578 0.2547986420844246 1.378444594707075 3.43963729226003 1.3067301378198568 1.5647303411064155 2.043293980780698
-1 1.0913358352352922 2.1175733214306947 0.929020839478381 3.090469607746358 0.09151751891798587 1.5634842729294367 1.8016069710014775 1.4861336762215835 1.6076296539436097 -0.26097034661822094
-1 -0.709300017934053 -0.14570511438959777 0.8487791028889955 -0.3957122997819824 0.23663565146376286 2.66035473479832 2.1479897842790923 1.2106691413007877 -0.45712691497148206 2.4225765811823203
-1 0.14756832470608838 2.3704041393692425 0.6496201584931938 -0.11807063222136005 -0.20506086896030706 1.5881151061076393 3.797132222832481 0.943542745977901 0.8565267747881888 1.1864294682583807
-1 -0.3889342935852145 -0.17743324011571104 1.3604682904339318 0.6593714174698198 -0.3584830057001256 3.514136269889732 0.595913513718282 0.1683068614180695 2.0746193584112143 0.6903921573893614
-1 0.2920446897752229 2.9937346155977957 2.251247553131803 0.6975169699248711 0.4494567463916379 1.319277335273955 0.5367328026447278 2.5267557692090836 0.350600102811225 0.5606888320387985
-1 1.228653481176321 1.0182555282617969 -0.5982787788962058 2.6333900117968314 2.0366003161170663 0.5499289981699178 2.542904251265296 2.2146577311919637 0.3954898163391639 0.6205263945903541
-1 -0.0520426119593238 1.590564747318753 1.6958053948956031 1.3511042599706389 -0.047969026912866974 0.55701288765553 0.9263968623271992 0.590838546777129 2.3308650721102633 0.5135257132439688
-1 1.016635594241282 1.8948650280358326 1.440434304566253 1.4592759362683134 1.6827383192498666 -1.0918246492897437 0.43238661798429845 1.5624487435653098 2.220285861909854 1.271128145985624
-1 -0.7222589043422267 0.5115698429182437 1.3516909750379982 1.6184323538658458 0.3138663124851314 -0.02913500500520727 0.8551827087816364 1.6317432725857857 0.6646228309777373 1.886929067576903
-1 1.4628654761642204 1.8652907041028732 0.6622303129185922 0.7509202647315306 -0.036376585463356426 0.7850159634599014 2.2985430427240017 1.0460715145011406 0.8526933674534585 1.1533090709516742
-1 1.0669747034293164 -0.1510400394042828 -0.34893623474816793 1.7754617342041603 1.3436972220233374 3.022419531056307 1.9684180926734447 1.4858550357170357 2.9588700999527395 -0.02437800790558642
-1 0.5379644371164043 -0.27906681292084 0.3380177280655655 0.33722013060203193 0.6571438211538795 1.2052933591547657 1.7731403611930516 0.5077273284789499 1.5626883295465674 -0.050171508356717576
-1 1.2224363031291428 2.179387632259403 1.729844754655598 1.7261086434406607 1.6565721133198088 1.889839925928689 1.8345686999088797 1.051447084834809 0.9359370646456183 0.7645291821631122
-1 2.60292814182841 0.8804157611166004 -0.955075955060207 1.2946117062161222 2.107044588585438 0.2497683006856819 1.6038124754155476 -0.7214552551237594 0.452098771396898 0.6986965061465407
-1 1.0412661702670807 -1.3958762787534025 3.074541266637782 1.76411325380808 -0.39903368929064653 1.3136620541582826 1.1746725568355456 -0.6576469095064521 0.15286303171879478 2.117286307501297
-1 0.31859147805604837 1.2450573919933268 -0.5933863589583486 1.616822450960686 2.3307511175574707 1.4675892671924506 -0.6797208500497198 -0.6357164936808151 2.6616070340209608 0.12503414768311838
-1 0.015640995722970286 0.9521770024879528 -0.021136921124242036 1.5781474391889052 0.7227013060272598 0.7987343733885311 -0.6768705185766593 1.2194260902982417 0.6115575336879959 1.776636860101025
-1 1.7473265876837165 -1.3416662707254097 -0.3178957317552682 -0.7952748363966 -0.0012367493892466719 1.5102140866553868 1.3893554303705593 1.253090374551591 0.37849714433826975 3.8427708908843417
-1 0.1249935088342321 0.9175321556781342 1.2521433252052363 0.10448935908110157 1.748729859258747 1.9013556247400216 2.348145639899152 0.4626753070549736 3.7821319980165344 0.47822934584228827
-1 1.5461491524524733 1.0442419265941036 -0.016418025211677234 -0.6189521317249826 0.9719604409404735 1.1409654487054224 0.5144932080563054 1.677400744669605 1.60852217407324 0.9996875540653996
-1 1.1571589981163284 2.815325710919601 0.20772173229184132 -0.27577989741307296 0.14104944330527658 0.2590225341905401 -0.33859238160667027 2.803757221911037 1.035764969030257 0.16925873998127916
-1 1.8759906736161591 -0.7858122581388844 1.0848147823038492 1.346569014348389 -0.7811951242276918 -0.28091748058441146 0.10734544787850497 1.1946024654289003 1.6406107469177638 1.418186454569726
-1 -0.2974414971504451 -0.7263225506198576 1.667022614186794 1.1033345452667596 -0.2451904831865781 -0.011381119202380274 -0.2081120315941396 0.19505925177058225 1.083883779309256 0.2476147974455678
-1 1.9875844064011776 -1.0551408447589177 0.9235522752742322 -0.1465157757078015 -0.24048981040870454 -0.3751333753617203 1.6243406244366847 -0.38149309424785227 -0.2845380129435624 -0.4586888921471284
-1 -0.43391027275254457 1.3012041634540212 0.34931152784647057 0.2724840573311986 1.895997027401461 0.7955372939424181 2.717841382622603 0.9983211958138658 3.297958269369362 0.28612843397709364
-1 0.09388869926828014 0.7292780962393748 -0.48425219833973965 1.2122506447105803 0.7074049606666732 1.0448613427298579 1.4758560188256675 -0.32361188073438485 2.040268428137505 1.685468904484563
-1 1.0792167846288987 -0.2826348408764243 1.3133025554220168 -0.29264376303967365 0.12334584816456384 1.7916405818476433 2.4401329350478367 1.373668417749465 1.1438238823893943 2.9513159396946955
-1 0.6272602458353195 0.012788348875383604 3.339583303835828 -0.5656471248096915 1.7436358009297308 -0.0849133378284781 1.8766630914593128 0.3286471991737121 0.8557785757636693 1.204343384424849
-1 0.9053623358277365 2.851790381485327 1.0805997920016692 -0.5635383000263379 0.9576644151670836 1.9289302434370748 -0.13805339731578536 3.4861795141210807 0.2005081416731367 1.6544819624039082
-1 0.4910096613955415 1.6681822364133903 0.8202936721704033 2.148200954440342 2.558162860929867 0.6606047330906034 0.7989603259919102 1.0689702044523541 0.7184320065316048 2.023034231513219
-1 1.1256411487276385 0.19900785835501755 1.2085575135898547 -1.356418780267496 0.785218957218392 2.70677848091574 1.9987708656840728 0.6868097252341125 -1.241646154239319 2.9393145029129917
-1 1.9337642982267669 -0.7156557544578908 0.16408179712477566 1.9408268646309592 1.0190820244131475 1.1951052545533123 0.4481509783235238 1.2668590723499928 0.8102310436768919 0.7718152165895394
-1 1.614923882092461 0.19469602471151815 3.766869874799438 -1.3377164159484254 -0.878559530240216 0.3364262245077355 1.8010436667360947 1.777688731609198 2.311140988026292 1.1771602185088652
-1 0.6784758917678138 -0.18464751605809093 1.6835398190359525 0.9616873095363908 1.8625881930711616 1.9970275330538905 1.0465679673330561 1.7874857759504277 1.7797672480031759 0.9806567017840313
-1 1.9543101838028707 -0.44413349405470304 0.3787949477054693 0.09081285199753486 2.460919892284841 0.29445632839265967 0.9120233970904723 1.120046161146032 0.3979415181383884 1.6677498018942478
-1 2.7931886788791984 0.05569901049144255 1.2190718219058607 1.3326923562520578 1.7863786156200971 1.8057619970370333 0.9782497583237075 1.1631245252370526 -0.10647683276082942 0.8291413719741013
-1 0.6746786109931104 0.693150020176567 0.8806942321642721 1.3171663922040504 -0.18964506284133353 1.752816912385852 0.0197418639082243 0.04087366490530042 -0.31356701603876047 1.1688888267402135
-1 -0.8047119894089716 -0.19086822099982692 0.7230280053386025 0.47661575325565886 2.783553868954165 0.39034536568120837 2.4620798409550657 0.3460544872000194 1.6811241975213127 -0.5755589941181993
-1 -0.43736971419082993 0.9731234165917454 0.044303702104787734 1.3285736602137515 1.8134256070231687 4.003995098206477 -0.5823423595861437 1.1000778881670024 2.275332508162996 1.7059404281570498
-1 2.7870499907770374 1.5359115092914868 0.4415840592158585 3.0819184178594012 1.0142235114013434 1.4175457438753696 0.7830675289154578 0.718110803107776 1.752603937821668 0.8681755199560836
-1 1.6629646464798866 1.5720752857585811 1.866918319229821 2.011503983207959 -0.08953127029042407 3.250764941529524 0.8681970712263898 1.8122090555675 0.30361209115382115 1.6190898270526166
-1 0.8689387257925889 1.088532128821611 -0.9638248404112064 -0.03629852962978575 1.5819544244821397 0.533196869581712 1.1629368405935705 0.5952984584910554 0.5901966383762997 0.8680425050414964
-1 0.5657393409043414 0.1269546832382663 -4.0341609669503065E-4 1.1489057321179976 0.25156572912668473 0.48265829258343707 1.051802672080171 -0.797907065268961 0.40336920791124586 0.34951103336108913
-1 2.842259431863403 0.4523061399118463 1.1073417696817962 0.820613792637092 1.2347466769629105 2.445490993196761 -0.1542908283123816 0.8816264920520589 1.7423151819076375 1.6594291913667136
-1 1.5860855260228202 2.8392671863491734 0.5188572450043611 1.047507505252711 3.054126605012979 -0.6006852937930467 0.34982369626834076 0.11607093207054109 1.829510982388106 0.001994427476862848
-1 0.17902283956677512 0.41558050427565774 1.5871923905064695 1.5996558530208187 0.07353003075760078 1.0705630115074813 2.675599132354674 0.7650850730679759 0.8607570887706816 0.9903122299033713
-1 0.7379554955291575 2.072325148209555 0.4462636170973716 0.6880836555742617 0.3535374515580053 0.19240929522338934 2.2791306741261153 1.7199300904991563 2.3790655960655718 -0.4294392660855837
-1 0.5642895627754023 0.9044762545519158 1.4797756442552041 0.6976030137900451 2.5013240752661825 0.8121543920897196 1.864316073466811 1.3213558088397361 2.17814424984865 1.8979547805463015
-1 1.103147738753372 1.616958446219673 2.8479619253624797 3.368348617090012 2.5438833831666434 1.6704650810547208 0.8562521160479526 0.7542938264829215 0.5266574196400498 -0.2890730154742367
-1 1.9142555817765898 0.8049202262783679 2.5019528805928912 0.5238106873271193 1.5359406981988988 2.8356323728714847 3.239716573932437 1.2510518752596296 1.715571851101242 1.222780980267732
-1 0.6041885893884307 0.5707299204297884 1.2540953158421435 1.5510759633503302 -0.4667440237195346 0.26676051631424014 -0.565572799800238 1.4387028778945943 0.9644694652315191 2.1255685675532967
-1 1.7491189390587218 1.2227275279214738 -0.8505836769821726 -0.903216529384467 0.29076052330579005 0.2892222629138922 2.3647508720986217 1.2652921314867005 1.0348376197540348 -0.2562195481430878
-1 2.3800831934663433 -0.010431805594117938 0.8430880161541389 1.278733772829872 1.585905177486663 0.28093811664192425 1.5849634563502026 1.078413585522204 0.4426572711340797 0.6530352928058241
-1 1.7049361022681717 -0.27653366462628215 0.9445796767766628 0.041969783781791725 0.3467762982688263 -0.4874473134901387 0.7531152429497019 0.30167927793354254 2.765258841783637 -0.23618185513880707
-1 0.8097421163995955 0.17729598233902988 2.5214858992792863 1.5180096630697852 1.9899028361613595 0.57436615658855 0.5307905908280097 0.9190155285250498 0.6466076660416842 -0.10626054479014013
-1 2.395022852849255 2.3321432458593208 1.6804528385827555 2.2258435456318937 1.4611936535655663 1.058998523699314 0.31838562794784586 0.39659928716273496 1.4494935872166117 1.391374864616476
-1 1.735291612472487 -0.3191446365558481 0.6607372043463824 1.541446196262466 0.4947578059034823 -0.8293819909066149 0.76596276473359 -0.0851263113957168 1.9200627040331277 1.5173271962047457
-1 0.48007434755469713 0.7936351950677151 1.365699852551887 1.1109515050883414 -0.12031241802004855 -0.18610833660205306 0.2974034656359261 1.3687489920730513 2.1059823724132523 0.941953020877809
-1 2.4520203316077964 1.11003521338105 0.4722773485870979 2.737384705503226 0.7192036221774767 0.6242245483941781 1.2609692406366446 2.0575095746651133 1.3495884659991346 2.0764197346896935
-1 -0.7842236897873944 1.492890069052242 1.765349236922137 1.300042277956386 1.5799338298744416 1.060819121020154 1.1674652333797013 -0.4149263766035056 0.09348961754442264 3.5461008823168543
-1 0.8620605536733185 0.08406312778559633 1.5415557685694021 0.2051913612441839 0.19504752604759068 1.534576255114414 3.107649420779101 1.020214612080108 0.3221723632541289 1.4874661690065234
-1 1.489728417116672 0.06558708406688907 -1.8670045751011424 1.7828483838262912 -0.683023788962926 1.79761793764676 1.5085129455490893 1.2434470961660735 0.5774571270514824 1.4932340982697638
-1 -1.5669127739356443 0.34356934741624334 3.0594253296534424 0.774762761699532 1.0055392162451373 1.3241023069988664 1.1749986092813367 2.19297533155391 1.0435550797072737 2.095514184709966
-1 -0.3634276403952408 1.4409978371532932 0.3823184763192483 0.6254885387609036 -0.35123251562864244 1.819196851350437 2.14116717870738 0.46320929513337494 0.5695755038115515 2.501714843566015
-1 0.013632028077828595 1.8215490521966027 1.7653867346915684 1.4163095749484134 0.25841398470159227 2.2048024054278192 0.9286824219992222 1.133706943250312 1.7330998187732773 1.3552028632095436
-1 1.012536342646575 1.4202805284853588 1.1660963924281333 2.7434608590955594 2.405339566810934 0.35678139532687714 0.7007075773809261 -0.1461824532706133 -0.1116775801341563 2.455669156783493
-1 1.7224210079670872 0.25824562782106875 1.896388948392676 1.5490245294926566 0.566495628127113 1.4439902246901806 -1.1659487820432086 1.2648317293133733 -0.8687762383751962 2.055108054071261
-1 3.5125527162365486 -0.022436189584495336 1.1332983732450903 -0.07450694962415794 0.09001591132041731 0.5853417525905302 3.337681467433381 -0.32222401787392774 2.539181628048545 1.0754745872100386
-1 0.2455099848454918 1.2693508037734986 1.6546347888138584 -2.148792530729241 0.46441142559185566 1.1734134286137057 1.0258039884088828 -0.5586646913499485 -0.3258731206571115 -0.821219883870792
-1 1.827217125452903 1.731864545109457 0.928872208086588 1.2056977735867256 1.818214291632629 0.6585878488136441 1.8002230735809155 0.8708150904043206 -1.5838120389612023 0.8585857536471672
-1 2.2021363682137154 0.4761145331093257 -0.025920931323458296 1.7449566792074553 0.8629966232032906 1.4723084204343524 1.6159540778305606 2.029453834185225 2.26325946376582 1.376244768900244
-1 0.010342658978543584 1.515273076994554 0.19611635441491626 1.654784841440513 -0.033943991780339244 0.6714632219862774 0.2641936457650498 -0.700825233754335 0.23452605282080619 1.621398184902529
-1 1.0480165819981573 0.8797819263901776 -0.641443663240362 0.12817609127433438 1.3647120235220283 -0.48615470921060977 1.0720144074421256 -0.38026314794700733 0.8069083073855456 1.3433152284915995
-1 0.3761857330260455 0.23219703324626284 1.921560210024654 0.38896862067672255 1.1468761246542036 0.8203362705962437 -0.23996402764305458 1.5950906570841252 3.639574852127676 -0.2443366415192889
-1 0.8759552320204246 0.33529291747248857 -0.2551391418074267 0.29090645845832075 -1.1529071816719476 0.7412858224772877 1.2719555749592364 1.3289131183268248 1.3228711885726534 1.5021325652417783
-1 0.439646111605676 0.8753273571625453 -0.5195310985749739 2.656469182704334 0.8907416242841371 1.4150606950578886 3.175298549230411 0.44910268745784754 0.8447367653706002 1.668648718911232
-1 1.1404102468865998 1.4857266483300324 -0.31291554366933605 1.3205568580259288 2.4092775306975023 1.6397731783027976 1.1251407071414252 2.3565497583137436 1.8353622317028138 -1.1683108743275552
-1 2.08122023149769 1.1571239260956436 -0.08056173908716335 0.768249986206349 1.3171573148662759 -0.18023949555734187 -0.25107977208536614 0.3528408329964078 0.7749381509220793 -0.7113421449812265
-1 0.1473845257811165 -1.0521567114122852 -0.47637816156748225 1.4949699096476212 2.271087115324705 1.3826153478446757 2.7436405167916025 -0.02075677223859529 1.1765040243159015 -0.025438785956181542
-1 2.7027482205114826 1.577562959861571 -0.5669337503778331 1.5215534981321372 1.2652067920381662 2.7463387790797444 -0.10995208915345178 -0.9887358827125716 0.7108329384066776 1.3629285100379036
-1 2.9573936017540556 0.1614860515756119 -0.3278573695860796 1.0550562822356224 1.4787913549079965 1.6928275048278305 1.0586362008998798 1.1651361732301 2.361382321862904 2.524722697822938
-1 -0.918683252112166 1.1912188403555544 -0.6386682219001243 0.12852707081177273 1.0186959070915036 -0.7396656648881279 1.390222924345315 -0.6776334559974988 1.6871484268646286 0.9835794195231572
-1 -0.9501651670329723 1.6369415588995389 0.6124916702658543 2.055786019572368 0.20091594691375603 0.27955238962400497 1.8462485957757835 0.766850497882725 0.6439523544318226 -0.45529021581249385
-1 0.08294835997064665 -0.27721496031157833 -0.35456350243850276 0.11228054309930591 3.4737188479123104 0.8438116500646802 1.2682583387249549 2.2187948258289913 1.6181904099869335 2.2762749025565983
-1 1.83339856452743 2.673091344347915 0.7389331991568107 2.067911927048983 1.3782410940205578 2.030974790626103 0.6888073746059981 -0.518101069445974 0.6230936256620102 1.633224100697245
-1 1.7398691778151973 1.1247533360425708 0.2807774763651275 -0.6955611341182046 1.592036824083598 -0.04050352181158767 1.3865010706574772 1.4019929481612587 -0.2642443959402707 0.5934301817863643
-1 -2.019173847473457 2.1681048424611418 1.3422907243645614 0.6467676712250852 0.49642291457381404 1.289806437146178 0.5287383514431835 2.8692305624115457 0.37484482468477054 2.4484351720405875
-1 0.024288362749408376 1.0351720632502537 1.6837605528916666 1.3872579136738206 1.2679651380538202 1.4021182744167016 -0.7041852642469104 1.6806756125489901 0.1307750250742319 2.3317291973580314
-1 -0.06080175616636896 1.0543357215752764 2.099562273809995 0.6174473985800795 0.5458218639483579 -0.1330076265446425 1.782807067124061 3.835868752952487 1.0749746574622228 2.2318191600680155
-1 2.7819388327740797 1.1294517177544148 1.4625685601160094 0.8160359631571115 1.5866067958993928 3.0076062737914184 1.5740992429858394 1.3901837375360109 2.7120095549614893 -0.5329184800190412
-1 -0.08342899095133993 3.2552165445304735 -0.6127389181137219 0.20728621073827602 1.1715077138725913 0.496873621214974 0.7991470651533773 0.5625481785655475 0.7904628851956959 0.485293468158445
-1 0.5879363673253968 0.5480289705171163 0.26878358296170424 0.9493365691333653 0.34421794272116246 1.4045876345319372 0.8323003475233924 1.3822841645472739 1.9408510354113169 2.3160979297534636
-1 2.049725023995715 1.138714228201635 2.228635558152831 1.4833354495511806 0.5549789742701208 1.3850264438047617 1.4418684507619366 3.131909530291612 3.2277156524053705 0.5657214292376471
-1 0.7278339716721132 0.8342775647290255 -0.7804056350094557 1.8999099617115354 1.5129989349558883 1.6238396258236993 -0.13761070763179828 0.6429461405182848 -0.2642956636249272 0.8065034962137944
-1 2.5931023834096854 0.9018261137939111 1.5584456516926881 -0.5802390356360938 1.941618818488975 0.9214260344294213 0.556884632504891 0.26832249168681577 2.4966263079255677 1.1243846486761992
-1 0.14419967158797142 0.9874339005630041 0.8076366869263152 0.515723994659785 -0.9385248237540935 -0.17924876736882722 1.1150091706474443 1.5543894995228547 1.615026336442979 1.1708620595483625
-1 2.1530687310737866 -1.8203657185808888 0.6380519600335401 2.02809789647314 0.30946138948160296 1.7692953099290327 1.0369557864170398 0.3326256746163322 -0.275581422683832 0.21583516634100164
-1 0.896534730391731 2.1309314580821708 0.9688774738233893 0.7810503130534793 1.3417441924762596 0.10748935054015485 0.8725839981470569 2.68470748226214 0.5000051011542708 1.6309858671990054
-1 0.2798388059875424 0.46301766350582063 -0.21330838748068315 1.516256000433057 -0.9521989902404524 1.8668922242244914 -1.429783656173199 0.24500379527846305 1.0717746705573634 2.929223328366103
-1 1.5580038958637812 1.4690967454818293 3.5043865357520065 0.8077006250670602 1.70873452721819 1.725133865080763 -0.17803725982825802 1.2072416111273427 0.7258484330322263 0.9666451576387228
-1 -0.2937927716783808 2.209449837105502 2.471323239279583 1.9931843786987273 0.4670001618859797 1.2200671907651737 1.3884758303330187 1.1014939571310298 1.2017172341718294 2.657179062084367
-1 0.9402246743347112 0.40154461288043775 3.407916599846658 0.732993794216273 0.7120872061718131 0.7443371156456304 0.261691914047522 -1.7816254435328527 1.1872515149455043 1.2859514985608926
-1 1.5116064491281778 2.2468889028407437 0.45828491922709613 1.2192147082911882 0.6354365593721796 -0.2656322662271462 0.22961524227015095 0.6580482520092654 0.8557895993898526 1.1404110974520998
-1 2.738506436693102 1.129940083852354 -0.2531479159181209 -0.3313565595449408 2.157889045868747 0.7757459702743189 2.5165730696859523 -0.5
<TRUNCATED>
[16/50] [abbrv] incubator-predictionio git commit: [PIO-56] No-setup
unit tests for core and data
Posted by ch...@apache.org.
[PIO-56] No-setup unit tests for core and data
Closes #355
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/cc4e2e0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/cc4e2e0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/cc4e2e0a
Branch: refs/heads/master
Commit: cc4e2e0a2303962f2b2112ec7612f39579786439
Parents: 9bbd1f5
Author: lucasbm88 <lu...@gmail.com>
Authored: Thu Jul 27 11:40:19 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Jul 27 11:40:19 2017 -0700
----------------------------------------------------------------------
core/build.sbt | 5 +-
.../controller/EvaluationTest.scala | 6 +-
.../controller/MetricEvaluatorTest.scala | 6 +-
.../apache/predictionio/workflow/BaseTest.scala | 59 ++++++++++++++++--
.../workflow/EvaluationWorkflowTest.scala | 5 +-
data/build.sbt | 5 +-
.../predictionio/data/storage/Storage.scala | 41 +++++++++++--
.../data/api/EventServiceSpec.scala | 39 ++++++------
.../data/api/SegmentIOAuthSpec.scala | 36 ++++++-----
.../data/storage/StorageMockContext.scala | 64 ++++++++++++++++++++
10 files changed, 211 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/build.sbt
----------------------------------------------------------------------
diff --git a/core/build.sbt b/core/build.sbt
index b95a957..14b3449 100644
--- a/core/build.sbt
+++ b/core/build.sbt
@@ -30,7 +30,10 @@ libraryDependencies ++= Seq(
"org.scalaj" %% "scalaj-http" % "1.1.6",
"org.slf4j" % "slf4j-log4j12" % "1.7.18",
"org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.specs2" %% "specs2" % "2.3.13" % "test",
+ "org.scalamock" %% "scalamock-scalatest-support" % "3.5.0" % "test",
+ "com.h2database" % "h2" % "1.4.196" % "test"
+)
parallelExecution in Test := false
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala b/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
index a64826c..b60d358 100644
--- a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
@@ -17,12 +17,10 @@
package org.apache.predictionio.controller
-import org.apache.predictionio.workflow.SharedSparkContext
-
+import org.apache.predictionio.workflow.{SharedSparkContext, SharedStorageContext}
import org.scalatest.FunSuite
import org.scalatest.Inside
import org.scalatest.Matchers._
-
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
@@ -42,7 +40,7 @@ object EvaluationSuite {
class EvaluationSuite
-extends FunSuite with Inside with SharedSparkContext {
+extends FunSuite with Inside with SharedSparkContext with SharedStorageContext {
import org.apache.predictionio.controller.EvaluationSuite._
test("Evaluation makes MetricEvaluator") {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala b/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
index 9aec6d5..165fc63 100644
--- a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
@@ -18,8 +18,7 @@
package org.apache.predictionio.controller
-import org.apache.predictionio.workflow.SharedSparkContext
-import org.apache.predictionio.workflow.WorkflowParams
+import org.apache.predictionio.workflow.{SharedSparkContext, SharedStorageContext, WorkflowParams}
import org.scalatest.FunSuite
object MetricEvaluatorSuite {
@@ -30,7 +29,8 @@ object MetricEvaluatorSuite {
object Evaluation0 extends Evaluation {}
}
-class MetricEvaluatorDevSuite extends FunSuite with SharedSparkContext {
+class MetricEvaluatorDevSuite extends FunSuite with SharedSparkContext
+with SharedStorageContext {
import org.apache.predictionio.controller.MetricEvaluatorSuite._
test("a") {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala b/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
index 004dbb1..59d23f1 100644
--- a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
@@ -15,20 +15,22 @@
* limitations under the License.
*/
-//package org.apache.spark
+// package org.apache.spark
package org.apache.predictionio.workflow
-import _root_.io.netty.util.internal.logging.{Slf4JLoggerFactory, InternalLoggerFactory}
+import _root_.io.netty.util.internal.logging.{InternalLoggerFactory, Slf4JLoggerFactory}
+import org.apache.predictionio.data.storage.{EnvironmentFactory, EnvironmentService}
import org.scalatest.BeforeAndAfterAll
import org.scalatest.BeforeAndAfterEach
import org.scalatest.Suite
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
+import org.scalamock.scalatest.MockFactory
/** Manages a local `sc` {@link SparkContext} variable, correctly stopping it
* after each test. */
-trait LocalSparkContext
+trait LocalSparkContext
extends BeforeAndAfterEach with BeforeAndAfterAll { self: Suite =>
@transient var sc: SparkContext = _
@@ -43,7 +45,7 @@ extends BeforeAndAfterEach with BeforeAndAfterAll { self: Suite =>
super.afterEach()
}
- def resetSparkContext() = {
+ def resetSparkContext() : Unit = {
LocalSparkContext.stop(sc)
sc = null
}
@@ -60,7 +62,7 @@ object LocalSparkContext {
}
/** Runs `f` by passing in `sc` and ensures that `sc` is stopped. */
- def withSpark[T](sc: SparkContext)(f: SparkContext => T) = {
+ def withSpark[T](sc: SparkContext)(f: SparkContext => T) : Unit = {
try {
f(sc)
} finally {
@@ -90,3 +92,50 @@ trait SharedSparkContext extends BeforeAndAfterAll { self: Suite =>
}
}
+trait SharedStorageContext extends BeforeAndAfterAll { self: Suite =>
+
+ override def beforeAll(): Unit ={
+ ConfigurationMockUtil.createJDBCMockedConfig
+ super.beforeAll()
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ }
+
+}
+
+object ConfigurationMockUtil extends MockFactory {
+
+ def createJDBCMockedConfig: Unit = {
+ val mockedEnvService = mock[EnvironmentService]
+ (mockedEnvService.envKeys _)
+ .expects
+ .returning(List("PIO_STORAGE_REPOSITORIES_METADATA_NAME",
+ "PIO_STORAGE_SOURCES_MYSQL_TYPE"))
+ .twice
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_NAME")
+ .returning("test_metadata")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_SOURCE")
+ .returning("MYSQL")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_SOURCES_MYSQL_TYPE")
+ .returning("jdbc")
+
+ (mockedEnvService.filter _)
+ .expects(*)
+ .returning(Map(
+ "URL" -> "jdbc:h2:~/test;MODE=MySQL;AUTO_SERVER=TRUE",
+ "USERNAME" -> "sa",
+ "PASSWORD" -> "")
+ )
+
+ EnvironmentFactory.environmentService = new Some(mockedEnvService)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala b/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
index 26fc936..70534e4 100644
--- a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
@@ -18,11 +18,12 @@
package org.apache.predictionio.workflow
import org.apache.predictionio.controller._
-
+import org.scalamock.scalatest.MockFactory
import org.scalatest.FunSuite
import org.scalatest.Matchers._
-class EvaluationWorkflowSuite extends FunSuite with SharedSparkContext {
+class EvaluationWorkflowSuite extends FunSuite with SharedStorageContext
+ with SharedSparkContext with MockFactory {
test("Evaluation return best engine params, simple result type: Double") {
val engine = new Engine1()
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/build.sbt
----------------------------------------------------------------------
diff --git a/data/build.sbt b/data/build.sbt
index b6b2410..abfa74f 100644
--- a/data/build.sbt
+++ b/data/build.sbt
@@ -28,7 +28,10 @@ libraryDependencies ++= Seq(
"org.clapper" %% "grizzled-slf4j" % "1.0.2",
"org.json4s" %% "json4s-native" % json4sVersion.value,
"org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.specs2" %% "specs2" % "3.3.1" % "test"
+ exclude("org.scalaz.stream", s"scalaz-stream_${scalaBinaryVersion.value}"),
+ "org.scalamock" %% "scalamock-specs2-support" % "3.5.0" % "test",
+ "com.h2database" % "h2" % "1.4.196" % "test")
parallelExecution in Test := false
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
index 744a5e6..fd05767 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
@@ -112,6 +112,33 @@ class StorageException(message: String, cause: Throwable)
def this(message: String) = this(message, null)
}
+class EnvironmentService{
+
+ def envKeys(): Iterable[String] = {
+ sys.env.keys
+ }
+
+ def getByKey(key: String): String = {
+ sys.env(key)
+ }
+
+ def filter(filterExpression: ((String, String)) => Boolean): Map[String, String] = {
+ sys.env.filter(filterExpression)
+ }
+}
+
+object EnvironmentFactory{
+
+ var environmentService: Option[EnvironmentService] = None
+
+ def create(): EnvironmentService = {
+ if(environmentService.isEmpty){
+ environmentService = new Some[EnvironmentService](new EnvironmentService)
+ }
+ environmentService.get
+ }
+}
+
/** Backend-agnostic data storage layer with lazy initialization. Use this
* object when you need to interface with Event Store in your engine.
*
@@ -123,6 +150,8 @@ object Storage extends Logging {
client: BaseStorageClient,
config: StorageClientConfig)
+ var environmentService: EnvironmentService = EnvironmentFactory.create
+
private case class DataObjectMeta(sourceName: String, namespace: String)
private var errors = 0
@@ -131,7 +160,7 @@ object Storage extends Logging {
private val sourceTypesRegex = """PIO_STORAGE_SOURCES_([^_]+)_TYPE""".r
- private val sourceKeys: Seq[String] = sys.env.keys.toSeq.flatMap { k =>
+ private val sourceKeys: Seq[String] = environmentService.envKeys.toSeq.flatMap { k =>
sourceTypesRegex findFirstIn k match {
case Some(sourceTypesRegex(sourceType)) => Seq(sourceType)
case None => Nil
@@ -152,7 +181,7 @@ object Storage extends Logging {
private val repositoryNamesRegex =
"""PIO_STORAGE_REPOSITORIES_([^_]+)_NAME""".r
- private val repositoryKeys: Seq[String] = sys.env.keys.toSeq.flatMap { k =>
+ private val repositoryKeys: Seq[String] = environmentService.envKeys.toSeq.flatMap { k =>
repositoryNamesRegex findFirstIn k match {
case Some(repositoryNamesRegex(repositoryName)) => Seq(repositoryName)
case None => Nil
@@ -175,8 +204,8 @@ object Storage extends Logging {
repositoryKeys.map(r =>
try {
val keyedPath = repositoriesPrefixPath(r)
- val name = sys.env(prefixPath(keyedPath, "NAME"))
- val sourceName = sys.env(prefixPath(keyedPath, "SOURCE"))
+ val name = environmentService.getByKey(prefixPath(keyedPath, "NAME"))
+ val sourceName = environmentService.getByKey(prefixPath(keyedPath, "SOURCE"))
if (sourceKeys.contains(sourceName)) {
r -> DataObjectMeta(
sourceName = sourceName,
@@ -244,8 +273,8 @@ object Storage extends Logging {
Option[ClientMeta] = {
try {
val keyedPath = sourcesPrefixPath(k)
- val sourceType = sys.env(prefixPath(keyedPath, "TYPE"))
- val props = sys.env.filter(t => t._1.startsWith(keyedPath)).map(
+ val sourceType = environmentService.getByKey(prefixPath(keyedPath, "TYPE"))
+ val props = environmentService.filter(t => t._1.startsWith(keyedPath)).map(
t => t._1.replace(s"${keyedPath}_", "") -> t._2)
val clientConfig = StorageClientConfig(
properties = props,
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala b/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
index 988146e..7a45ca1 100644
--- a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
+++ b/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
@@ -18,40 +18,41 @@
package org.apache.predictionio.data.api
-import org.apache.predictionio.data.storage.Storage
-
+import org.apache.predictionio.data.storage.{Storage, StorageMockContext}
import akka.testkit.TestProbe
-import akka.actor.ActorSystem
-import akka.actor.Props
-
+import akka.actor.{ActorRef, ActorSystem, Props}
import spray.http.HttpEntity
import spray.http.HttpResponse
import spray.http.ContentTypes
import spray.httpx.RequestBuilding.Get
-
import org.specs2.mutable.Specification
class EventServiceSpec extends Specification {
val system = ActorSystem("EventServiceSpecSystem")
- val eventClient = Storage.getLEvents()
- val accessKeysClient = Storage.getMetaDataAccessKeys()
- val channelsClient = Storage.getMetaDataChannels()
-
- val eventServiceActor = system.actorOf(
- Props(
- new EventServiceActor(
- eventClient,
- accessKeysClient,
- channelsClient,
- EventServerConfig()
+ def createEventServiceActor: ActorRef = {
+ val eventClient = Storage.getLEvents()
+ val accessKeysClient = Storage.getMetaDataAccessKeys()
+ val channelsClient = Storage.getMetaDataChannels()
+
+ system.actorOf(
+ Props(
+ new EventServiceActor(
+ eventClient,
+ accessKeysClient,
+ channelsClient,
+ EventServerConfig()
+ )
)
)
- )
+ }
+
"GET / request" should {
- "properly produce OK HttpResponses" in {
+ "properly produce OK HttpResponses" in new StorageMockContext {
+ Thread.sleep(2000)
+ val eventServiceActor = createEventServiceActor
val probe = TestProbe()(system)
probe.send(eventServiceActor, Get("/"))
probe.expectMsg(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala b/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
index a276c59..5927824 100644
--- a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
+++ b/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
@@ -17,17 +17,18 @@
package org.apache.predictionio.data.api
-import akka.actor.{ActorSystem, Props}
+import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.TestProbe
import org.apache.predictionio.data.storage._
import org.joda.time.DateTime
+import org.scalamock.specs2.MockContext
import org.specs2.mutable.Specification
import spray.http.HttpHeaders.RawHeader
import spray.http.{ContentTypes, HttpEntity, HttpResponse}
import spray.httpx.RequestBuilding._
import sun.misc.BASE64Encoder
-import scala.concurrent.{Future, ExecutionContext}
+import scala.concurrent.{ExecutionContext, Future}
class SegmentIOAuthSpec extends Specification {
@@ -78,23 +79,26 @@ class SegmentIOAuthSpec extends Specification {
}
}
- val channelsClient = Storage.getMetaDataChannels()
- val eventServiceActor = system.actorOf(
- Props(
- new EventServiceActor(
- eventClient,
- accessKeysClient,
- channelsClient,
- EventServerConfig()
+ val base64Encoder = new BASE64Encoder
+
+ def createEventServiceActor(): ActorRef = {
+ val channelsClient = Storage.getMetaDataChannels()
+ system.actorOf(
+ Props(
+ new EventServiceActor(
+ eventClient,
+ accessKeysClient,
+ channelsClient,
+ EventServerConfig()
+ )
)
)
- )
-
- val base64Encoder = new BASE64Encoder
+ }
"Event Service" should {
- "reject with CredentialsRejected with invalid credentials" in {
+ "reject with CredentialsRejected with invalid credentials" in new StorageMockContext {
+ val eventServiceActor = createEventServiceActor
val accessKey = "abc123:"
val probe = TestProbe()(system)
probe.send(
@@ -119,6 +123,7 @@ class SegmentIOAuthSpec extends Specification {
}
"reject with CredentialsMissed without credentials" in {
+ val eventServiceActor = createEventServiceActor
val probe = TestProbe()(system)
probe.send(
eventServiceActor,
@@ -137,6 +142,7 @@ class SegmentIOAuthSpec extends Specification {
}
"process SegmentIO identity request properly" in {
+ val eventServiceActor = createEventServiceActor
val jsonReq =
"""
|{
@@ -190,3 +196,5 @@ class SegmentIOAuthSpec extends Specification {
step(system.shutdown())
}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala b/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala
new file mode 100644
index 0000000..8476c91
--- /dev/null
+++ b/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage
+
+import org.scalamock.specs2.MockContext
+
+trait StorageMockContext extends MockContext {
+
+ if(!EnvironmentFactory.environmentService.isDefined){
+ val mockedEnvService = mock[EnvironmentService]
+ (mockedEnvService.envKeys _)
+ .expects
+ .returning(List("PIO_STORAGE_REPOSITORIES_METADATA_NAME",
+ "PIO_STORAGE_SOURCES_MYSQL_TYPE",
+ "PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME",
+ "PIO_STORAGE_SOURCES_EVENTDATA_TYPE"))
+ .twice
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_NAME")
+ .returning("test_metadata")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_SOURCE")
+ .returning("MYSQL")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME")
+ .returning("test_eventdata")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE")
+ .returning("MYSQL")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_SOURCES_MYSQL_TYPE")
+ .returning("jdbc")
+
+ (mockedEnvService.filter _)
+ .expects(*)
+ .returning(Map(
+ "URL" -> "jdbc:h2:~/test;MODE=MySQL;AUTO_SERVER=TRUE",
+ "USERNAME" -> "sa",
+ "PASSWORD" -> "")
+ )
+
+ EnvironmentFactory.environmentService = new Some(mockedEnvService)
+ }
+}
[43/50] [abbrv] incubator-predictionio git commit: Update
webhooks.html.md.erb
Posted by ch...@apache.org.
Update webhooks.html.md.erb
Webhook is provided by the application that provides data, so PredictionIO is responsible for exposing webhook connectors.
Closes #433
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/aa86def5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/aa86def5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/aa86def5
Branch: refs/heads/master
Commit: aa86def567a7c1b252e0096c6d91a7e5f36d51af
Parents: 4750657
Author: aayush kumar <aa...@gmail.com>
Authored: Tue Sep 12 20:35:58 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Sep 12 20:36:25 2017 -0700
----------------------------------------------------------------------
docs/manual/source/datacollection/webhooks.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/aa86def5/docs/manual/source/datacollection/webhooks.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/datacollection/webhooks.html.md.erb b/docs/manual/source/datacollection/webhooks.html.md.erb
index f255a8a..ba53a87 100644
--- a/docs/manual/source/datacollection/webhooks.html.md.erb
+++ b/docs/manual/source/datacollection/webhooks.html.md.erb
@@ -21,6 +21,6 @@ limitations under the License.
Webhooks are used to collects data for your application from multiple channels automatically.
-Apache PredictionIO (incubating) offers webhooks for Segment.io and MailChimp
+Apache PredictionIO (incubating) offers webhook connectors for Segment.io and MailChimp
(backend only). Several users also have expressed interest in Magento, Shopify
and KeenIO so we'd be happy to help if anyone wishes to implement these.
[36/50] [abbrv] incubator-predictionio git commit: [PIO-117] Cannot
delete event data on ESLEvents
Posted by ch...@apache.org.
[PIO-117] Cannot delete event data on ESLEvents
Closes #428
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/6cf89c50
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/6cf89c50
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/6cf89c50
Branch: refs/heads/master
Commit: 6cf89c50252988bf3b2007c82d666b4205a98beb
Parents: 8dcd948
Author: Shinsuke Sugaya <sh...@apache.org>
Authored: Fri Sep 8 09:00:25 2017 +0900
Committer: Shinsuke Sugaya <sh...@apache.org>
Committed: Fri Sep 8 09:00:25 2017 +0900
----------------------------------------------------------------------
.../data/storage/elasticsearch/ESLEvents.scala | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6cf89c50/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
index 5e1f4c1..185be92 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
@@ -287,16 +287,10 @@ class ESLEvents(val client: RestClient, config: StorageClientConfig, val index:
Map("refresh" -> ESUtils.getEventDataRefresh(config)).asJava,
entity)
val jsonResponse = parse(EntityUtils.toString(response.getEntity))
- val result = (jsonResponse \ "result").extract[String]
- result match {
- case "deleted" => true
- case _ =>
- error(s"[$result] Failed to update $index/$estype:$eventId")
- false
- }
+ (jsonResponse \ "deleted").extract[Int] > 0
} catch {
case e: IOException =>
- error(s"Failed to update $index/$estype:$eventId", e)
+ error(s"Failed to delete $index/$estype:$eventId", e)
false
}
}
[14/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
[PIO-107] Removal of examples under examples/experimental.
Closes #416
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/dec9f84c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/dec9f84c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/dec9f84c
Branch: refs/heads/master
Commit: dec9f84ce6a772fb97523ae1e40eb95d9a65b44a
Parents: c903de9
Author: shimamoto <sh...@apache.org>
Authored: Tue Jul 25 12:30:43 2017 +0900
Committer: shimamoto <sh...@apache.org>
Committed: Tue Jul 25 12:30:43 2017 +0900
----------------------------------------------------------------------
examples/experimental/README.md | 20 -
examples/experimental/build-all-scala.sh | 26 -
.../experimental/data/helloworld/.gitignore | 1 -
examples/experimental/data/helloworld/data1.csv | 14 -
examples/experimental/data/helloworld/data2.csv | 21 -
examples/experimental/data/lr_data.txt | 1000 ------------
examples/experimental/data/movielens.txt | 1501 ------------------
.../java-local-helloworld/MyAlgorithm.java | 67 -
.../java-local-helloworld/MyDataSource.java | 56 -
.../java-local-helloworld/MyEngineFactory.java | 34 -
.../java-local-helloworld/MyModel.java | 34 -
.../MyPredictedResult.java | 28 -
.../java-local-helloworld/MyQuery.java | 28 -
.../java-local-helloworld/MyTrainingData.java | 39 -
.../java-local-helloworld/README.md | 93 --
.../java-local-helloworld/build.sbt | 28 -
.../java-local-helloworld/engine.json | 5 -
.../java-local-helloworld/project/assembly.sbt | 1 -
.../java-local-regression/README.md | 96 --
.../java-local-regression/build.sbt | 30 -
.../java-local-regression/engine.json | 33 -
.../java-local-regression/project/assembly.sbt | 1 -
.../src/main/java/DataSource.java | 90 --
.../src/main/java/DataSourceParams.java | 27 -
.../src/main/java/DefaultAlgorithm.java | 39 -
.../src/main/java/DefaultAlgorithmParams.java | 34 -
.../src/main/java/EngineFactory.java | 34 -
.../src/main/java/MeanSquareEvaluator.java | 53 -
.../src/main/java/OLSAlgorithm.java | 61 -
.../src/main/java/Preparator.java | 43 -
.../src/main/java/PreparatorParams.java | 30 -
.../src/main/java/Run.java | 94 --
.../src/main/java/Serving.java | 35 -
.../src/main/java/TrainingData.java | 37 -
.../main/java/examples/algorithmsParams.json | 5 -
.../main/java/examples/dataSourceParams.json | 3 -
.../src/main/java/examples/manifest.json | 6 -
.../main/java/examples/preparatorParams.json | 3 -
.../experimental/java-local-tutorial/README.md | 29 -
.../experimental/java-local-tutorial/build.sbt | 32 -
.../java-local-tutorial/data/test/ratings.csv | 11 -
.../experimental/java-local-tutorial/fetch.sh | 31 -
.../java-local-tutorial/project/assembly.sbt | 1 -
.../src/main/java/recommendations/.gitignore | 1 -
.../recommendations/tutorial1/AlgoParams.java | 28 -
.../recommendations/tutorial1/Algorithm.java | 215 ---
.../recommendations/tutorial1/DataSource.java | 85 -
.../tutorial1/DataSourceParams.java | 28 -
.../tutorial1/EngineFactory.java | 34 -
.../java/recommendations/tutorial1/Model.java | 46 -
.../java/recommendations/tutorial1/Query.java | 35 -
.../recommendations/tutorial1/TrainingData.java | 56 -
.../java/recommendations/tutorial1/engine.json | 6 -
.../tutorial1/params/algorithms.json | 5 -
.../tutorial1/params/datasource.json | 1 -
.../java/recommendations/tutorial2/Runner1.java | 70 -
.../java/recommendations/tutorial2/Runner2.java | 78 -
.../recommendations/tutorial3/DataSource.java | 126 --
.../tutorial3/EngineFactory.java | 38 -
.../recommendations/tutorial3/Evaluator.java | 65 -
.../java/recommendations/tutorial3/Runner3.java | 59 -
.../CollaborativeFilteringAlgorithm.java | 216 ---
.../CollaborativeFilteringAlgorithmParams.java | 28 -
.../tutorial4/CollaborativeFilteringModel.java | 46 -
.../recommendations/tutorial4/DataSource.java | 173 --
.../tutorial4/DataSourceParams.java | 34 -
.../tutorial4/EngineFactory.java | 36 -
.../tutorial4/FeatureBasedAlgorithm.java | 98 --
.../tutorial4/FeatureBasedAlgorithmParams.java | 41 -
.../tutorial4/FeatureBasedModel.java | 39 -
.../recommendations/tutorial4/Preparator.java | 56 -
.../recommendations/tutorial4/PreparedData.java | 38 -
.../java/recommendations/tutorial4/Query.java | 35 -
.../recommendations/tutorial4/Runner4a.java | 60 -
.../recommendations/tutorial4/Runner4b.java | 62 -
.../recommendations/tutorial4/Runner4c.java | 55 -
.../recommendations/tutorial4/Runner4d.java | 57 -
.../java/recommendations/tutorial4/Serving.java | 40 -
.../tutorial4/SingleEngineFactory.java | 36 -
.../recommendations/tutorial4/TrainingData.java | 67 -
.../tutorial4/jsons/algorithms.json | 26 -
.../tutorial4/jsons/datasource.json | 4 -
.../tutorial4/multiple-algo-engine.json | 7 -
.../tutorial4/single-algo-engine.json | 6 -
.../tutorial4/single-jsons/algorithms.json | 11 -
.../tutorial4/single-jsons/datasource.json | 4 -
.../tutorial5/EngineFactory.java | 38 -
.../tutorial5/MahoutAlgoModel.java | 93 --
.../tutorial5/MahoutAlgoParams.java | 33 -
.../tutorial5/MahoutAlgorithm.java | 70 -
.../java/recommendations/tutorial5/Runner5.java | 60 -
.../recommendations/tutorial5/manifest.json | 6 -
.../tutorial5/params/algorithmsParams.json | 5 -
.../tutorial5/params/dataSourceParams.json | 1 -
.../java-parallel-helloworld/build.sbt | 28 -
.../java-parallel-helloworld/engine.json | 11 -
.../project/assembly.sbt | 1 -
.../src/main/java/parallel/Algorithm.java | 111 --
.../src/main/java/parallel/DataSource.java | 75 -
.../src/main/java/parallel/EngineFactory.java | 40 -
.../src/main/java/parallel/Model.java | 58 -
.../src/main/java/parallel/Preparator.java | 41 -
.../src/main/java/parallel/Query.java | 28 -
.../src/main/java/parallel/Runner.java | 68 -
.../src/main/java/parallel/Serving.java | 34 -
.../src/main/java/parallel/build.sbt | 31 -
.../experimental/scala-cleanup-app/README.md | 29 -
.../experimental/scala-cleanup-app/build.sbt | 29 -
.../experimental/scala-cleanup-app/engine.json | 11 -
.../scala-cleanup-app/project/assembly.sbt | 1 -
.../src/main/scala/Algorithm.scala | 49 -
.../src/main/scala/DataSource.scala | 97 --
.../src/main/scala/Engine.scala | 36 -
.../src/main/scala/Preparator.scala | 39 -
.../src/main/scala/Serving.scala | 30 -
.../.gitignore | 1 -
.../scala-local-friend-recommendation/README.md | 127 --
.../scala-local-friend-recommendation/build.sbt | 29 -
.../file_random.py | 207 ---
.../keyword_similarity_engine.json | 19 -
.../project/assembly.sbt | 1 -
.../random_engine.json | 19 -
.../scala/FriendRecommendationAlgoParams.scala | 23 -
.../scala/FriendRecommendationDataSource.scala | 131 --
.../FriendRecommendationDataSourceParams.scala | 27 -
.../scala/FriendRecommendationPrediction.scala | 24 -
.../main/scala/FriendRecommendationQuery.scala | 25 -
.../FriendRecommendationTrainingData.scala | 36 -
.../main/scala/KeywordSimilarityAlgorithm.scala | 83 -
.../scala/KeywordSimilarityEngineFactory.scala | 33 -
.../src/main/scala/KeywordSimilarityModel.scala | 32 -
.../src/main/scala/RandomAlgorithm.scala | 41 -
.../src/main/scala/RandomEngineFactory.scala | 33 -
.../src/main/scala/RandomModel.scala | 22 -
.../scala-local-helloworld/HelloWorld.scala | 106 --
.../scala-local-helloworld/README.md | 92 --
.../scala-local-helloworld/build.sbt | 30 -
.../scala-local-helloworld/engine.json | 5 -
.../scala-local-helloworld/project/assembly.sbt | 1 -
.../scala-local-movielens-evaluation/build.sbt | 31 -
.../engine.json | 5 -
.../project/assembly.sbt | 1 -
.../src/main/scala/Evaluation.scala | 145 --
.../src/main/scala/ItemRecEvaluation.scala | 91 --
.../blacklisted.txt | 2 -
.../scala-local-movielens-filtering/build.sbt | 31 -
.../scala-local-movielens-filtering/engine.json | 65 -
.../project/assembly.sbt | 1 -
.../src/main/scala/Engine.scala | 35 -
.../src/main/scala/Filtering.scala | 39 -
.../scala-local-regression/README.md | 165 --
.../scala-local-regression/Run.scala | 172 --
.../scala-local-regression/build.sbt | 30 -
.../scala-local-regression/engine.json | 17 -
.../scala-local-regression/project/assembly.sbt | 1 -
.../README.md | 65 -
.../build.sbt | 29 -
.../data/circle.txt | 4 -
.../data/edge_list_small.txt | 20 -
.../data/simple.txt | 4 -
.../engine-forest.json | 23 -
.../engine.json | 21 -
.../project/assembly.sbt | 1 -
.../src/main/scala/DataSource.scala | 98 --
.../src/main/scala/DeltaSimRankRDD.scala | 185 ---
.../src/main/scala/Engine.scala | 48 -
.../src/main/scala/Preparator.scala | 42 -
.../src/main/scala/Sampling.scala | 181 ---
.../src/main/scala/Serving.scala | 30 -
.../src/main/scala/SimRankAlgorithm.scala | 60 -
.../.gitignore | 3 -
.../scala-parallel-recommendation-cat/README.md | 94 --
.../scala-parallel-recommendation-cat/build.sbt | 29 -
.../data/import_eventserver.py | 101 --
.../data/send_query.py | 24 -
.../engine.json | 21 -
.../project/assembly.sbt | 1 -
.../src/main/scala/ALSAlgorithm.scala | 279 ----
.../src/main/scala/DataSource.scala | 131 --
.../src/main/scala/Engine.scala | 48 -
.../src/main/scala/Preparator.scala | 41 -
.../src/main/scala/Serving.scala | 29 -
.../.gitignore | 2 -
.../build.sbt | 29 -
.../data/send_query.py | 24 -
.../engine.json | 20 -
.../project/assembly.sbt | 1 -
.../src/main/scala/ALSAlgorithm.scala | 76 -
.../src/main/scala/ALSModel.scala | 80 -
.../src/main/scala/DataSource.scala | 65 -
.../src/main/scala/Engine.scala | 45 -
.../src/main/scala/Preparator.scala | 36 -
.../src/main/scala/Serving.scala | 29 -
.../.gitignore | 2 -
.../build.sbt | 29 -
.../data/import_eventserver.py | 107 --
.../data/send_query.py | 24 -
.../engine.json | 20 -
.../project/assembly.sbt | 1 -
.../src/main/scala/ALSAlgorithm.scala | 72 -
.../src/main/scala/ALSModel.scala | 83 -
.../src/main/scala/DataSource.scala | 128 --
.../src/main/scala/Engine.scala | 45 -
.../src/main/scala/Preparator.scala | 42 -
.../src/main/scala/Serving.scala | 30 -
.../.gitignore | 2 -
.../build.sbt | 35 -
.../data/insert_sample_ratings_mongo.js | 24 -
.../data/send_query.py | 24 -
.../engine.json | 23 -
.../project/assembly.sbt | 1 -
.../src/main/scala/ALSAlgorithm.scala | 76 -
.../src/main/scala/ALSModel.scala | 80 -
.../src/main/scala/DataSource.scala | 84 -
.../src/main/scala/Engine.scala | 45 -
.../src/main/scala/Preparator.scala | 36 -
.../src/main/scala/Serving.scala | 29 -
.../scala-parallel-regression/README.md | 56 -
.../scala-parallel-regression/Run.scala | 137 --
.../scala-parallel-regression/build.sbt | 33 -
.../scala-parallel-regression/engine.json | 21 -
.../project/assembly.sbt | 1 -
.../.gitignore | 2 -
.../build.sbt | 29 -
.../engine.json | 18 -
.../project/assembly.sbt | 1 -
.../src/main/scala/DIMSUMAlgorithm.scala | 232 ---
.../src/main/scala/DataSource.scala | 134 --
.../src/main/scala/Engine.scala | 48 -
.../src/main/scala/Preparator.scala | 41 -
.../src/main/scala/Serving.scala | 29 -
.../.gitignore | 2 -
.../build.sbt | 29 -
.../data/import_eventserver.py | 90 --
.../data/send_query.py | 24 -
.../engine.json | 21 -
.../project/assembly.sbt | 1 -
.../src/main/scala/ALSAlgorithm.scala | 261 ---
.../src/main/scala/DataSource.scala | 131 --
.../src/main/scala/Engine.scala | 48 -
.../src/main/scala/Preparator.scala | 41 -
.../src/main/scala/Serving.scala | 29 -
.../scala-parallel-trim-app/build.sbt | 29 -
.../scala-parallel-trim-app/engine.json | 13 -
.../project/assembly.sbt | 1 -
.../src/main/scala/Algorithm.scala | 49 -
.../src/main/scala/DataSource.scala | 78 -
.../src/main/scala/Engine.scala | 36 -
.../src/main/scala/Preparator.scala | 39 -
.../src/main/scala/Serving.scala | 30 -
.../scala-recommendations/README.md | 146 --
.../scala-recommendations/build.sbt | 31 -
.../scala-recommendations/engine.json | 21 -
.../scala-recommendations/project/assembly.sbt | 1 -
.../src/main/scala/Run.scala | 175 --
.../experimental/scala-refactor-test/build.sbt | 30 -
.../scala-refactor-test/engine.json | 18 -
.../scala-refactor-test/project/assembly.sbt | 1 -
.../src/main/scala/Algorithm.scala | 56 -
.../src/main/scala/DataSource.scala | 74 -
.../src/main/scala/Engine.scala | 63 -
.../src/main/scala/Evaluator.scala | 38 -
.../src/main/scala/Preparator.scala | 40 -
.../src/main/scala/Serving.scala | 32 -
examples/experimental/scala-stock/README.md | 388 -----
examples/experimental/scala-stock/build.sbt | 55 -
.../scala-stock/project/assembly.sbt | 1 -
.../scala-stock/project/plugins.sbt | 1 -
.../scala-stock/src/main/scala/Algorithm.scala | 83 -
.../src/main/scala/BackTestingMetrics.scala | 226 ---
.../scala-stock/src/main/scala/Data.scala | 138 --
.../scala-stock/src/main/scala/DataSource.scala | 64 -
.../scala-stock/src/main/scala/Indicators.scala | 141 --
.../src/main/scala/RegressionStrategy.scala | 156 --
.../scala-stock/src/main/scala/Run.scala | 166 --
.../src/main/scala/YahooDataSource.scala | 483 ------
.../examples/stock/backtesting.scala.html | 125 --
277 files changed, 16784 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/README.md b/examples/experimental/README.md
deleted file mode 100644
index 55f0b4a..0000000
--- a/examples/experimental/README.md
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-**NOTE:** This *$PIO_HOME/examples/experimental/* directory contains experimental examples which may not be compatible with the latest stable PredictionIO release.
-
-Please refer to *$PIO_HOME/templates/* and *$PIO_HOME/examples/* for stable examples.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/build-all-scala.sh
----------------------------------------------------------------------
diff --git a/examples/experimental/build-all-scala.sh b/examples/experimental/build-all-scala.sh
deleted file mode 100755
index 8e0de98..0000000
--- a/examples/experimental/build-all-scala.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-
-for d in `find . -type d -maxdepth 1 | grep scala`
-do
- echo "Building $d..."
- ( cd $d && ../../bin/pio build )
-done
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/data/helloworld/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/data/helloworld/.gitignore b/examples/experimental/data/helloworld/.gitignore
deleted file mode 100644
index 2033008..0000000
--- a/examples/experimental/data/helloworld/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-data.csv
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/data/helloworld/data1.csv
----------------------------------------------------------------------
diff --git a/examples/experimental/data/helloworld/data1.csv b/examples/experimental/data/helloworld/data1.csv
deleted file mode 100644
index f4dd123..0000000
--- a/examples/experimental/data/helloworld/data1.csv
+++ /dev/null
@@ -1,14 +0,0 @@
-Mon,75
-Tue,80
-Wed,74
-Thu,79
-Fri,75
-Sat,74
-Sun,72
-Mon,76
-Tue,81
-Wed,70
-Thu,72
-Fri,84
-Sat,73
-Sun,71
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/data/helloworld/data2.csv
----------------------------------------------------------------------
diff --git a/examples/experimental/data/helloworld/data2.csv b/examples/experimental/data/helloworld/data2.csv
deleted file mode 100644
index c708693..0000000
--- a/examples/experimental/data/helloworld/data2.csv
+++ /dev/null
@@ -1,21 +0,0 @@
-Mon,75
-Tue,80
-Wed,74
-Thu,79
-Fri,75
-Sat,74
-Sun,72
-Mon,76
-Tue,81
-Wed,70
-Thu,72
-Fri,84
-Sat,73
-Sun,71
-Mon,79
-Tue,73
-Wed,75
-Thu,78
-Fri,81
-Sat,70
-Sun,71
[21/50] [abbrv] incubator-predictionio git commit: [PIO-105] Batch
Predictions
Posted by ch...@apache.org.
[PIO-105] Batch Predictions
Implement a new pio batchpredict command to enable massive, fast,
batch predictions from a trained model. Read a multi-object JSON file as
the input format, with one query object per line. Similarly, write
results to a multi-object JSON file, with one prediction result + its
original query per line.
Closes #412
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/cfa3f5da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/cfa3f5da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/cfa3f5da
Branch: refs/heads/master
Commit: cfa3f5dab533a67688ec2f84182eccedc56fa84e
Parents: 965c73f
Author: Mars Hall <ma...@heroku.com>
Authored: Tue Aug 1 14:56:47 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Aug 1 14:56:47 2017 -0700
----------------------------------------------------------------------
.../predictionio/workflow/BatchPredict.scala | 230 +++++++++++++++++++
.../predictionio/tools/RunBatchPredict.scala | 72 ++++++
.../predictionio/tools/commands/Engine.scala | 55 ++++-
.../predictionio/tools/console/Console.scala | 58 ++++-
.../apache/predictionio/tools/console/Pio.scala | 13 +-
.../tools/console/batchpredict.scala.txt | 25 ++
.../predictionio/tools/console/main.scala.txt | 1 +
7 files changed, 450 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cfa3f5da/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala b/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
new file mode 100644
index 0000000..2fb0545
--- /dev/null
+++ b/core/src/main/scala/org/apache/predictionio/workflow/BatchPredict.scala
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.workflow
+
+import java.io.Serializable
+
+import com.twitter.bijection.Injection
+import com.twitter.chill.{KryoBase, KryoInjection, ScalaKryoInstantiator}
+import de.javakaffee.kryoserializers.SynchronizedCollectionsSerializer
+import grizzled.slf4j.Logging
+import org.apache.predictionio.controller.{Engine, Utils}
+import org.apache.predictionio.core.{BaseAlgorithm, BaseServing, Doer}
+import org.apache.predictionio.data.storage.{EngineInstance, Storage}
+import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
+import org.apache.spark.rdd.RDD
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import scala.language.existentials
+
+case class BatchPredictConfig(
+ inputFilePath: String = "batchpredict-input.json",
+ outputFilePath: String = "batchpredict-output.json",
+ queryPartitions: Option[Int] = None,
+ engineInstanceId: String = "",
+ engineId: Option[String] = None,
+ engineVersion: Option[String] = None,
+ engineVariant: String = "",
+ env: Option[String] = None,
+ verbose: Boolean = false,
+ debug: Boolean = false,
+ jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
+
+object BatchPredict extends Logging {
+
+ class KryoInstantiator(classLoader: ClassLoader) extends ScalaKryoInstantiator {
+ override def newKryo(): KryoBase = {
+ val kryo = super.newKryo()
+ kryo.setClassLoader(classLoader)
+ SynchronizedCollectionsSerializer.registerSerializers(kryo)
+ kryo
+ }
+ }
+
+ object KryoInstantiator extends Serializable {
+ def newKryoInjection : Injection[Any, Array[Byte]] = {
+ val kryoInstantiator = new KryoInstantiator(getClass.getClassLoader)
+ KryoInjection.instance(kryoInstantiator)
+ }
+ }
+
+ val engineInstances = Storage.getMetaDataEngineInstances
+ val modeldata = Storage.getModelDataModels
+
+ def main(args: Array[String]): Unit = {
+ val parser = new scopt.OptionParser[BatchPredictConfig]("BatchPredict") {
+ opt[String]("input") action { (x, c) =>
+ c.copy(inputFilePath = x)
+ } text("Path to file containing input queries; a " +
+ "multi-object JSON file with one object per line.")
+ opt[String]("output") action { (x, c) =>
+ c.copy(outputFilePath = x)
+ } text("Path to file containing output predictions; a " +
+ "multi-object JSON file with one object per line.")
+ opt[Int]("query-partitions") action { (x, c) =>
+ c.copy(queryPartitions = Some(x))
+ } text("Limit concurrency of predictions by setting the number " +
+ "of partitions used internally for the RDD of queries.")
+ opt[String]("engineId") action { (x, c) =>
+ c.copy(engineId = Some(x))
+ } text("Engine ID.")
+ opt[String]("engineId") action { (x, c) =>
+ c.copy(engineId = Some(x))
+ } text("Engine ID.")
+ opt[String]("engineVersion") action { (x, c) =>
+ c.copy(engineVersion = Some(x))
+ } text("Engine version.")
+ opt[String]("engine-variant") required() action { (x, c) =>
+ c.copy(engineVariant = x)
+ } text("Engine variant JSON.")
+ opt[String]("env") action { (x, c) =>
+ c.copy(env = Some(x))
+ } text("Comma-separated list of environmental variables (in 'FOO=BAR' " +
+ "format) to pass to the Spark execution environment.")
+ opt[String]("engineInstanceId") required() action { (x, c) =>
+ c.copy(engineInstanceId = x)
+ } text("Engine instance ID.")
+ opt[Unit]("verbose") action { (x, c) =>
+ c.copy(verbose = true)
+ } text("Enable verbose output.")
+ opt[Unit]("debug") action { (x, c) =>
+ c.copy(debug = true)
+ } text("Enable debug output.")
+ opt[String]("json-extractor") action { (x, c) =>
+ c.copy(jsonExtractor = JsonExtractorOption.withName(x))
+ }
+ }
+
+ parser.parse(args, BatchPredictConfig()) map { config =>
+ WorkflowUtils.modifyLogging(config.verbose)
+ engineInstances.get(config.engineInstanceId) map { engineInstance =>
+
+ val engine = getEngine(engineInstance)
+
+ run(config, engineInstance, engine)
+
+ } getOrElse {
+ error(s"Invalid engine instance ID. Aborting batch predict.")
+ }
+ }
+ }
+
+ def getEngine(engineInstance: EngineInstance): Engine[_, _, _, _, _, _] = {
+
+ val engineFactoryName = engineInstance.engineFactory
+
+ val (engineLanguage, engineFactory) =
+ WorkflowUtils.getEngine(engineFactoryName, getClass.getClassLoader)
+ val maybeEngine = engineFactory()
+
+ // EngineFactory return a base engine, which may not be deployable.
+ if (!maybeEngine.isInstanceOf[Engine[_,_,_,_,_,_]]) {
+ throw new NoSuchMethodException(
+ s"Engine $maybeEngine cannot be used for batch predict")
+ }
+
+ maybeEngine.asInstanceOf[Engine[_,_,_,_,_,_]]
+ }
+
+ def run[Q, P](
+ config: BatchPredictConfig,
+ engineInstance: EngineInstance,
+ engine: Engine[_, _, _, Q, P, _]): Unit = {
+
+ val engineParams = engine.engineInstanceToEngineParams(
+ engineInstance, config.jsonExtractor)
+
+ val kryo = KryoInstantiator.newKryoInjection
+
+ val modelsFromEngineInstance =
+ kryo.invert(modeldata.get(engineInstance.id).get.models).get.
+ asInstanceOf[Seq[Any]]
+
+ val prepareSparkContext = WorkflowContext(
+ batch = engineInstance.engineFactory,
+ executorEnv = engineInstance.env,
+ mode = "Batch Predict (model)",
+ sparkEnv = engineInstance.sparkConf)
+
+ val models = engine.prepareDeploy(
+ prepareSparkContext,
+ engineParams,
+ engineInstance.id,
+ modelsFromEngineInstance,
+ params = WorkflowParams()
+ )
+
+ val algorithms = engineParams.algorithmParamsList.map { case (n, p) =>
+ Doer(engine.algorithmClassMap(n), p)
+ }
+
+ val servingParamsWithName = engineParams.servingParams
+
+ val serving = Doer(engine.servingClassMap(servingParamsWithName._1),
+ servingParamsWithName._2)
+
+ val runSparkContext = WorkflowContext(
+ batch = engineInstance.engineFactory,
+ executorEnv = engineInstance.env,
+ mode = "Batch Predict (runner)",
+ sparkEnv = engineInstance.sparkConf)
+
+ val inputRDD: RDD[String] = runSparkContext.
+ textFile(config.inputFilePath).
+ filter(_.trim.nonEmpty)
+ val queriesRDD: RDD[String] = config.queryPartitions match {
+ case Some(p) => inputRDD.repartition(p)
+ case None => inputRDD
+ }
+
+ val predictionsRDD: RDD[String] = queriesRDD.map { queryString =>
+ val jsonExtractorOption = config.jsonExtractor
+ // Extract Query from Json
+ val query = JsonExtractor.extract(
+ jsonExtractorOption,
+ queryString,
+ algorithms.head.queryClass,
+ algorithms.head.querySerializer,
+ algorithms.head.gsonTypeAdapterFactories
+ )
+ // Deploy logic. First call Serving.supplement, then Algo.predict,
+ // finally Serving.serve.
+ val supplementedQuery = serving.supplementBase(query)
+ // TODO: Parallelize the following.
+ val predictions = algorithms.zipWithIndex.map { case (a, ai) =>
+ a.predictBase(models(ai), supplementedQuery)
+ }
+ // Notice that it is by design to call Serving.serve with the
+ // *original* query.
+ val prediction = serving.serveBase(query, predictions)
+ // Combine query with prediction, so the batch results are
+ // self-descriptive.
+ val predictionJValue = JsonExtractor.toJValue(
+ jsonExtractorOption,
+ Map("query" -> query,
+ "prediction" -> prediction),
+ algorithms.head.querySerializer,
+ algorithms.head.gsonTypeAdapterFactories)
+ // Return JSON string
+ compact(render(predictionJValue))
+ }
+
+ predictionsRDD.saveAsTextFile(config.outputFilePath)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cfa3f5da/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala
new file mode 100644
index 0000000..35572c9
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunBatchPredict.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.tools
+
+import org.apache.predictionio.tools.Common._
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.workflow.JsonExtractorOption
+import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
+
+import java.io.File
+import grizzled.slf4j.Logging
+
+import scala.sys.process._
+
+case class BatchPredictArgs(
+ inputFilePath: String = "batchpredict-input.json",
+ outputFilePath: String = "batchpredict-output.json",
+ queryPartitions: Option[Int] = None,
+ variantJson: Option[File] = None,
+ jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
+
+
+object RunBatchPredict extends Logging {
+
+ def runBatchPredict(
+ engineInstanceId: String,
+ batchPredictArgs: BatchPredictArgs,
+ sparkArgs: SparkArgs,
+ pioHome: String,
+ engineDirPath: String,
+ verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
+ val jarFiles = jarFilesForScala(engineDirPath).map(_.toURI) ++
+ Option(new File(pioHome, "plugins").listFiles())
+ .getOrElse(Array.empty[File]).map(_.toURI)
+ val args = Seq[String](
+ "--input",
+ batchPredictArgs.inputFilePath,
+ "--output",
+ batchPredictArgs.outputFilePath,
+ "--engineInstanceId",
+ engineInstanceId,
+ "--engine-variant",
+ batchPredictArgs.variantJson.getOrElse(
+ new File(engineDirPath, "engine.json")).getCanonicalPath) ++
+ (if (batchPredictArgs.queryPartitions.isEmpty) Seq()
+ else Seq("--query-partitions",
+ batchPredictArgs.queryPartitions.get.toString)) ++
+ (if (verbose) Seq("--verbose") else Seq()) ++
+ Seq("--json-extractor", batchPredictArgs.jsonExtractor.toString)
+
+ Runner.runOnSpark(
+ "org.apache.predictionio.workflow.BatchPredict",
+ args, sparkArgs, jarFiles, pioHome, verbose)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cfa3f5da/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
index e49c3fc..e3460a5 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -21,8 +21,9 @@ import org.apache.predictionio.core.BuildInfo
import org.apache.predictionio.controller.Utils
import org.apache.predictionio.data.storage
import org.apache.predictionio.tools.EitherLogging
-import org.apache.predictionio.tools.{RunWorkflow, RunServer}
-import org.apache.predictionio.tools.{DeployArgs, WorkflowArgs, SparkArgs, ServerArgs}
+import org.apache.predictionio.tools.{RunWorkflow, RunServer, RunBatchPredict}
+import org.apache.predictionio.tools.{
+ DeployArgs, WorkflowArgs, SparkArgs, ServerArgs, BatchPredictArgs}
import org.apache.predictionio.tools.console.Console
import org.apache.predictionio.tools.Common._
import org.apache.predictionio.tools.ReturnTypes._
@@ -262,6 +263,56 @@ object Engine extends EitherLogging {
}
}
+ /** Batch predict with an engine.
+ *
+ * @param ea An instance of [[EngineArgs]]
+ * @param engineInstanceId An instance of [[engineInstanceId]]
+ * @param batchPredictArgs An instance of [[BatchPredictArgs]]
+ * @param sparkArgs An instance of [[SparkArgs]]
+ * @param pioHome [[String]] with a path to PIO installation
+ * @param verbose A [[Boolean]]
+ * @return An instance of [[Expected]] contaning either [[Left]]
+ * with an error message or [[Right]] with a handle to process
+ * of a running angine and a function () => Unit,
+ * that must be called when the process is complete
+ */
+ def batchPredict(
+ ea: EngineArgs,
+ engineInstanceId: Option[String],
+ batchPredictArgs: BatchPredictArgs,
+ sparkArgs: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
+ val engineDirPath = getEngineDirPath(ea.engineDir)
+ val verifyResult = Template.verifyTemplateMinVersion(
+ new File(engineDirPath, "template.json"))
+ if (verifyResult.isLeft) {
+ return Left(verifyResult.left.get)
+ }
+ val ei = Console.getEngineInfo(
+ batchPredictArgs.variantJson.getOrElse(new File(engineDirPath, "engine.json")),
+ engineDirPath)
+ val engineInstances = storage.Storage.getMetaDataEngineInstances
+ val engineInstance = engineInstanceId map { eid =>
+ engineInstances.get(eid)
+ } getOrElse {
+ engineInstances.getLatestCompleted(
+ ei.engineId, ei.engineVersion, ei.variantId)
+ }
+ engineInstance map { r =>
+ RunBatchPredict.runBatchPredict(
+ r.id, batchPredictArgs, sparkArgs, pioHome, engineDirPath, verbose)
+ } getOrElse {
+ engineInstanceId map { eid =>
+ logAndFail(s"Invalid engine instance ID ${eid}. Aborting.")
+ } getOrElse {
+ logAndFail(s"No valid engine instance found for engine ${ei.engineId} " +
+ s"${ei.engineVersion}.\nTry running 'train' before 'batchpredict'. Aborting.")
+ }
+ }
+ }
+
/** Running a driver on spark.
* The function starts a process and returns immediately
*
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cfa3f5da/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
index 535905a..4a72635 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
@@ -27,7 +27,8 @@ import org.apache.predictionio.tools.commands.{
DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
BuildArgs, EngineArgs}
import org.apache.predictionio.tools.{
- EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
+ EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs,
+ DeployArgs, BatchPredictArgs}
import org.apache.predictionio.workflow.{JsonExtractorOption, WorkflowUtils}
import org.json4s._
import org.json4s.native.JsonMethods._
@@ -42,6 +43,7 @@ case class ConsoleArgs(
workflow: WorkflowArgs = WorkflowArgs(),
accessKey: AccessKeyArgs = AccessKeyArgs(),
deploy: DeployArgs = DeployArgs(),
+ batchPredict: BatchPredictArgs = BatchPredictArgs(),
eventServer: EventServerArgs = EventServerArgs(),
adminServer: AdminServerArgs = AdminServerArgs(),
dashboard: DashboardArgs = DashboardArgs(),
@@ -323,6 +325,46 @@ object Console extends Logging {
} text("Port to unbind from. Default: 8000")
)
note("")
+ cmd("batchpredict").
+ text("Use an engine instance to process batch predictions. This\n" +
+ "command will pass all pass-through arguments to its underlying\n" +
+ "spark-submit command. All algorithm classes used in the engine\n" +
+ "must be serializable.").
+ action { (_, c) =>
+ c.copy(commands = c.commands :+ "batchpredict")
+ } children(
+ opt[String]("input") action { (x, c) =>
+ c.copy(batchPredict = c.batchPredict.copy(inputFilePath = x))
+ } text("Path to file containing queries; a multi-object JSON file\n" +
+ "with one query object per line. Accepts any valid Hadoop\n" +
+ "file URL. Default: batchpredict-input.json"),
+ opt[String]("output") action { (x, c) =>
+ c.copy(batchPredict = c.batchPredict.copy(outputFilePath = x))
+ } text("Path to file to receive results; a multi-object JSON file\n" +
+ "with one object per line, the prediction + original query.\n" +
+ "Accepts any valid Hadoop file URL. Actual output will be\n" +
+ "written as Hadoop partition files in a directory with the\n" +
+ "output name. Default: batchpredict-output.json"),
+ opt[Int]("query-partitions") action { (x, c) =>
+ c.copy(batchPredict = c.batchPredict.copy(queryPartitions = Some(x)))
+ } text("Limit concurrency of predictions by setting the number\n" +
+ "of partitions used internally for the RDD of queries.\n" +
+ "Default: number created by Spark context's `textFile`"),
+ opt[String]("engine-instance-id") action { (x, c) =>
+ c.copy(engineInstanceId = Some(x))
+ } text("Engine instance ID."),
+ opt[String]("json-extractor") action { (x, c) =>
+ c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
+ } validate { x =>
+ if (JsonExtractorOption.values.map(_.toString).contains(x)) {
+ success
+ } else {
+ val validOptions = JsonExtractorOption.values.mkString("|")
+ failure(s"$x is not a valid json-extractor option [$validOptions]")
+ }
+ }
+ )
+ note("")
cmd("dashboard").
text("Launch a dashboard at the specific IP and port.").
action { (_, c) =>
@@ -644,6 +686,19 @@ object Console extends Logging {
ca.verbose)
case Seq("undeploy") =>
Pio.undeploy(ca.deploy)
+ case Seq("batchpredict") =>
+ Pio.batchPredict(
+ ca.engine,
+ ca.engineInstanceId,
+ BatchPredictArgs(
+ ca.batchPredict.inputFilePath,
+ ca.batchPredict.outputFilePath,
+ ca.batchPredict.queryPartitions,
+ ca.workflow.variantJson,
+ ca.workflow.jsonExtractor),
+ ca.spark,
+ ca.pioHome.get,
+ ca.verbose)
case Seq("dashboard") =>
Pio.dashboard(ca.dashboard)
case Seq("eventserver") =>
@@ -756,6 +811,7 @@ object Console extends Logging {
"build" -> txt.build().toString,
"train" -> txt.train().toString,
"deploy" -> txt.deploy().toString,
+ "batchpredict" -> txt.batchpredict().toString,
"eventserver" -> txt.eventserver().toString,
"adminserver" -> txt.adminserver().toString,
"app" -> txt.app().toString,
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cfa3f5da/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
index dd78717..ef4581b 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
@@ -18,7 +18,8 @@
package org.apache.predictionio.tools.console
import org.apache.predictionio.tools.{
- EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
+ EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs,
+ DeployArgs, BatchPredictArgs}
import org.apache.predictionio.tools.commands.{
DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
BuildArgs, EngineArgs, Management, Engine, Import, Export,
@@ -104,6 +105,16 @@ object Pio extends Logging {
def undeploy(da: DeployArgs): Int = Engine.undeploy(da)
+ def batchPredict(
+ ea: EngineArgs,
+ engineInstanceId: Option[String],
+ batchPredictArgs: BatchPredictArgs,
+ sparkArgs: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Int =
+ processAwaitAndClean(Engine.batchPredict(
+ ea, engineInstanceId, batchPredictArgs, sparkArgs, pioHome, verbose))
+
def dashboard(da: DashboardArgs): Int = {
Management.dashboard(da).awaitTermination
0
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cfa3f5da/tools/src/main/twirl/org/apache/predictionio/tools/console/batchpredict.scala.txt
----------------------------------------------------------------------
diff --git a/tools/src/main/twirl/org/apache/predictionio/tools/console/batchpredict.scala.txt b/tools/src/main/twirl/org/apache/predictionio/tools/console/batchpredict.scala.txt
new file mode 100644
index 0000000..d9d5d74
--- /dev/null
+++ b/tools/src/main/twirl/org/apache/predictionio/tools/console/batchpredict.scala.txt
@@ -0,0 +1,25 @@
+Usage: pio batchpredict [--input <value>]
+ [--output <value>]
+ [--query-partitions <value>]
+ [--engine-instance-id <value>]
+
+Use an engine instance to process batch predictions. This command will pass all
+pass-through arguments to its underlying spark-submit command. All algorithm
+classes used in the engine must be serializable.
+
+ --input <value>
+ Path to file containing queries; a multi-object JSON file with one
+ query object per line. Accepts any valid Hadoop file URL.
+ Default: batchpredict-input.json
+ --output <value>
+ Path to file to receive results; a multi-object JSON file with one
+ object per line, the prediction + original query. Accepts any
+ valid Hadoop file URL. Actual output will be written as Hadoop
+ partition files in a directory with the output name.
+ Default: batchpredict-output.json
+ --query-partitions <value>
+ Limit concurrency of predictions by setting the number of partitions
+ used internally for the RDD of queries.
+ Default: number created by Spark context's `textFile`
+ --engine-instance-id <value>
+ Engine instance ID. Default: the latest trained instance.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cfa3f5da/tools/src/main/twirl/org/apache/predictionio/tools/console/main.scala.txt
----------------------------------------------------------------------
diff --git a/tools/src/main/twirl/org/apache/predictionio/tools/console/main.scala.txt b/tools/src/main/twirl/org/apache/predictionio/tools/console/main.scala.txt
index 5efa4bf..01be96d 100644
--- a/tools/src/main/twirl/org/apache/predictionio/tools/console/main.scala.txt
+++ b/tools/src/main/twirl/org/apache/predictionio/tools/console/main.scala.txt
@@ -38,6 +38,7 @@ The most commonly used pio commands are:
build Build an engine at the current directory
train Kick off a training using an engine
deploy Deploy an engine as an engine server
+ batchpredict Process bulk predictions with an engine
eventserver Launch an Event Server
app Manage apps that are used by the Event Server
accesskey Manage app access keys
[04/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by ch...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/c903de9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/c903de9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/c903de9c
Branch: refs/heads/master
Commit: c903de9c6213c115196e9ecd95abd7258dee6764
Parents: 96d4c54 83f5ef7
Author: Naoki Takezoe <ta...@apache.org>
Authored: Wed Jul 19 10:28:27 2017 +0900
Committer: Naoki Takezoe <ta...@apache.org>
Committed: Wed Jul 19 10:28:27 2017 +0900
----------------------------------------------------------------------
.../source/templates/similarproduct/dase.html.md.erb | 14 +++++++-------
.../multi-events-multi-algos.html.md.erb | 2 +-
.../similarproduct/return-item-properties.html.md.erb | 2 +-
3 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[49/50] [abbrv] incubator-predictionio git commit: Revise release
notes
Posted by ch...@apache.org.
Revise release notes
Clarify breaking changes; link issue IDs to JIRA
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/93405616
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/93405616
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/93405616
Branch: refs/heads/master
Commit: 93405616c64d4d5195c65845662fc4613baa1498
Parents: d2a6aa9
Author: Mars Hall <ma...@heroku.com>
Authored: Sun Sep 17 09:46:22 2017 -0700
Committer: Chan Lee <ch...@gmail.com>
Committed: Sun Sep 17 09:46:22 2017 -0700
----------------------------------------------------------------------
RELEASE.md | 76 ++++++++++++++++++++++++++++++---------------------------
1 file changed, 40 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/93405616/RELEASE.md
----------------------------------------------------------------------
diff --git a/RELEASE.md b/RELEASE.md
index b506981..f94b3a6 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -27,38 +27,42 @@ Sep ?, 2017
#### New Features
-- PIO-61: S3 support for model data
-- PIO-69, PIO-91: Binary distribution of PredictionIO
-- PIO-105, PIO-110, PIO-111: Batch predictions
-- PIO-95: Configurable request timeout for REST API
-- PIO-114: Basic HTTP authentication for Elasticsearch 5.x StorageClient
-- PIO-116: PySpark support
+- [PIO-61](https://issues.apache.org/jira/browse/PIO-61): S3 support for model data
+- [PIO-69](https://issues.apache.org/jira/browse/PIO-69), [PIO-91](https://issues.apache.org/jira/browse/PIO-91): Binary distribution of PredictionIO
+- [PIO-105](https://issues.apache.org/jira/browse/PIO-105), [PIO-110](https://issues.apache.org/jira/browse/PIO-110), [PIO-111](https://issues.apache.org/jira/browse/PIO-111): Batch predictions
+- [PIO-95](https://issues.apache.org/jira/browse/PIO-95): Raise request timeout for REST API to 35-seconds
+- [PIO-114](https://issues.apache.org/jira/browse/PIO-114): Basic HTTP authentication for Elasticsearch 5.x StorageClient
+- [PIO-116](https://issues.apache.org/jira/browse/PIO-116): PySpark support
+
+#### Breaking changes
+
+- [PIO-106](https://issues.apache.org/jira/browse/PIO-106): Elasticsearch 5.x StorageClient should reuse RestClient (see the [pull request](https://github.com/apache/incubator-predictionio/pull/421))
#### Behavior Changes
-- PIO-59: `pio app new` uses /dev/urandom/ to generate entropy.
-- PIO-72: `pio-shell` properly loads storage dependencies.
-- PIO-83, PIO-119: Default environment changed to Spark 2.1.1, Scala 2.11.8,
+- [PIO-59](https://issues.apache.org/jira/browse/PIO-59): `pio app new` uses /dev/urandom/ to generate entropy.
+- [PIO-72](https://issues.apache.org/jira/browse/PIO-72): `pio-shell` properly loads storage dependencies.
+- [PIO-83](https://issues.apache.org/jira/browse/PIO-83), [PIO-119](https://issues.apache.org/jira/browse/PIO-119): Default environment changed to Spark 2.1.1, Scala 2.11.8,
and Elasticsearch 5.5.2.
-- PIO-99: `pio-build` checks for compilation errors before proceeding
+- [PIO-99](https://issues.apache.org/jira/browse/PIO-99): `pio-build` checks for compilation errors before proceeding
to build engine.
-- PIO-100: `pio` commands no longer display SLF4J warning messages.
+- [PIO-100](https://issues.apache.org/jira/browse/PIO-100): `pio` commands no longer display SLF4J warning messages.
#### Other Changes
-- PIO-56: Core unit tests no longer require meta data setup.
-- PIO-60, PIO-62: Minor fixes in authorship information and license checking.
-- PIO-63: Apache incubator logo and disclaimer is displayed on the website.
-- PIO-65: Integration tests on Travis caches downloaded jars.
-- PIO-66: More detailed documentation regarding release process and adding
+- [PIO-56](https://issues.apache.org/jira/browse/PIO-56): Core unit tests no longer require meta data setup.
+- [PIO-60](https://issues.apache.org/jira/browse/PIO-60), [PIO-62](https://issues.apache.org/jira/browse/PIO-62): Minor fixes in authorship information and license checking.
+- [PIO-63](https://issues.apache.org/jira/browse/PIO-63): Apache incubator logo and disclaimer is displayed on the website.
+- [PIO-65](https://issues.apache.org/jira/browse/PIO-65): Integration tests on Travis caches downloaded jars.
+- [PIO-66](https://issues.apache.org/jira/browse/PIO-66): More detailed documentation regarding release process and adding
JIRA tickets.
-- PIO-90: Improved performance for /batch/events.json API call.
-- PIO-94: More detailed stack trace for REST API errors.
-- PIO-97: Update examples in official templates.
-- PIO-102, PIO-106, PIO-117, PIO-118, PIO-120: Bug fixes, refactoring, and
- improved performance on elasticsearch behavior.
-- PIO-104: Bug fix regarding plugins.
-- PIO-107: Obsolete experimental examples are removed.
+- [PIO-90](https://issues.apache.org/jira/browse/PIO-90): Improved performance for /batch/events.json API call.
+- [PIO-94](https://issues.apache.org/jira/browse/PIO-94): More detailed stack trace for REST API errors.
+- [PIO-97](https://issues.apache.org/jira/browse/PIO-97): Update examples in official templates.
+- [PIO-102](https://issues.apache.org/jira/browse/PIO-102), [PIO-117](https://issues.apache.org/jira/browse/PIO-117), [PIO-118](https://issues.apache.org/jira/browse/PIO-118), [PIO-120](https://issues.apache.org/jira/browse/PIO-120): Bug fixes, refactoring, and
+ improved performance on Elasticsearch behavior.
+- [PIO-104](https://issues.apache.org/jira/browse/PIO-104): Bug fix regarding plugins.
+- [PIO-107](https://issues.apache.org/jira/browse/PIO-107): Obsolete experimental examples are removed.
#### Credits
@@ -75,34 +79,34 @@ Apr 25, 2017
#### New Features
-- PIO-30: Scala 2.11 support
-- PIO-30: Spark 2 support
-- PIO-49: Elasticsearch 5 support
-- PIO-30, PIO-49: Flexible build system
-- PIO-47, PIO-51: Removal of engine manifests
-- PIO-49: Modularized storage backend modules
-- PIO-45: Self cleaning data source
+- [PIO-30](https://issues.apache.org/jira/browse/PIO-30): Scala 2.11 support
+- [PIO-30](https://issues.apache.org/jira/browse/PIO-30): Spark 2 support
+- [PIO-49](https://issues.apache.org/jira/browse/PIO-49): Elasticsearch 5 support
+- [PIO-30](https://issues.apache.org/jira/browse/PIO-30), [PIO-49](https://issues.apache.org/jira/browse/PIO-49): Flexible build system
+- [PIO-47](https://issues.apache.org/jira/browse/PIO-47), [PIO-51](https://issues.apache.org/jira/browse/PIO-51): Removal of engine manifests
+- [PIO-49](https://issues.apache.org/jira/browse/PIO-49): Modularized storage backend modules
+- [PIO-45](https://issues.apache.org/jira/browse/PIO-45): Self cleaning data source
#### Behavior Changes
-- PIO-25: `pio-start-all` will no longer start PostgreSQL if it is not being
+- [PIO-25](https://issues.apache.org/jira/browse/PIO-25): `pio-start-all` will no longer start PostgreSQL if it is not being
used.
-- PIO-47, PIO-51: `pio build` no longer requires access to the metadata
+- [PIO-47](https://issues.apache.org/jira/browse/PIO-47), [PIO-51](https://issues.apache.org/jira/browse/PIO-51): `pio build` no longer requires access to the metadata
repository. `pio` commands will now accept an optional `--engine-dir`
parameter if you want to run `pio build`, `pio train`, or `pio deploy` outside
of an engine directory. This is an interim solution before an engine registry
feature becomes available in the future.
-- PIO-49: PostgreSQL JDBC driver is no longer bundled with the core assembly. If
+- [PIO-49](https://issues.apache.org/jira/browse/PIO-49): PostgreSQL JDBC driver is no longer bundled with the core assembly. If
you are using PostgreSQL, you must download the JDBC driver and update your
configuration to point to the correct JDBC driver file.
-- PIO-54: New generated access keys will no longer start with a `-` character.
+- [PIO-54](https://issues.apache.org/jira/browse/PIO-54): New generated access keys will no longer start with a `-` character.
#### Other Changes
-- PIO-28: Code refactoring of the command line interface. It is now possible to
+- [PIO-28](https://issues.apache.org/jira/browse/PIO-28): Code refactoring of the command line interface. It is now possible to
develop new interfaces that perform the same functionalities provided by the
CLI.
-- PIO-53: Integration tests can now be tied to every single Git commit, without
+- [PIO-53](https://issues.apache.org/jira/browse/PIO-53): Integration tests can now be tied to every single Git commit, without
the need to update the official test Docker image.
- The meta data and model data access object methods are now public and marked
as Core Developer API.
[10/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4d.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4d.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4d.java
deleted file mode 100644
index 6b224f1..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4d.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.LJavaFirstServing;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-import org.apache.predictionio.controller.IdentityPreparator;
-
-public class Runner4d {
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file directory as argument");
- System.exit(1);
- }
-
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(args[0], true))
- // 1 -> -1., 2 -> -.5, 3 -> 0., 4 -> .5, 5 -> 1.
- .addAlgorithmParams("featurebased", new FeatureBasedAlgorithmParams(1.0, 5.0, 3.0, 0.5))
- .addAlgorithmParams("featurebased", new FeatureBasedAlgorithmParams(4.0, 5.0, 3.0, 0.5))
- .build();
-
- JavaWorkflow.runEngine(
- (new EngineFactory()).apply(),
- engineParams,
- new WorkflowParamsBuilder().batch("Recommendation.tutorial4.Runner4d").verbose(3).build()
- );
- }
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Serving.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Serving.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Serving.java
deleted file mode 100644
index aba1611..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Serving.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.LJavaServing;
-import org.apache.predictionio.controller.java.EmptyParams;
-import java.lang.Iterable;
-
-public class Serving extends LJavaServing<EmptyParams, Query, Float> {
- public Serving() {}
-
- public Float serve(Query query, Iterable<Float> predictions) {
- float sum = 0.0f;
- int count = 0;
-
- for (Float v: predictions) {
- if (!v.isNaN()) {
- sum += v;
- count += 1;
- }
- }
- return (count == 0) ? Float.NaN : sum / count;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/SingleEngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/SingleEngineFactory.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/SingleEngineFactory.java
deleted file mode 100644
index 9c5cf06..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/SingleEngineFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-import org.apache.predictionio.controller.java.LJavaFirstServing;
-
-public class SingleEngineFactory implements IJavaEngineFactory {
- public JavaEngine<TrainingData, EmptyParams, PreparedData, Query, Float, Object> apply() {
- return new JavaEngineBuilder<
- TrainingData, EmptyParams, PreparedData, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass(Preparator.class)
- .addAlgorithmClass("featurebased", FeatureBasedAlgorithm.class)
- .servingClass(LJavaFirstServing.apply(FeatureBasedAlgorithm.class))
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/TrainingData.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/TrainingData.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/TrainingData.java
deleted file mode 100644
index 61970cd..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/TrainingData.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
-public class TrainingData implements Serializable {
- public final List<Rating> ratings;
- public final List<String> genres;
- public final Map<Integer, String[]> itemInfo;
- public final Map<Integer, String[]> userInfo;
-
- public TrainingData(List<Rating> ratings, List<String> genres, Map<Integer, String[]> itemInfo,
- Map<Integer, String[]> userInfo) {
- this.ratings = ratings;
- this.genres = genres;
- this.itemInfo = itemInfo;
- this.userInfo = userInfo;
- }
-
- public TrainingData(TrainingData data) {
- ratings = data.ratings;
- genres = data.genres;
- itemInfo = data.itemInfo;
- userInfo = data.userInfo;
- }
-
- @Override
- public String toString() {
- return "TrainingData: rating.size=" + ratings.size() + " genres.size=" + genres.size()
- + " itemInfo.size=" + itemInfo.size() + " userInfo.size=" + userInfo.size();
- }
-
- public static class Rating implements Serializable {
- public int uid; // user ID
- public int iid; // item ID
- public float rating;
-
- public Rating(int uid, int iid, float rating) {
- this.uid = uid;
- this.iid = iid;
- this.rating = rating;
- }
-
- @Override
- public String toString() {
- return "(" + uid + "," + iid + "," + rating + ")";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/algorithms.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/algorithms.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/algorithms.json
deleted file mode 100644
index cd57d8b..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/algorithms.json
+++ /dev/null
@@ -1,26 +0,0 @@
-[
- {
- "name": "featurebased",
- "params": {
- "min": 1.0,
- "max": 5.0,
- "drift": 3.0,
- "scale": 0.5
- }
- },
- {
- "name": "featurebased",
- "params": {
- "min": 4.0,
- "max": 5.0,
- "drift": 3.0,
- "scale": 0.5
- }
- },
- {
- "name": "collaborative",
- "params": {
- "threshold": 0.2
- }
- }
-]
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/datasource.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/datasource.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/datasource.json
deleted file mode 100644
index 11a3a34..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/jsons/datasource.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "dir" : "data/ml-100k/",
- "addFakeData": true
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/multiple-algo-engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/multiple-algo-engine.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/multiple-algo-engine.json
deleted file mode 100644
index 2f757c2..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/multiple-algo-engine.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "id": "org.apache.predictionio.examples.java.recommendations.tutorial4.EngineFactory",
- "version": "0.9.1",
- "name": "FeatureBased Recommendations Engine",
- "engineFactory": "org.apache.predictionio.examples.java.recommendations.tutorial4.EngineFactory"
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-algo-engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-algo-engine.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-algo-engine.json
deleted file mode 100644
index ceaf576..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-algo-engine.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "id": "org.apache.predictionio.examples.java.recommendations.tutorial4.SingleEngineFactory",
- "version": "0.9.1",
- "name": "FeatureBased Recommendations Engine",
- "engineFactory": "org.apache.predictionio.examples.java.recommendations.tutorial4.SingleEngineFactory"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/algorithms.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/algorithms.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/algorithms.json
deleted file mode 100644
index c0f2b77..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/algorithms.json
+++ /dev/null
@@ -1,11 +0,0 @@
-[
- {
- "name": "featurebased",
- "params": {
- "min": 1.0,
- "max": 5.0,
- "drift": 3.0,
- "scale": 0.5
- }
- }
-]
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/datasource.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/datasource.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/datasource.json
deleted file mode 100644
index 11a3a34..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/single-jsons/datasource.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "dir" : "data/ml-100k/",
- "addFakeData": true
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/EngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/EngineFactory.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/EngineFactory.java
deleted file mode 100644
index ef59694..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/EngineFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial5;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial3.DataSource;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.TrainingData;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Query;
-
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaSimpleEngine;
-import org.apache.predictionio.controller.java.JavaSimpleEngineBuilder;
-
-public class EngineFactory implements IJavaEngineFactory {
- public JavaSimpleEngine<TrainingData, Object, Query, Float, Float> apply() {
- return new JavaSimpleEngineBuilder<
- TrainingData, Object, Query, Float, Float> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass() // Use default Preparator
- .addAlgorithmClass("MyMahoutRecommendationAlgo", MahoutAlgorithm.class)
- .servingClass() // Use default Serving
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoModel.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoModel.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoModel.java
deleted file mode 100644
index 5c87706..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoModel.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial5;
-
-import java.io.Serializable;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectStreamException;
-import org.apache.mahout.cf.taste.recommender.Recommender;
-import org.apache.mahout.cf.taste.impl.recommender.GenericItemBasedRecommender;
-import org.apache.mahout.cf.taste.model.DataModel;
-import org.apache.mahout.cf.taste.similarity.ItemSimilarity;
-import org.apache.mahout.cf.taste.impl.similarity.LogLikelihoodSimilarity;
-import org.apache.mahout.cf.taste.impl.similarity.TanimotoCoefficientSimilarity;
-import com.esotericsoftware.kryo.KryoSerializable;
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MahoutAlgoModel implements Serializable, KryoSerializable {
-
- private DataModel dataModel;
- private MahoutAlgoParams params;
- private transient Recommender recommender; // declare "transient" because it's not serializable.
-
- final static Logger logger = LoggerFactory.getLogger(MahoutAlgoModel.class);
-
- public MahoutAlgoModel(DataModel dataModel, MahoutAlgoParams params) {
- this.dataModel = dataModel;
- this.params = params;
- this.recommender = buildRecommender(this.dataModel, this.params);
- }
-
- public Recommender getRecommender() {
- return this.recommender;
- }
-
- private Recommender buildRecommender(DataModel dataModel, MahoutAlgoParams params) {
- ItemSimilarity similarity;
- switch (params.itemSimilarity) {
- case MahoutAlgoParams.LOG_LIKELIHOOD:
- similarity = new LogLikelihoodSimilarity(dataModel);
- break;
- case MahoutAlgoParams.TANIMOTO_COEFFICIENT:
- similarity = new TanimotoCoefficientSimilarity(dataModel);
- break;
- default:
- logger.error("Invalid itemSimilarity: " + params.itemSimilarity +
- ". LogLikelihoodSimilarity is used.");
- similarity = new LogLikelihoodSimilarity(dataModel);
- break;
- }
- return new GenericItemBasedRecommender(
- dataModel,
- similarity
- );
- }
-
- // KryoSerializable interface
- public void write (Kryo kryo, Output output) {
- kryo.writeClassAndObject(output, this.dataModel);
- kryo.writeClassAndObject(output, this.params);
- }
-
- // KryoSerializable interface
- public void read (Kryo kryo, Input input) {
- this.dataModel = (DataModel) kryo.readClassAndObject(input);
- this.params = (MahoutAlgoParams) kryo.readClassAndObject(input);
- this.recommender = buildRecommender(this.dataModel, this.params); // recover the recommender
- }
-
- @Override
- public String toString() {
- return "Mahout Recommender";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoParams.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoParams.java
deleted file mode 100644
index 9c95cc6..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgoParams.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial5;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class MahoutAlgoParams implements JavaParams {
-
- String itemSimilarity;
-
- final static String LOG_LIKELIHOOD = "LogLikelihoodSimilarity";
- final static String TANIMOTO_COEFFICIENT = "TanimotoCoefficientSimilarity";
-
- public MahoutAlgoParams(String itemSimilarity) {
- this.itemSimilarity = itemSimilarity;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgorithm.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgorithm.java
deleted file mode 100644
index 766ef52..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/MahoutAlgorithm.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial5;
-
-import org.apache.predictionio.controller.java.LJavaAlgorithm;
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.TrainingData;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Query;
-import org.apache.predictionio.engines.util.MahoutUtil;
-
-import org.apache.mahout.cf.taste.recommender.Recommender;
-import org.apache.mahout.cf.taste.model.DataModel;
-import org.apache.mahout.cf.taste.common.TasteException;
-import scala.Tuple4;
-import java.util.List;
-import java.util.ArrayList;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/* Simple Mahout ItemBased Algorithm integration for demonstration purpose */
-public class MahoutAlgorithm extends
- LJavaAlgorithm<EmptyParams, TrainingData, MahoutAlgoModel, Query, Float> {
-
- final static Logger logger = LoggerFactory.getLogger(MahoutAlgorithm.class);
-
- MahoutAlgoParams params;
-
- public MahoutAlgorithm(MahoutAlgoParams params) {
- this.params = params;
- }
-
- @Override
- public MahoutAlgoModel train(TrainingData data) {
- List<Tuple4<Integer, Integer, Float, Long>> ratings = new ArrayList<
- Tuple4<Integer, Integer, Float, Long>>();
- for (TrainingData.Rating r : data.ratings) {
- // no timestamp
- ratings.add(new Tuple4<Integer, Integer, Float, Long>(r.uid, r.iid, r.rating, 0L));
- }
- DataModel dataModel = MahoutUtil.jBuildDataModel(ratings);
- return new MahoutAlgoModel(dataModel, params);
- }
-
- @Override
- public Float predict(MahoutAlgoModel model, Query query) {
- float predicted;
- try {
- predicted = model.getRecommender().estimatePreference((long) query.uid, (long) query.iid);
- } catch (TasteException e) {
- predicted = Float.NaN;
- }
- return predicted;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/Runner5.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/Runner5.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/Runner5.java
deleted file mode 100644
index 2a8e853..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/Runner5.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial5;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial1.DataSourceParams;
-import org.apache.predictionio.examples.java.recommendations.tutorial3.Evaluator;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaSimpleEngine;
-import org.apache.predictionio.controller.java.JavaSimpleEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-public class Runner5 {
-
- public static void runEvaluation(String filePath) {
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(filePath))
- .addAlgorithmParams("MyMahoutRecommendationAlgo",
- new MahoutAlgoParams("LogLikelihoodSimilarity"))
- .build();
-
- JavaWorkflow.runEngine(
- (new EngineFactory()).apply(),
- engineParams,
- Evaluator.class,
- new EmptyParams(),
- new WorkflowParamsBuilder().batch("MyEngine").verbose(3).build()
- );
- }
-
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file path as argument");
- System.exit(1);
- }
- runEvaluation(args[0]);
- System.exit(0); // clean shutdown is needed for spark
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/manifest.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/manifest.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/manifest.json
deleted file mode 100644
index b9cdbe3..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/manifest.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "id": "org.apache.predictionio.examples.java.recommendations.tutorial5.EngineFactory",
- "version": "0.8.1-SNAPSHOT",
- "name": "Simple Mahout Recommendations Engine",
- "engineFactory": "org.apache.predictionio.examples.java.recommendations.tutorial5.EngineFactory"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/algorithmsParams.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/algorithmsParams.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/algorithmsParams.json
deleted file mode 100644
index 7decbca..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/algorithmsParams.json
+++ /dev/null
@@ -1,5 +0,0 @@
-[
- { "name": "MyMahoutRecommendationAlgo",
- "params" : { "itemSimilarity" : "LogLikelihoodSimilarity" }
- }
-]
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/dataSourceParams.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/dataSourceParams.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/dataSourceParams.json
deleted file mode 100644
index bf00f87..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial5/params/dataSourceParams.json
+++ /dev/null
@@ -1 +0,0 @@
-{ "filePath" : "data/ml-100k/u.data" }
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/build.sbt b/examples/experimental/java-parallel-helloworld/build.sbt
deleted file mode 100644
index ea89b68..0000000
--- a/examples/experimental/java-parallel-helloworld/build.sbt
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "example-java-parallel-helloworld"
-
-organization := "org.sample"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/engine.json b/examples/experimental/java-parallel-helloworld/engine.json
deleted file mode 100644
index 5ce3737..0000000
--- a/examples/experimental/java-parallel-helloworld/engine.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "id": "default",
- "description": "Parallel Hello World Engine",
- "engineFactory": "org.apache.predictionio.examples.java.parallel.EngineFactory",
- "algorithms": [
- {
- "name": "ParallelAlgorithm",
- "params": {}
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/project/assembly.sbt b/examples/experimental/java-parallel-helloworld/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/java-parallel-helloworld/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Algorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Algorithm.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Algorithm.java
deleted file mode 100644
index 23635d6..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Algorithm.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.PJavaAlgorithm;
-
-import java.io.Serializable;
-import java.util.List;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.api.java.function.Function2;
-import org.apache.spark.api.java.function.PairFunction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import scala.Tuple2;
-
-public class Algorithm extends PJavaAlgorithm<
- EmptyParams, JavaPairRDD<String, Float>, Model, Query, Float> {
-
- final static Logger logger = LoggerFactory.getLogger(Algorithm.class);
-
- public static class ReadingAndCount implements Serializable {
- public float reading;
- public int count;
-
- public ReadingAndCount(float reading, int count) {
- this.reading = reading;
- this.count = count;
- }
-
- public ReadingAndCount(float reading) {
- this(reading, 1);
- }
-
- @Override
- public String toString() {
- return "(reading = " + reading + ", count = " + count + ")";
- }
- }
-
- @Override
- public Model train(JavaPairRDD<String, Float> data) {
- // take averages just like the local helloworld program
- JavaPairRDD<String, Float> averages = data.mapValues(
- new Function<Float, ReadingAndCount>() {
- @Override
- public ReadingAndCount call(Float reading) {
- return new ReadingAndCount(reading);
- }
- }).reduceByKey(
- new Function2<ReadingAndCount, ReadingAndCount, ReadingAndCount>() {
- @Override
- public ReadingAndCount call(ReadingAndCount rac1, ReadingAndCount rac2) {
- return new ReadingAndCount(rac1.reading + rac2.reading, rac1.count + rac2.count);
- }
- }).mapValues(
- new Function<ReadingAndCount, Float>() {
- @Override
- public Float call(ReadingAndCount rac) {
- return rac.reading / rac.count;
- }
- });
- return new Model(averages);
- }
-
- @Override
- public JavaPairRDD<Object, Float> batchPredict(Model model,
- JavaPairRDD<Object, Query> indexedQueries) {
- return model.temperatures.join(indexedQueries.mapToPair(
- new PairFunction<Tuple2<Object, Query>, String, Object>() {
- @Override // reverse the query tuples, then join
- public Tuple2 call(Tuple2<Object, Query> tuple) {
- return new Tuple2(tuple._2.day, tuple._1);
- }
- })).mapToPair(
- new PairFunction<Tuple2<String, Tuple2<Float, Object>>, Object, Float>() {
- @Override // map result back to predictions, dropping the day
- public Tuple2 call(Tuple2<String, Tuple2<Float, Object>> tuple) {
- return new Tuple2(tuple._2._2, tuple._2._1);
- }
- });
- }
-
- @Override
- public Float predict(Model model, Query query) {
- final String day = query.day;
- List<Float> reading = model.temperatures.lookup(day);
- if (reading.size() == 0) {
- return -10000f; // JSON does not support NaN
- }
- return reading.get(0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/DataSource.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/DataSource.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/DataSource.java
deleted file mode 100644
index c2ed854..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/DataSource.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.PJavaDataSource;
-
-import java.util.List;
-import java.util.ArrayList;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.PairFunction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import scala.Tuple2;
-import scala.Tuple3;
-
-public class DataSource extends PJavaDataSource<
- EmptyParams, Object, JavaPairRDD<String, Float>, Query, Object> {
-
- final static Logger logger = LoggerFactory.getLogger(DataSource.class);
-
- public DataSource() {
- }
-
- @Override
- public Iterable<Tuple3<Object, JavaPairRDD<String, Float>, JavaPairRDD<Query, Object>>>
- read(JavaSparkContext jsc) {
- JavaPairRDD<String, Float> readings = jsc.textFile("../data/helloworld/data.csv")
- .mapToPair(new PairFunction<String, String, Float>() {
- @Override
- public Tuple2 call(String line) {
- String[] tokens = line.split("[\t,]");
- Tuple2 reading = null;
- try {
- reading = new Tuple2(
- tokens[0],
- Float.parseFloat(tokens[1]));
- } catch (Exception e) {
- logger.error("Can't parse reading file. Caught Exception: " + e.getMessage());
- System.exit(1);
- }
- return reading;
- }
- });
-
- List<Tuple3<Object, JavaPairRDD<String, Float>, JavaPairRDD<Query, Object>>> data =
- new ArrayList<>();
-
- data.add(new Tuple3(
- null,
- readings,
- jsc.parallelizePairs(new ArrayList<Tuple2<Query, Object>>())
- ));
-
- return data;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/EngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/EngineFactory.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/EngineFactory.java
deleted file mode 100644
index 2b14367..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/EngineFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.LJavaFirstServing;
-import org.apache.predictionio.controller.java.PJavaEngine;
-import org.apache.predictionio.controller.java.PJavaEngineBuilder;
-
-import java.util.HashMap;
-
-import org.apache.spark.api.java.JavaPairRDD;
-
-public class EngineFactory implements IJavaEngineFactory {
- public PJavaEngine<JavaPairRDD<String, Float>, Object, JavaPairRDD<String, Float>, Query, Float,
- Object> apply() {
- return new PJavaEngineBuilder<JavaPairRDD<String, Float>, Object, JavaPairRDD<String, Float>,
- Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass(Preparator.class)
- .addAlgorithmClass("ParallelAlgorithm", Algorithm.class)
- .servingClass(Serving.class)
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Model.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Model.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Model.java
deleted file mode 100644
index ca662d0..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Model.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import java.io.Serializable;
-import java.lang.StringBuilder;
-import java.util.List;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import scala.Tuple2;
-
-public class Model implements Serializable {
- private static int LIST_THRESHOLD = 20;
-
- public JavaPairRDD<String, Float> temperatures;
-
- public Model(JavaPairRDD<String, Float> temperatures) {
- this.temperatures = temperatures;
- }
-
- @Override
- public String toString() {
- boolean longList = temperatures.count() > LIST_THRESHOLD ? true : false;
- List<Tuple2<String, Float>> readings =
- temperatures.take(longList ? LIST_THRESHOLD : (int) temperatures.count());
- StringBuilder builder = new StringBuilder();
- builder.append("(");
- boolean first = true;
- for (Tuple2<String, Float> reading : readings) {
- if (!first) {
- builder.append(", ");
- } else {
- first = false;
- }
- builder.append(reading);
- }
- if (longList) {
- builder.append(", ...");
- }
- builder.append(")");
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Preparator.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Preparator.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Preparator.java
deleted file mode 100644
index 6391999..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Preparator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.PJavaPreparator;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.Function;
-
-public class Preparator extends
- PJavaPreparator<EmptyParams, JavaPairRDD<String, Float>, JavaPairRDD<String, Float>> {
-
- @Override
- public JavaPairRDD<String, Float> prepare(JavaSparkContext jsc,
- JavaPairRDD<String, Float> data) {
- return data.mapValues(new Function<Float, Float>() {
- @Override
- public Float call(Float temperature) {
- // let's convert it to degrees Celsius
- return (temperature - 32.0f) / 9 * 5;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Query.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Query.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Query.java
deleted file mode 100644
index e596c2d..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Query.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import java.io.Serializable;
-
-public class Query implements Serializable {
- public String day;
-
- public Query(String day) {
- this.day = day;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Runner.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Runner.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Runner.java
deleted file mode 100644
index 53273a7..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Runner.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import org.apache.predictionio.controller.IEngineFactory;
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.PJavaEngine;
-import org.apache.predictionio.controller.java.PJavaEngineBuilder;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-import org.apache.spark.api.java.JavaPairRDD;
-
-public class Runner {
-
- // During development, one can build a semi-engine, only add the first few layers. In this
- // particular example, we only add until dataSource layer
- private static class HalfBakedEngineFactory implements IJavaEngineFactory {
- public PJavaEngine<JavaPairRDD<String, Float>, Object, JavaPairRDD<String, Float>,
- Query, Float, Object> apply() {
- return new PJavaEngineBuilder<
- JavaPairRDD<String, Float>, Object, JavaPairRDD<String, Float>, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass(Preparator.class)
- .addAlgorithmClass("ParallelAlgorithm", Algorithm.class)
- .servingClass(Serving.class)
- .build();
- }
- }
-
- public static void runComponents() {
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .addAlgorithmParams("ParallelAlgorithm", new EmptyParams())
- .servingParams(new EmptyParams())
- .build();
- JavaWorkflow.runEngine(
- (new HalfBakedEngineFactory()).apply(),
- engineParams,
- null,
- new EmptyParams(),
- new WorkflowParamsBuilder().batch("MyEngine").verbose(3).build()
- );
- }
-
- public static void main(String[] args) {
- runComponents();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Serving.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Serving.java b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Serving.java
deleted file mode 100644
index 40c1e7b..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/Serving.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.parallel;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.LJavaServing;
-
-public class Serving extends LJavaServing<EmptyParams, Query, Float> {
-
- public Serving() {
-
- }
-
- @Override
- public Float serve(Query query, Iterable<Float> predictions) {
- return predictions.iterator().next();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-parallel-helloworld/src/main/java/parallel/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/build.sbt b/examples/experimental/java-parallel-helloworld/src/main/java/parallel/build.sbt
deleted file mode 100644
index 56dd2ba..0000000
--- a/examples/experimental/java-parallel-helloworld/src/main/java/parallel/build.sbt
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "example-java-parallel"
-
-organization := "org.apache.predictionio.examples.java"
-
-resolvers += Resolver.sonatypeRepo("snapshots")
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.8.0-SNAPSHOT" % "provided",
- "org.apache.predictionio" %% "data" % "0.8.0-SNAPSHOT" % "provided",
- "org.apache.spark" %% "spark-core" % "1.0.2" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/README.md b/examples/experimental/scala-cleanup-app/README.md
deleted file mode 100644
index 84b6fba..0000000
--- a/examples/experimental/scala-cleanup-app/README.md
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Removing old events from app
-
-## Documentation
-
-This shows how to remove old events from the certain app.
-
-Parameters in engine.json are appId and cutoffTime.
-All events in that appId before the cutoffTime are removed,
-including $set, $unset and $delete
-(so please adapt it for use when you want to preserve these special events).
-
-To use, edit `engine.json`, run `pio build` then `pio train`.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/build.sbt b/examples/experimental/scala-cleanup-app/build.sbt
deleted file mode 100644
index d7ba8a6..0000000
--- a/examples/experimental/scala-cleanup-app/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "template-scala-parallel-vanilla"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.5" % "provided",
- "org.apache.spark" %% "spark-core" % "1.3.1" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.3.1" % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/engine.json b/examples/experimental/scala-cleanup-app/engine.json
deleted file mode 100644
index 6f7f334..0000000
--- a/examples/experimental/scala-cleanup-app/engine.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "id": "default",
- "description": "Default settings",
- "engineFactory": "org.apache.predictionio.examples.experimental.cleanupapp.VanillaEngine",
- "datasource": {
- "params" : {
- "appId": 1000000000,
- "cutoffTime": "2014-04-29T00:00:00.000Z"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/project/assembly.sbt b/examples/experimental/scala-cleanup-app/project/assembly.sbt
deleted file mode 100644
index 54c3252..0000000
--- a/examples/experimental/scala-cleanup-app/project/assembly.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/src/main/scala/Algorithm.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/src/main/scala/Algorithm.scala b/examples/experimental/scala-cleanup-app/src/main/scala/Algorithm.scala
deleted file mode 100644
index 95c69fc..0000000
--- a/examples/experimental/scala-cleanup-app/src/main/scala/Algorithm.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.cleanupapp
-
-import org.apache.predictionio.controller.P2LAlgorithm
-import org.apache.predictionio.controller.Params
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-import grizzled.slf4j.Logger
-
-//case class AlgorithmParams(mult: Int) extends Params
-
-//class Algorithm(val ap: AlgorithmParams)
-class Algorithm
- extends P2LAlgorithm[TrainingData, Model, Query, PredictedResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- def train(sc: SparkContext, data: TrainingData): Model = {
- new Model
- }
-
- def predict(model: Model, query: Query): PredictedResult = {
- // Prefix the query with the model data
- PredictedResult(p = "")
- }
-}
-
-class Model extends Serializable {
- override def toString = "Model"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/src/main/scala/DataSource.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/src/main/scala/DataSource.scala b/examples/experimental/scala-cleanup-app/src/main/scala/DataSource.scala
deleted file mode 100644
index c10bd83..0000000
--- a/examples/experimental/scala-cleanup-app/src/main/scala/DataSource.scala
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.cleanupapp
-
-import org.apache.predictionio.controller.PDataSource
-import org.apache.predictionio.controller.EmptyEvaluationInfo
-import org.apache.predictionio.controller.EmptyActualResult
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.Storage
-import org.apache.predictionio.workflow.StopAfterReadInterruption
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-import com.github.nscala_time.time.Imports._
-
-import grizzled.slf4j.Logger
-
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.{Await, Future}
-
-case class DataSourceParams(
- appId: Int,
- cutoffTime: DateTime
-) extends Params
-
-class DataSource(val dsp: DataSourceParams)
- extends PDataSource[TrainingData,
- EmptyEvaluationInfo, Query, EmptyActualResult] {
-
- @transient lazy val logger = Logger[this.type]
-
- override
- def readTraining(sc: SparkContext): TrainingData = {
- val eventsDb = Storage.getPEvents()
- val lEventsDb = Storage.getLEvents()
- logger.info(s"CleanupApp: $dsp")
-
- val countBefore = eventsDb.find(
- appId = dsp.appId
- )(sc).count
- logger.info(s"Event count before cleanup: $countBefore")
-
- val countRemove = eventsDb.find(
- appId = dsp.appId,
- untilTime = Some(dsp.cutoffTime)
- )(sc).count
- logger.info(s"Number of events to remove: $countRemove")
-
- logger.info(s"Remove events from appId ${dsp.appId}")
- val eventsToRemove: Array[String] = eventsDb.find(
- appId = dsp.appId,
- untilTime = Some(dsp.cutoffTime)
- )(sc).map { case e =>
- e.eventId.getOrElse("")
- }.collect
-
- var lastFuture: Future[Boolean] = Future[Boolean] {true}
- eventsToRemove.foreach { case eventId =>
- if (eventId != "") {
- lastFuture = lEventsDb.futureDelete(eventId, dsp.appId)
- }
- }
- // No, it's not correct to just wait for the last result.
- // This program only demonstrates how to remove old events.
- Await.result(lastFuture, scala.concurrent.duration.Duration(5, "minutes"))
- logger.info(s"Finish cleaning up events to appId ${dsp.appId}")
-
- val countAfter = eventsDb.find(
- appId = dsp.appId
- )(sc).count
- logger.info(s"Event count after cleanup: $countAfter")
-
- throw new StopAfterReadInterruption()
- }
-}
-
-class TrainingData(
-) extends Serializable {
- override def toString = ""
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/src/main/scala/Engine.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/src/main/scala/Engine.scala b/examples/experimental/scala-cleanup-app/src/main/scala/Engine.scala
deleted file mode 100644
index 5a0d5a4..0000000
--- a/examples/experimental/scala-cleanup-app/src/main/scala/Engine.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.cleanupapp
-
-import org.apache.predictionio.controller.IEngineFactory
-import org.apache.predictionio.controller.Engine
-import org.apache.predictionio.controller._
-
-case class Query(q: String) extends Serializable
-
-case class PredictedResult(p: String) extends Serializable
-
-object VanillaEngine extends IEngineFactory {
- def apply() = {
- new Engine(
- classOf[DataSource],
- PIdentityPreparator(classOf[DataSource]),
- Map("" -> classOf[Algorithm]),
- classOf[Serving])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/src/main/scala/Preparator.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/src/main/scala/Preparator.scala b/examples/experimental/scala-cleanup-app/src/main/scala/Preparator.scala
deleted file mode 100644
index ab8514a..0000000
--- a/examples/experimental/scala-cleanup-app/src/main/scala/Preparator.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.cleanupapp
-
-import org.apache.predictionio.controller.PPreparator
-import org.apache.predictionio.data.storage.Event
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-/*
-class Preparator
- extends PPreparator[TrainingData, PreparedData] {
-
- def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
- new PreparedData(events = trainingData.events)
- }
-}
-
-class PreparedData(
- val events: RDD[Event]
-) extends Serializable
-*/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-cleanup-app/src/main/scala/Serving.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-cleanup-app/src/main/scala/Serving.scala b/examples/experimental/scala-cleanup-app/src/main/scala/Serving.scala
deleted file mode 100644
index 82620c3..0000000
--- a/examples/experimental/scala-cleanup-app/src/main/scala/Serving.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.experimental.cleanupapp
-
-import org.apache.predictionio.controller.LServing
-
-class Serving
- extends LServing[Query, PredictedResult] {
-
- override
- def serve(query: Query,
- predictedResults: Seq[PredictedResult]): PredictedResult = {
- predictedResults.head
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/.gitignore
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/.gitignore b/examples/experimental/scala-local-friend-recommendation/.gitignore
deleted file mode 100644
index 8fce603..0000000
--- a/examples/experimental/scala-local-friend-recommendation/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-data/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/README.md
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/README.md b/examples/experimental/scala-local-friend-recommendation/README.md
deleted file mode 100644
index 0d9ef0f..0000000
--- a/examples/experimental/scala-local-friend-recommendation/README.md
+++ /dev/null
@@ -1,127 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-##Our Engine
-We are creating an engine in PredictionIO for friend/item recommendation in social network settings. It learns from user profiles, item information, social interactions as well as past recommendation history and builds a model to provide suggestions on friend/item for users.
-
-#####Two algorithms are implemented:
-1. Random
-2. Keyword Similarity KNN
-
-#####Expected data:
-The dataset for KDD Cup 2012 Track 1 is required. <br />
-The KDD cup page can be found <a href="https://www.kddcup2012.org/c/kddcup2012-track1">here</a><br />
-The dataset are <a href="https://www.kddcup2012.org/c/kddcup2012-track1/data">here</a><br />
-The below files are required to put into the *data* folder.
-
-1. item.txt
-2. user_profile.txt
-3. user\_key\_word.txt
-4. user_action.txt
-5. user_sns.txt
-6. rec\_log\_train.txt
-
-#####Sampling a subset of data:
-You can sample a subset of the data with *file_random.py*
-```
-python file_random $UserSize $ItemSize
-```
-*$UserSize* and *$ItemSize* are the sample sizes of users and items respectively.
-
-Put the input files into the data folder
-
-The program runs with files:
-
-1. item.txt
-2. user_profile.txt
-3. user\_key\_word.txt
-4. user\_action.txt
-5. user\_sns.txt
-6. rec\_log\_train.txt
-
-And output files:
-
-1. mini_item.txt
-2. mini\_user\_profile.txt
-3. mini\_user_key_word.txt
-4. mini\_user_action.txt
-5. mini\_user\_sns.txt
-6. mini\_rec\_log_train.txt
-
-After sampling, please set the file path parameters in *$EngineJson* (described below) to point to the output in order to use them.
-
-#####Notice about Spark settings:
-As the data set is large, we recommend setting spark memories to be large. Please set the below two lines with the two values *$E_M* and *$D_M* in the *$SPARK_HOME/conf/spark-defaults.conf*
-
-1. spark.executor.memory *$E_M*
-2. spark.driver.memory *$D_M*
-
-We have tested "Random" and "Keyword Similarity KNN" algorithms with *$E_M* = 16g and *$D_M* = 16g.
-
-#####To run the engine, you need to Build + Train + Deploy:
-```
-$PIO_HOME/bin/pio build -v $EngineJson
-
-$PIO_HOME/bin/pio train -v $EngineJson
-
-$PIO_HOME/bin/pio deploy -v $EngineJson
-```
-
-$EngineJson is
-
-1. "random_engine.json" for "Random"
-2. "keyword_similarity_engine.json" for "Keyword Similarity KNN"
-
-Note: if the accesskey error rises when deploying, please set it to any dummy value and then the program will work.
-
-#####To query:
-```
-curl -H "Content-Type: application/json" -d '{ "user": $UserId , "item" : $ItemId}' http://localhost:8000/queries.json
-```
-
-*$UserId* and *$ItemId* are the user and item you want to query.
-
-#####Prediction provided:
-Our local algorithm provides two predicted values as below for each user-item pair queried.
-
-1. confidence (how confident the algorithm is to predict that the user will accept the item)
-2. acceptance (when the confidence is high, the algorithm will predict that the user will accept the item)
-
-#####Example *$UserId*-*$ItemId* pairs:
-While "Random" provides a uniformly distributed random result for any query,
-"Keyword Similarity KNN" may generate a zero confidence score for many queries.
-This is due to the sparsity of the keyword space in the given data that many id pairs
-do not have overlapping keywords. Below are six example pairs of *$UserId*-*$ItemId*
-in the full data set that generates positive scores.
-
-1. 1051216-1774844
-2. 1052586-1774844
-3. 1001726-1775038
-4. 1019789-1775038
-5. 1041986-1774522
-6. 1048438-1774522
-
-#####Threshold training for "Keyword Similarity KNN":
-When the confidence is higher than a threshold,
-the algorithm will predict that the user will accept the item.
-Instead of mannually setting the threshold,
-perceptron was implemented to train the threshold automatically.
-However, due to the high complexity in time and space,
-this part is commented out.
-Users can change the corresponding parts in FriendRecommendationDataSource.scala
-and eywordSimilarityAlgorithm.scala
-with comments about "training an acceptance threshold".
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/scala-local-friend-recommendation/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-friend-recommendation/build.sbt b/examples/experimental/scala-local-friend-recommendation/build.sbt
deleted file mode 100644
index 63ff005..0000000
--- a/examples/experimental/scala-local-friend-recommendation/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AssemblyKeys._
-
-assemblySettings
-
-name := "examples-friendrecommendation"
-
-organization := "org.apache.predictionio"
-
-libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.predictionio" %% "data" % "0.9.1" % "provided",
- "org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
[11/50] [abbrv] incubator-predictionio git commit: [PIO-107] Removal
of examples under examples/experimental.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSource.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSource.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSource.java
deleted file mode 100644
index f8c5907..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSource.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import org.apache.predictionio.controller.java.LJavaDataSource;
-import scala.Tuple2;
-import scala.Tuple3;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.lang.Iterable;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Scanner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DataSource extends LJavaDataSource<
- DataSourceParams, Object, TrainingData, Query, Object> {
-
- final static Logger logger = LoggerFactory.getLogger(DataSource.class);
-
- DataSourceParams params;
-
- public DataSource(DataSourceParams params) {
- this.params = params;
- }
-
- @Override
- public Iterable<Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Object>>>> read() {
-
- File ratingFile = new File(params.filePath);
- Scanner sc = null;
-
- try {
- sc = new Scanner(ratingFile);
- } catch (FileNotFoundException e) {
- logger.error("Caught FileNotFoundException " + e.getMessage());
- System.exit(1);
- }
-
- List<TrainingData.Rating> ratings = new ArrayList<TrainingData.Rating>();
-
- while (sc.hasNext()) {
- String line = sc.nextLine();
- String[] tokens = line.split("[\t,]");
- try {
- TrainingData.Rating rating = new TrainingData.Rating(
- Integer.parseInt(tokens[0]),
- Integer.parseInt(tokens[1]),
- Float.parseFloat(tokens[2]));
- ratings.add(rating);
- } catch (Exception e) {
- logger.error("Can't parse rating file. Caught Exception: " + e.getMessage());
- System.exit(1);
- }
- }
-
- List<Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Object>>>> data =
- new ArrayList<Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Object>>>>();
-
- data.add(new Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Object>>>(
- null,
- new TrainingData(ratings),
- new ArrayList<Tuple2<Query, Object>>()
- ));
-
- return data;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSourceParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSourceParams.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSourceParams.java
deleted file mode 100644
index aae78c6..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/DataSourceParams.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class DataSourceParams implements JavaParams {
- public String filePath; // file path
-
- public DataSourceParams(String path) {
- this.filePath = path;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/EngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/EngineFactory.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/EngineFactory.java
deleted file mode 100644
index 66bcee9..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/EngineFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaSimpleEngine;
-import org.apache.predictionio.controller.java.JavaSimpleEngineBuilder;
-
-public class EngineFactory implements IJavaEngineFactory {
- public JavaSimpleEngine<TrainingData, Object, Query, Float, Object> apply() {
- return new JavaSimpleEngineBuilder<
- TrainingData, Object, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass() // Use default Preparator
- .addAlgorithmClass("MyRecommendationAlgo", Algorithm.class)
- .servingClass() // Use default Serving
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Model.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Model.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Model.java
deleted file mode 100644
index 0d4a2ab..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Model.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import java.io.Serializable;
-import java.util.Map;
-import org.apache.commons.math3.linear.RealVector;
-
-public class Model implements Serializable {
- public Map<Integer, RealVector> itemSimilarity;
- public Map<Integer, RealVector> userHistory;
-
- public Model(Map<Integer, RealVector> itemSimilarity,
- Map<Integer, RealVector> userHistory) {
- this.itemSimilarity = itemSimilarity;
- this.userHistory = userHistory;
- }
-
- @Override
- public String toString() {
- String s;
- if ((itemSimilarity.size() > 20) || (userHistory.size() > 20)) {
- s = "Model: [itemSimilarity.size=" + itemSimilarity.size() + "]\n"
- +"[userHistory.size=" + userHistory.size() + "]";
- } else {
- s = "Model: [itemSimilarity: " + itemSimilarity.toString() + "]\n"
- +"[userHistory: " + userHistory.toString() + "]";
- }
- return s;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Query.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Query.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Query.java
deleted file mode 100644
index b17e5a0..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/Query.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import java.io.Serializable;
-
-public class Query implements Serializable {
- public int uid; // user ID
- public int iid; // item ID
-
- public Query(int uid, int iid) {
- this.uid = uid;
- this.iid = iid;
- }
-
- @Override
- public String toString() {
- return "(" + uid + "," + iid + ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/TrainingData.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/TrainingData.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/TrainingData.java
deleted file mode 100644
index a4cb567..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/TrainingData.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial1;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class TrainingData implements Serializable {
- public List<Rating> ratings;
-
- public TrainingData(List<Rating> ratings) {
- this.ratings = ratings;
- }
-
- @Override
- public String toString() {
- String s;
- if (ratings.size() > 20)
- s = "TrainingData.size=" + ratings.size();
- else
- s = ratings.toString();
- return s;
- }
-
- public static class Rating implements Serializable {
- public int uid; // user ID
- public int iid; // item ID
- public float rating;
-
- public Rating(int uid, int iid, float rating) {
- this.uid = uid;
- this.iid = iid;
- this.rating = rating;
- }
-
- @Override
- public String toString() {
- return "(" + uid + "," + iid + "," + rating + ")";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/engine.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/engine.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/engine.json
deleted file mode 100644
index 12c3927..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/engine.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "id": "org.apache.predictionio.examples.java.recommendations.tutorial1.EngineFactory",
- "version": "0.9.1",
- "name": "Simple Recommendations Engine",
- "engineFactory": "org.apache.predictionio.examples.java.recommendations.tutorial1.EngineFactory"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/algorithms.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/algorithms.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/algorithms.json
deleted file mode 100644
index 0f9715e..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/algorithms.json
+++ /dev/null
@@ -1,5 +0,0 @@
-[
- { "name": "MyRecommendationAlgo",
- "params" : { "threshold" : 0.2 }
- }
-]
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/datasource.json
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/datasource.json b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/datasource.json
deleted file mode 100644
index bf00f87..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial1/params/datasource.json
+++ /dev/null
@@ -1 +0,0 @@
-{ "filePath" : "data/ml-100k/u.data" }
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner1.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner1.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner1.java
deleted file mode 100644
index 71b694a..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner1.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial2;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial1.TrainingData;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Query;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.DataSource;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.DataSourceParams;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaSimpleEngine;
-import org.apache.predictionio.controller.java.JavaSimpleEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-import java.util.HashMap;
-
-public class Runner1 {
-
- // During development, one can build a semi-engine, only add the first few layers. In this
- // particular example, we only add until dataSource layer
- private static class HalfBakedEngineFactory implements IJavaEngineFactory {
- public JavaSimpleEngine<TrainingData, Object, Query, Float, Object> apply() {
- return new JavaSimpleEngineBuilder<
- TrainingData, Object, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .build();
- }
- }
-
- public static void runComponents(String filePath) {
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(filePath))
- .build();
-
- JavaWorkflow.runEngine(
- (new HalfBakedEngineFactory()).apply(),
- engineParams,
- null,
- new EmptyParams(),
- new WorkflowParamsBuilder().batch("MyEngine").verbose(3).build()
- );
- }
-
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file path as argument");
- System.exit(1);
- }
- runComponents(args[0]);
- System.exit(0); // clean shutdown is needed for spark
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner2.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner2.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner2.java
deleted file mode 100644
index 1d82ad1..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial2/Runner2.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial2;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial1.TrainingData;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Query;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.DataSource;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.DataSourceParams;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Algorithm;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.AlgoParams;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaSimpleEngine;
-import org.apache.predictionio.controller.java.JavaSimpleEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-import org.apache.predictionio.controller.IdentityPreparator;
-
-public class Runner2 {
-
- // During development, one can build a semi-engine, only add the first few layers. In this
- // particular example, we only add until dataSource layer
- private static class HalfBakedEngineFactory implements IJavaEngineFactory {
- public JavaSimpleEngine<TrainingData, Object, Query, Float, Object> apply() {
- return new JavaSimpleEngineBuilder<
- TrainingData, Object, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass() // Use default Preparator
- .addAlgorithmClass("MyRecommendationAlgo", Algorithm.class) // Add Algorithm
- .build();
- }
- }
-
- public static void runComponents(String filePath) {
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(filePath))
- .addAlgorithmParams("MyRecommendationAlgo", new AlgoParams(0.2)) // Add Algorithm Params
- .build();
-
- JavaWorkflow.runEngine(
- (new HalfBakedEngineFactory()).apply(),
- engineParams,
- null,
- new EmptyParams(),
- new WorkflowParamsBuilder().batch("MyEngine").verbose(3).build()
- );
- }
-
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file path as argument");
- System.exit(1);
- }
- runComponents(args[0]);
- System.exit(0); // clean shutdown is needed for spark
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/DataSource.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/DataSource.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/DataSource.java
deleted file mode 100644
index 1ee125d..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/DataSource.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial3;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial1.TrainingData;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Query;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.DataSourceParams;
-
-import org.apache.predictionio.controller.java.LJavaDataSource;
-import scala.Tuple2;
-import scala.Tuple3;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.lang.Iterable;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Scanner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-import java.util.Collections;
-
-public class DataSource extends LJavaDataSource<
- DataSourceParams, Object, TrainingData, Query, Float> {
-
- final static Logger logger = LoggerFactory.getLogger(DataSource.class);
-
- DataSourceParams params;
-
- public DataSource(DataSourceParams params) {
- this.params = params;
- }
-
- @Override
- public Iterable<Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Float>>>> read() {
-
- File ratingFile = new File(params.filePath);
- Scanner sc = null;
-
- try {
- sc = new Scanner(ratingFile);
- } catch (FileNotFoundException e) {
- logger.error("Caught FileNotFoundException " + e.getMessage());
- System.exit(1);
- }
-
- List<TrainingData.Rating> ratings = new ArrayList<TrainingData.Rating>();
-
- while (sc.hasNext()) {
- String line = sc.nextLine();
- String[] tokens = line.split("[\t,]");
- try {
- TrainingData.Rating rating = new TrainingData.Rating(
- Integer.parseInt(tokens[0]),
- Integer.parseInt(tokens[1]),
- Float.parseFloat(tokens[2]));
- ratings.add(rating);
- } catch (Exception e) {
- logger.error("Can't parse rating file. Caught Exception: " + e.getMessage());
- System.exit(1);
- }
- }
-
- int size = ratings.size();
- float trainingPercentage = 0.8f;
- float testPercentage = 1 - trainingPercentage;
- int iterations = 3;
-
- // cap by original size
- int trainingEndIndex = Math.min(size,
- (int) (ratings.size() * trainingPercentage));
- int testEndIndex = Math.min(size,
- trainingEndIndex + (int) (ratings.size() * testPercentage));
- // trainingEndIndex + 10);
-
- Random rand = new Random(0); // seed
-
- List<Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Float>>>> data = new
- ArrayList<Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Float>>>>();
-
- for (int i = 0; i < iterations; i++) {
- Collections.shuffle(ratings, new Random(rand.nextInt()));
-
- // create a new ArrayList because subList() returns view and not serialzable
- List<TrainingData.Rating> trainingRatings =
- new ArrayList<TrainingData.Rating>(ratings.subList(0, trainingEndIndex));
- List<TrainingData.Rating> testRatings = ratings.subList(trainingEndIndex, testEndIndex);
- TrainingData td = new TrainingData(trainingRatings);
- List<Tuple2<Query, Float>> qaList = prepareValidation(testRatings);
-
- data.add(new Tuple3<Object, TrainingData, Iterable<Tuple2<Query, Float>>>(
- null, td, qaList));
- }
-
- return data;
- }
-
- private List<Tuple2<Query, Float>> prepareValidation(List<TrainingData.Rating> testRatings) {
- List<Tuple2<Query, Float>> validationList = new ArrayList<Tuple2<Query, Float>>();
-
- for (TrainingData.Rating r : testRatings) {
- validationList.add(new Tuple2<Query, Float>(
- new Query(r.uid, r.iid),
- r.rating));
- }
-
- return validationList;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/EngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/EngineFactory.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/EngineFactory.java
deleted file mode 100644
index 67c2afc..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/EngineFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial3;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Algorithm;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.TrainingData;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Query;
-
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaSimpleEngine;
-import org.apache.predictionio.controller.java.JavaSimpleEngineBuilder;
-
-public class EngineFactory implements IJavaEngineFactory {
- public JavaSimpleEngine<TrainingData, Object, Query, Float, Float> apply() {
- return new JavaSimpleEngineBuilder<
- TrainingData, Object, Query, Float, Float> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass() // Use default Preparator
- .addAlgorithmClass("MyRecommendationAlgo", Algorithm.class)
- .servingClass() // Use default Serving
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Evaluator.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Evaluator.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Evaluator.java
deleted file mode 100644
index a5c712d..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Evaluator.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial3;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial1.Query;
-import org.apache.predictionio.controller.java.JavaEvaluator;
-import org.apache.predictionio.controller.java.EmptyParams;
-
-import scala.Tuple2;
-import java.util.Arrays;
-import org.apache.commons.collections.IteratorUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** Root mean square error */
-public class Evaluator
- extends JavaEvaluator<EmptyParams, Object, Query, Float, Float,
- Double, Double, String> {
-
- final static Logger logger = LoggerFactory.getLogger(Evaluator.class);
-
- @Override
- public Double evaluateUnit(Query query, Float predicted, Float actual) {
- logger.info("Q: " + query.toString() + " P: " + predicted + " A: " + actual);
- // return squared error
- double error;
- if (predicted.isNaN())
- error = -actual;
- else
- error = predicted - actual;
- return (error * error);
- }
-
- @Override
- public Double evaluateSet(Object dataParams, Iterable<Double> metricUnits) {
- double sum = 0.0;
- int count = 0;
- for (double squareError : metricUnits) {
- sum += squareError;
- count += 1;
- }
- return Math.sqrt(sum / count);
- }
-
- @Override
- public String evaluateAll(
- Iterable<Tuple2<Object, Double>> input) {
- return Arrays.toString(IteratorUtils.toArray(input.iterator()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Runner3.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Runner3.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Runner3.java
deleted file mode 100644
index dbd5baa..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial3/Runner3.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial3;
-
-import org.apache.predictionio.examples.java.recommendations.tutorial1.DataSourceParams;
-import org.apache.predictionio.examples.java.recommendations.tutorial1.AlgoParams;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaSimpleEngine;
-import org.apache.predictionio.controller.java.JavaSimpleEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-public class Runner3 {
-
- public static void runEvaluation(String filePath) {
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(filePath))
- .addAlgorithmParams("MyRecommendationAlgo", new AlgoParams(0.2))
- .build();
-
- JavaWorkflow.runEngine(
- (new EngineFactory()).apply(),
- engineParams,
- Evaluator.class,
- new EmptyParams(),
- new WorkflowParamsBuilder().batch("MyEngine").verbose(3).build()
- );
- }
-
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file path as argument");
- System.exit(1);
- }
- runEvaluation(args[0]);
- System.exit(0); // clean shutdown is needed for spark
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithm.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithm.java
deleted file mode 100644
index 70e5eb2..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithm.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.LJavaAlgorithm;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.commons.math3.linear.ArrayRealVector;
-import org.apache.commons.math3.linear.OpenMapRealVector;
-import scala.Tuple2;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Queue;
-import java.util.PriorityQueue;
-import java.util.Comparator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CollaborativeFilteringAlgorithm extends
- LJavaAlgorithm<CollaborativeFilteringAlgorithmParams, PreparedData, CollaborativeFilteringModel,
- Query, Float> {
-
- final static Logger logger = LoggerFactory.getLogger(CollaborativeFilteringAlgorithm.class);
-
- CollaborativeFilteringAlgorithmParams params;
-
- public CollaborativeFilteringAlgorithm(CollaborativeFilteringAlgorithmParams params) {
- this.params = params;
- }
-
- @Override
- public CollaborativeFilteringModel train(PreparedData data) {
- // pre-process
- Map<Integer, Map<Integer, Float>> itemMap = new HashMap<Integer, Map<Integer, Float>>();
- Map<Integer, Integer> userIndexMap = new HashMap<Integer, Integer>();
- Map<Integer, Integer> itemIndexMap = new HashMap<Integer, Integer>();
-
- int itemIndex = 0;
- int userIndex = 0;
- for (TrainingData.Rating r : data.ratings) {
- Map<Integer, Float> userRating = itemMap.get(r.iid);
- if (userRating == null) {
- // new item
- userRating = new HashMap<Integer, Float>();
- itemMap.put(r.iid, userRating);
- itemIndexMap.put(r.iid, itemIndex);
- itemIndex += 1; // increment item index for next item
- }
- userRating.put(r.uid, r.rating);
-
- // update user index
- Integer u = userIndexMap.get(r.uid);
- if (u == null) {
- // new user
- userIndexMap.put(r.uid, userIndex);
- userIndex += 1;
- }
- }
-
- int numOfItems = itemIndexMap.size();
- int numOfUsers = userIndexMap.size();
-
- Map<Integer, RealVector> itemVectors = new HashMap<Integer, RealVector>();
- Map<Integer, RealVector> userHistory = new HashMap<Integer, RealVector>();
-
- for (Map.Entry<Integer, Map<Integer, Float>> entry : itemMap.entrySet()) {
- Integer itemID = entry.getKey();
- Integer iindex = itemIndexMap.get(itemID);
- Map<Integer, Float> userRatingMap = entry.getValue();
- RealVector item = new ArrayRealVector(numOfUsers); // dimension is numOfUsers
- for (Map.Entry<Integer, Float> r : userRatingMap.entrySet()) {
- Integer userID = r.getKey();
- Float rating = r.getValue();
- Integer uindex = userIndexMap.get(userID);
- item.setEntry(uindex, rating);
- // update user History
- RealVector user = userHistory.get(userID);
- if (user == null) {
- user = new OpenMapRealVector(numOfItems);
- userHistory.put(userID, user);
- }
- user.setEntry(iindex, rating);
- }
- itemVectors.put(itemID, item);
- }
-
- // calculate sim
-
- Map<Integer, RealVector> itemSimilarity = new HashMap<Integer, RealVector>();
- List<Integer> item1List = new ArrayList<Integer>(itemIndexMap.keySet());
- List<Integer> item2List = new ArrayList<Integer>(item1List);
-
- int numSimilarItems = 100;
- Comparator<IndexAndScore> comparator = new IndexAndScoreComparator();
- Map<Integer, Queue<IndexAndScore>> topItemSimilarity =
- new HashMap<Integer, Queue<IndexAndScore>>();
-
- for (Integer itemID1 : item1List) {
- item2List.remove(0);
- Integer index1 = itemIndexMap.get(itemID1);
- for (Integer itemID2: item2List) {
- RealVector vector1 = itemVectors.get(itemID1);
- RealVector vector2 = itemVectors.get(itemID2);
- double score = vector1.cosine(vector2);
- if (score > params.threshold) {
- Integer index2 = itemIndexMap.get(itemID2);
- setTopItemSimilarity(topItemSimilarity, itemID1, index2, score, numSimilarItems,
- comparator);
- setTopItemSimilarity(topItemSimilarity, itemID2, index1, score, numSimilarItems,
- comparator);
- }
- }
- }
-
- for (Map.Entry<Integer, Queue<IndexAndScore>> entry : topItemSimilarity.entrySet()) {
- Iterator<IndexAndScore> it = entry.getValue().iterator();
- RealVector vector = new OpenMapRealVector(numOfItems);
- while (it.hasNext()) {
- IndexAndScore d = it.next();
- vector.setEntry(d.index, d.score);
- }
- itemSimilarity.put(entry.getKey(), vector);
- }
-
- return new CollaborativeFilteringModel(itemSimilarity, userHistory);
- }
-
- private class IndexAndScore {
- int index;
- double score;
- public IndexAndScore(int index, double score) {
- this.index = index;
- this.score = score;
- }
- }
-
- private class IndexAndScoreComparator implements Comparator<IndexAndScore> {
- @Override
- public int compare(IndexAndScore o1, IndexAndScore o2) {
- int r = 0;
- if (o1.score < o2.score)
- r = -1;
- else if (o1.score > o2.score)
- r = 1;
- return r;
- }
- }
-
- private void setTopItemSimilarity(Map<Integer, Queue<IndexAndScore>> topItemSimilarity,
- Integer itemID1, Integer index2, double score, int capacity,
- Comparator<IndexAndScore> comparator) {
- Queue<IndexAndScore> queue = topItemSimilarity.get(itemID1);
- if (queue == null) {
- queue = new PriorityQueue<IndexAndScore>(capacity, comparator);
- topItemSimilarity.put(itemID1, queue);
- }
- IndexAndScore entry = new IndexAndScore(index2, score);
- if (queue.size() < capacity)
- queue.add(entry);
- else if (comparator.compare(queue.peek(), entry) < 0) {
- queue.poll();
- queue.add(entry);
- }
- }
-
- @Override
- public Float predict(CollaborativeFilteringModel model, Query query) {
- RealVector itemVector = model.itemSimilarity.get(query.iid);
- RealVector userVector = model.userHistory.get(query.uid);
- if (itemVector == null) {
- // cold start item, can't be handled by this algo, return hard code value.
- return Float.NaN;
- } else if (userVector == null) {
- // new user, can't be handled by this algo, return hard code value.
- return Float.NaN;
- } else {
- //logger.info("(" + query.uid + "," + query.iid + ")");
- //logger.info(itemVector.toString());
- //logger.info(userVector.toString());
- double accum = 0.0;
- double accumSim = 0.0;
- for (int i = 0; i < itemVector.getDimension(); i++) {
- double weight = itemVector.getEntry(i);
- double rating = userVector.getEntry(i);
- if ((weight != 0) && (rating != 0)) {
- accum += weight * rating;
- accumSim += Math.abs(weight);
- }
- }
-
- if (accumSim == 0.0) {
- return Float.NaN;
- } else {
- return (float) (accum / accumSim);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithmParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithmParams.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithmParams.java
deleted file mode 100644
index 1a43426..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringAlgorithmParams.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class CollaborativeFilteringAlgorithmParams implements JavaParams {
- public double threshold;
-
- public CollaborativeFilteringAlgorithmParams(double threshold) {
- this.threshold = threshold;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringModel.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringModel.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringModel.java
deleted file mode 100644
index 9836a24..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/CollaborativeFilteringModel.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import java.io.Serializable;
-import java.util.Map;
-import org.apache.commons.math3.linear.RealVector;
-
-public class CollaborativeFilteringModel implements Serializable {
- public Map<Integer, RealVector> itemSimilarity;
- public Map<Integer, RealVector> userHistory;
-
- public CollaborativeFilteringModel(Map<Integer, RealVector> itemSimilarity,
- Map<Integer, RealVector> userHistory) {
- this.itemSimilarity = itemSimilarity;
- this.userHistory = userHistory;
- }
-
- @Override
- public String toString() {
- String s;
- if ((itemSimilarity.size() > 20) || (userHistory.size() > 20)) {
- s = "Model: [itemSimilarity.size=" + itemSimilarity.size() + "]\n"
- +"[userHistory.size=" + userHistory.size() + "]";
- } else {
- s = "Model: [itemSimilarity: " + itemSimilarity.toString() + "]\n"
- +"[userHistory: " + userHistory.toString() + "]";
- }
- return s;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSource.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSource.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSource.java
deleted file mode 100644
index 3944e7e..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSource.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import java.util.Arrays;
-import org.apache.predictionio.controller.java.LJavaDataSource;
-import org.apache.predictionio.controller.java.EmptyParams;
-import scala.Tuple2;
-import scala.Tuple3;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.lang.Iterable;
-import java.util.List;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Scanner;
-import java.util.HashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-import java.io.FileReader;
-import java.io.BufferedReader;
-
-
-public class DataSource extends LJavaDataSource<
- DataSourceParams, EmptyParams, TrainingData, Query, Object> {
-
- final static Logger logger = LoggerFactory.getLogger(DataSource.class);
-
- DataSourceParams params;
-
- public static class FakeData {
- // User -1 is a action movie lover. He should have high ratings for Item -2 ("Cold Action
- // Movie"). Notice that Item -2 is new, hence have no rating. Feature-based algorithm should be
- // able to return a high rating using user profile.
- public static final List<String> itemData = Arrays.asList(
- "-1|Action Movie (2046)|01-Jan-2046||http://i.dont.exist/|0|1|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0",
- "-2|Cold Action Movie II(2047)|01-Jan-2047||http://i.dont.exist/|0|1|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0",
- "-3|Documentary (1997)|01-July-1997||http://no.pun.intended/|0|0|0|0|0|0|0|1|0|0|0|0|0|0|0|0|0|0|0");
- public static final List<String> userData = Arrays.asList(
- "-1|30|M|action_lover|94087",
- "-2|30|M|documentary_lover|94087",
- "-3|30|M|cold|94087");
- public static final List<String> ratingData = Arrays.asList(
- "-1,-1,5,881250949",
- "-2,-3,5,881250949",
- "-2,1,5,881250949");
- }
-
- public DataSource(DataSourceParams params) {
- this.params = params;
- }
-
- public List<String[]> readFile(String filepath, String delimiter) {
- return readFile(filepath, delimiter, new ArrayList<String>());
- }
-
- public List<String[]> readFile(String filepath, String delimiter, List<String> fakeData) {
- List<String[]> tokensList = new ArrayList<String[]>();
- try {
- List<String> lines = new ArrayList<String>();
- lines.addAll(fakeData);
-
- BufferedReader in = new BufferedReader(new FileReader(filepath));
-
- while (in.ready()) {
- String s = in.readLine();
- lines.add(s);
- }
- in.close();
-
- for (String line: lines) {
- String[] tokens = line.split(delimiter);
- tokensList.add(tokens);
- }
-
- } catch (FileNotFoundException e) {
- logger.error("Caught FileNotFoundException " + e.getMessage());
- System.exit(1);
- } catch (Exception e) {
- logger.error("Can't parse file. Caught Exception: " + e.getMessage()
- + "Trace: " + Arrays.toString(e.getStackTrace()));
- System.exit(1);
- }
-
- return tokensList;
- }
-
- public List<TrainingData.Rating> getRatings() {
- List<TrainingData.Rating> ratings = new ArrayList<TrainingData.Rating>();
-
- List<String[]> tokensList = readFile(params.dir + "u.data", "[\t,]",
- (params.addFakeData) ? FakeData.ratingData : new ArrayList<String>());
-
- for (String[] tokens: tokensList) {
- TrainingData.Rating rating = new TrainingData.Rating(
- Integer.parseInt(tokens[0]),
- Integer.parseInt(tokens[1]),
- Float.parseFloat(tokens[2]));
- ratings.add(rating);
- }
-
- return ratings;
- }
-
- public List<String> getGenres() {
- List<String> genres = new ArrayList<String>();
- for (String[] tokens: readFile(params.dir + "u.genre", "[\t,]")) {
- if (!tokens[0].equals("")) {
- genres.add(tokens[0]);
- }
- }
- return genres;
- }
-
- public Map<Integer, String[]> getItemInfo() {
- List<String[]> tokensList = readFile(params.dir + "u.item", "[\\|]",
- (params.addFakeData) ? FakeData.itemData : new ArrayList<String>());
-
-
- Map<Integer, String[]> itemInfo = new HashMap <> ();
- for (String[] tokens : tokensList) {
- itemInfo.put(Integer.parseInt(tokens[0]), tokens);
- }
-
- return itemInfo;
- }
-
- public Map<Integer, String[]> getUserInfo() {
- List<String[]> tokensList = readFile(params.dir + "u.user", "[\\|]",
- (params.addFakeData) ? FakeData.userData : new ArrayList<String>());
-
- Map<Integer, String[]> userInfo = new HashMap <> ();
- for (String[] tokens : tokensList) {
- userInfo.put(Integer.parseInt(tokens[0]), tokens);
- }
- return userInfo;
- }
-
- @Override
- public Iterable<Tuple3<EmptyParams, TrainingData, Iterable<Tuple2<Query, Object>>>> read() {
- List<Tuple3<EmptyParams, TrainingData, Iterable<Tuple2<Query, Object>>>> data =
- new ArrayList<Tuple3<EmptyParams, TrainingData, Iterable<Tuple2<Query, Object>>>>();
-
- data.add(new Tuple3<EmptyParams, TrainingData, Iterable<Tuple2<Query, Object>>>(
- new EmptyParams(),
- new TrainingData(getRatings(), getGenres(), getItemInfo(), getUserInfo()),
- new ArrayList<Tuple2<Query, Object>>()
- ));
-
- return data;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSourceParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSourceParams.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSourceParams.java
deleted file mode 100644
index 46ad0ec..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/DataSourceParams.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-public class DataSourceParams implements JavaParams {
- public String dir;
- public boolean addFakeData;
-
- public DataSourceParams(String dir, boolean addFakeData) {
- this.dir = dir;
- this.addFakeData = addFakeData;
- }
-
- public DataSourceParams(String dir) {
- this(dir, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/EngineFactory.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/EngineFactory.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/EngineFactory.java
deleted file mode 100644
index 8861a4c..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/EngineFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-
-public class EngineFactory implements IJavaEngineFactory {
- public JavaEngine<TrainingData, EmptyParams, PreparedData, Query, Float, Object> apply() {
- return new JavaEngineBuilder<
- TrainingData, EmptyParams, PreparedData, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass(Preparator.class)
- .addAlgorithmClass("featurebased", FeatureBasedAlgorithm.class)
- .addAlgorithmClass("collaborative", CollaborativeFilteringAlgorithm.class)
- .servingClass(Serving.class)
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithm.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithm.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithm.java
deleted file mode 100644
index 89f59f2..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithm.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.LJavaAlgorithm;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.commons.math3.linear.ArrayRealVector;
-
-public class FeatureBasedAlgorithm
- extends LJavaAlgorithm<
- FeatureBasedAlgorithmParams, PreparedData, FeatureBasedModel, Query, Float> {
- public final FeatureBasedAlgorithmParams params;
- final static Logger logger = LoggerFactory.getLogger(FeatureBasedAlgorithm.class);
-
- public FeatureBasedAlgorithm(FeatureBasedAlgorithmParams params) {
- this.params = params;
- }
-
- public FeatureBasedModel train(PreparedData data) {
- Map<Integer, RealVector> userFeatures = new HashMap<Integer, RealVector>();
- Map<Integer, Integer> userActions = new HashMap<Integer, Integer>();
-
- for (Integer uid : data.userInfo.keySet()) {
- userFeatures.put(uid, new ArrayRealVector(data.featureCount));
- userActions.put(uid, 0);
- }
-
- for (TrainingData.Rating rating : data.ratings) {
- final int uid = rating.uid;
- final int iid = rating.iid;
- final double rate = rating.rating;
-
- // Skip features outside the range.
- if (!(params.min <= rate && rate <= params.max)) continue;
-
- final double actualRate = (rate - params.drift) * params.scale;
- final RealVector userFeature = userFeatures.get(uid);
- final RealVector itemFeature = data.itemFeatures.get(iid);
- userFeature.combineToSelf(1, actualRate, itemFeature);
-
- userActions.put(uid, userActions.get(uid) + 1);
- }
-
- // Normalize userFeatures by l-inf-norm
- for (Integer uid : userFeatures.keySet()) {
- final RealVector feature = userFeatures.get(uid);
- feature.mapDivideToSelf(feature.getLInfNorm());
- }
-
- // Normalize itemFeatures by weight
- Map<Integer, RealVector> itemFeatures = new HashMap<Integer, RealVector>();
- for (Integer iid : data.itemFeatures.keySet()) {
- final RealVector feature = data.itemFeatures.get(iid);
- final RealVector normalizedFeature = feature.mapDivide(feature.getL1Norm());
- itemFeatures.put(iid, normalizedFeature);
- }
-
- return new FeatureBasedModel(userFeatures, userActions, itemFeatures);
- }
-
- public Float predict(FeatureBasedModel model, Query query) {
- final int uid = query.uid;
- final int iid = query.iid;
-
- if (!model.userFeatures.containsKey(uid)) {
- return Float.NaN;
- }
-
- if (!model.itemFeatures.containsKey(iid)) {
- return Float.NaN;
- }
-
- final RealVector userFeature = model.userFeatures.get(uid);
- final RealVector itemFeature = model.itemFeatures.get(iid);
-
- return new Float(userFeature.dotProduct(itemFeature));
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithmParams.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithmParams.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithmParams.java
deleted file mode 100644
index ae03fad..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedAlgorithmParams.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.JavaParams;
-
-// actual score = (rating - drift) / scale if min <= rating <= max
-// if rating is outside [min, max], that scoring will not be used.
-public class FeatureBasedAlgorithmParams implements JavaParams {
- public final double min;
- public final double max;
- public final double drift;
- public final double scale;
-
- public FeatureBasedAlgorithmParams(double min, double max, double drift, double scale) {
- this.min = min;
- this.max = max;
- this.drift = drift;
- this.scale = scale;
- }
-
- public FeatureBasedAlgorithmParams(double min, double max) {
- this(min, max, 0.0, 1.0);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedModel.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedModel.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedModel.java
deleted file mode 100644
index 5b50c5a..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/FeatureBasedModel.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import java.io.Serializable;
-import java.util.Map;
-import org.apache.commons.math3.linear.RealVector;
-
-public class FeatureBasedModel implements Serializable {
- // Feature value is always between -1 and 1.
- public final Map<Integer, RealVector> userFeatures;
- public final Map<Integer, Integer> userActions;
- public final Map<Integer, RealVector> itemFeatures;
-
- public FeatureBasedModel(
- Map<Integer, RealVector> userFeatures,
- Map<Integer, Integer> userActions,
- Map<Integer, RealVector> itemFeatures) {
- this.userFeatures = userFeatures;
- this.userActions = userActions;
- this.itemFeatures = itemFeatures;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Preparator.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Preparator.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Preparator.java
deleted file mode 100644
index 3902371..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Preparator.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.LJavaPreparator;
-import org.apache.predictionio.controller.java.EmptyParams;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.HashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.math3.linear.ArrayRealVector;
-import org.apache.commons.math3.linear.RealVector;
-
-public class Preparator extends LJavaPreparator<EmptyParams, TrainingData, PreparedData> {
-
- final static Logger logger = LoggerFactory.getLogger(Preparator.class);
- final int indexOffset = 5;
-
- public Preparator() {}
-
- public PreparedData prepare(TrainingData trainingData) {
- Map<Integer, RealVector> itemFeatures = new HashMap<Integer, RealVector>();
-
- int featureSize = trainingData.genres.size();
-
- for (Integer iid: trainingData.itemInfo.keySet()) {
- String[] info = trainingData.itemInfo.get(iid);
-
- RealVector features = new ArrayRealVector(featureSize);
- for (int i = 0; i < featureSize; i++) {
- features.setEntry(i, Double.parseDouble(info[i + indexOffset]));
- }
- itemFeatures.put(iid, features);
- }
-
- return new PreparedData(trainingData, itemFeatures, featureSize);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/PreparedData.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/PreparedData.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/PreparedData.java
deleted file mode 100644
index b07d86d..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/PreparedData.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import java.util.Map;
-import org.apache.commons.math3.linear.RealVector;
-
-public class PreparedData extends TrainingData {
- public final Map<Integer, RealVector> itemFeatures;
- public final int featureCount;
-
- public PreparedData(TrainingData data, Map<Integer, RealVector> itemFeatures, int featureCount) {
- super(data);
- this.itemFeatures = itemFeatures;
- this.featureCount = featureCount;
- }
-
- @Override
- public String toString() {
- return super.toString() + " itemFeatures.size=" + itemFeatures.size()
- + " featureCount=" + featureCount;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Query.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Query.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Query.java
deleted file mode 100644
index 302de0c..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Query.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import java.io.Serializable;
-
-public class Query implements Serializable {
- public int uid; // user ID
- public int iid; // item ID
-
- public Query(int uid, int iid) {
- this.uid = uid;
- this.iid = iid;
- }
-
- @Override
- public String toString() {
- return "(" + uid + "," + iid + ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4a.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4a.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4a.java
deleted file mode 100644
index a52fe76..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4a.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-import org.apache.predictionio.controller.IdentityPreparator;
-
-public class Runner4a {
-
- private static class HalfBakedEngineFactory implements IJavaEngineFactory {
- public JavaEngine<TrainingData, EmptyParams, TrainingData, Query, Float, Object> apply() {
- return new JavaEngineBuilder<
- TrainingData, EmptyParams, TrainingData, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .build();
- }
- }
-
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file directory as argument");
- System.exit(1);
- }
-
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(args[0], true))
- .build();
-
- JavaWorkflow.runEngine(
- (new HalfBakedEngineFactory()).apply(),
- engineParams,
- new WorkflowParamsBuilder().batch("Recommendation.tutorial4.Runner4a").verbose(3).build()
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4b.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4b.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4b.java
deleted file mode 100644
index 6964946..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4b.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-import org.apache.predictionio.controller.IdentityPreparator;
-
-public class Runner4b {
-
- private static class HalfBakedEngineFactory implements IJavaEngineFactory {
- public JavaEngine<TrainingData, EmptyParams, PreparedData, Query, Float, Object> apply() {
- return new JavaEngineBuilder<
- TrainingData, EmptyParams, PreparedData, Query, Float, Object> ()
- .dataSourceClass(DataSource.class)
- .preparatorClass(Preparator.class)
- .build();
- }
- }
-
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file directory as argument");
- System.exit(1);
- }
-
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(args[0], true))
- .build();
-
- JavaWorkflow.runEngine(
- (new HalfBakedEngineFactory()).apply(),
- engineParams,
- new WorkflowParamsBuilder().batch("Recommendation.tutorial4.Runner4b").verbose(3).build()
- );
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dec9f84c/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4c.java
----------------------------------------------------------------------
diff --git a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4c.java b/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4c.java
deleted file mode 100644
index 681bd5f..0000000
--- a/examples/experimental/java-local-tutorial/src/main/java/recommendations/tutorial4/Runner4c.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.examples.java.recommendations.tutorial4;
-
-import org.apache.predictionio.controller.java.EmptyParams;
-import org.apache.predictionio.controller.java.IJavaEngineFactory;
-import org.apache.predictionio.controller.java.JavaEngine;
-import org.apache.predictionio.controller.java.JavaEngineBuilder;
-import org.apache.predictionio.controller.java.JavaEngineParams;
-import org.apache.predictionio.controller.java.JavaEngineParamsBuilder;
-import org.apache.predictionio.controller.java.LJavaFirstServing;
-import org.apache.predictionio.controller.java.JavaWorkflow;
-import org.apache.predictionio.controller.java.WorkflowParamsBuilder;
-
-import java.util.HashMap;
-
-import org.apache.predictionio.controller.IdentityPreparator;
-
-public class Runner4c {
- public static void main(String[] args) {
- if (args.length == 0) {
- System.out.println("Error: Please specify the file directory as argument");
- System.exit(1);
- }
-
- JavaEngineParams engineParams = new JavaEngineParamsBuilder()
- .dataSourceParams(new DataSourceParams(args[0], true))
- // 1 -> -1., 2 -> -.5, 3 -> 0., 4 -> .5, 5 -> 1.
- .addAlgorithmParams("featurebased", new FeatureBasedAlgorithmParams(1.0, 5.0, 3.0, 0.5))
- .build();
-
- JavaWorkflow.runEngine(
- (new SingleEngineFactory()).apply(),
- engineParams,
- new WorkflowParamsBuilder().batch("Recommendation.tutorial4.Runner4c").verbose(3).build()
- );
- }
-}
-
-