You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2016/11/01 17:56:19 UTC
[1/3] incubator-predictionio git commit: [PIO-28] Console refactor
Repository: incubator-predictionio
Updated Branches:
refs/heads/develop f7c30d37b -> e4a3c0c9f
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
----------------------------------------------------------------------
diff --git a/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt b/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
index 48ea23e..e660116 100644
--- a/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
+++ b/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
@@ -1,15 +1,3 @@
Usage: pio upgrade <from version> <to version> <old App ID> <new app ID>
-Migrate the event data from old App ID to new App ID which can be used with <to version> of PredictionIO.
-
- <from version>
- The version upgraded from.
-
- <to version>
- The version upgraded to.
-
- <old App ID>
- Old App ID.
-
- <new app ID>
- Mew App ID.
+No longer supported!
[2/3] incubator-predictionio git commit: [PIO-28] Console refactor
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala
deleted file mode 100644
index 53b1b40..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala
+++ /dev/null
@@ -1,540 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.data.storage
-
-import grizzled.slf4j.Logging
-
-case class AppArgs(
- id: Option[Int] = None,
- name: String = "",
- channel: String = "",
- dataDeleteChannel: Option[String] = None,
- all: Boolean = false,
- force: Boolean = false,
- description: Option[String] = None)
-
-object App extends Logging {
- def create(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps()
- // get the client in the beginning so error exit right away if can't access client
- val events = storage.Storage.getLEvents()
- apps.getByName(ca.app.name) map { app =>
- error(s"App ${ca.app.name} already exists. Aborting.")
- 1
- } getOrElse {
- ca.app.id.map { id =>
- apps.get(id) map { app =>
- error(
- s"App ID ${id} already exists and maps to the app '${app.name}'. " +
- "Aborting.")
- return 1
- }
- }
- val appid = apps.insert(storage.App(
- id = ca.app.id.getOrElse(0),
- name = ca.app.name,
- description = ca.app.description))
- appid map { id =>
- val dbInit = events.init(id)
- val r = if (dbInit) {
- info(s"Initialized Event Store for this app ID: ${id}.")
- val accessKeys = storage.Storage.getMetaDataAccessKeys
- val accessKey = accessKeys.insert(storage.AccessKey(
- key = ca.accessKey.accessKey,
- appid = id,
- events = Seq()))
- accessKey map { k =>
- info("Created new app:")
- info(s" Name: ${ca.app.name}")
- info(s" ID: ${id}")
- info(s"Access Key: ${k}")
- 0
- } getOrElse {
- error(s"Unable to create new access key.")
- 1
- }
- } else {
- error(s"Unable to initialize Event Store for this app ID: ${id}.")
- // revert back the meta data change
- try {
- apps.delete(id)
- 0
- } catch {
- case e: Exception =>
- error(s"Failed to revert back the App meta-data change.", e)
- error(s"The app ${ca.app.name} CANNOT be used!")
- error(s"Please run 'pio app delete ${ca.app.name}' " +
- "to delete this app!")
- 1
- }
- }
- events.close()
- r
- } getOrElse {
- error(s"Unable to create new app.")
- 1
- }
- }
- }
-
- def list(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps.getAll().sortBy(_.name)
- val accessKeys = storage.Storage.getMetaDataAccessKeys
- val title = "Name"
- val ak = "Access Key"
- info(f"$title%20s | ID | $ak%64s | Allowed Event(s)")
- apps foreach { app =>
- val keys = accessKeys.getByAppid(app.id)
- keys foreach { k =>
- val events =
- if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
- info(f"${app.name}%20s | ${app.id}%4d | ${k.key}%64s | $events%s")
- }
- }
- info(s"Finished listing ${apps.size} app(s).")
- 0
- }
-
- def show(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps
- val accessKeys = storage.Storage.getMetaDataAccessKeys
- val channels = storage.Storage.getMetaDataChannels
- apps.getByName(ca.app.name) map { app =>
- info(s" App Name: ${app.name}")
- info(s" App ID: ${app.id}")
- info(s" Description: ${app.description.getOrElse("")}")
- val keys = accessKeys.getByAppid(app.id)
-
- var firstKey = true
- keys foreach { k =>
- val events =
- if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
- if (firstKey) {
- info(f" Access Key: ${k.key}%s | ${events}%s")
- firstKey = false
- } else {
- info(f" ${k.key}%s | ${events}%s")
- }
- }
-
- val chans = channels.getByAppid(app.id)
- var firstChan = true
- val titleName = "Channel Name"
- val titleID = "Channel ID"
- chans.foreach { ch =>
- if (firstChan) {
- info(f" Channels: ${titleName}%16s | ${titleID}%10s ")
- firstChan = false
- }
- info(f" ${ch.name}%16s | ${ch.id}%10s")
- }
- 0
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- 1
- }
- }
-
- def delete(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps
- val accesskeys = storage.Storage.getMetaDataAccessKeys
- val channels = storage.Storage.getMetaDataChannels
- val events = storage.Storage.getLEvents()
- val status = apps.getByName(ca.app.name) map { app =>
- info(s"The following app (including all channels) will be deleted. Are you sure?")
- info(s" App Name: ${app.name}")
- info(s" App ID: ${app.id}")
- info(s" Description: ${app.description.getOrElse("")}")
- val chans = channels.getByAppid(app.id)
- var firstChan = true
- val titleName = "Channel Name"
- val titleID = "Channel ID"
- chans.foreach { ch =>
- if (firstChan) {
- info(f" Channels: ${titleName}%16s | ${titleID}%10s ")
- firstChan = false
- }
- info(f" ${ch.name}%16s | ${ch.id}%10s")
- }
-
- val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
- choice match {
- case "YES" => {
- // delete channels
- val delChannelStatus: Seq[Int] = chans.map { ch =>
- if (events.remove(app.id, Some(ch.id))) {
- info(s"Removed Event Store of the channel ID: ${ch.id}")
- try {
- channels.delete(ch.id)
- info(s"Deleted channel ${ch.name}")
- 0
- } catch {
- case e: Exception =>
- error(s"Error deleting channel ${ch.name}.", e)
- 1
- }
- } else {
- error(s"Error removing Event Store of the channel ID: ${ch.id}.")
- return 1
- }
- }
-
- if (delChannelStatus.exists(_ != 0)) {
- error("Error occurred while deleting channels. Aborting.")
- return 1
- }
-
- try {
- events.remove(app.id)
- info(s"Removed Event Store for this app ID: ${app.id}")
- } catch {
- case e: Exception =>
- error(s"Error removing Event Store for this app. Aborting.", e)
- return 1
- }
-
- accesskeys.getByAppid(app.id) foreach { key =>
- try {
- accesskeys.delete(key.key)
- info(s"Removed access key ${key.key}")
- } catch {
- case e: Exception =>
- error(s"Error removing access key ${key.key}. Aborting.", e)
- return 1
- }
- }
-
- try {
- apps.delete(app.id)
- info(s"Deleted app ${app.name}.")
- } catch {
- case e: Exception =>
- error(s"Error deleting app ${app.name}. Aborting.", e)
- return 1
- }
-
- info("Done.")
- 0
- }
- case _ =>
- info("Aborted.")
- 0
- }
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- 1
- }
- events.close()
- status
- }
-
- def dataDelete(ca: ConsoleArgs): Int = {
- if (ca.app.all) {
- dataDeleteAll(ca)
- } else {
- dataDeleteOne(ca)
- }
- }
-
- def dataDeleteOne(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps
- val channels = storage.Storage.getMetaDataChannels
- apps.getByName(ca.app.name) map { app =>
-
- val channelId = ca.app.dataDeleteChannel.map { ch =>
- val channelMap = channels.getByAppid(app.id).map(c => (c.name, c.id)).toMap
- if (!channelMap.contains(ch)) {
- error(s"Unable to delete data for channel.")
- error(s"Channel ${ch} doesn't exist.")
- return 1
- }
-
- channelMap(ch)
- }
-
- if (channelId.isDefined) {
- info(s"Data of the following channel will be deleted. Are you sure?")
- info(s"Channel Name: ${ca.app.dataDeleteChannel.get}")
- info(s" Channel ID: ${channelId.get}")
- info(s" App Name: ${app.name}")
- info(s" App ID: ${app.id}")
- info(s" Description: ${app.description}")
- } else {
- info(s"Data of the following app (default channel only) will be deleted. Are you sure?")
- info(s" App Name: ${app.name}")
- info(s" App ID: ${app.id}")
- info(s" Description: ${app.description}")
- }
-
- val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
-
- choice match {
- case "YES" => {
- val events = storage.Storage.getLEvents()
- // remove table
- val r1 = if (events.remove(app.id, channelId)) {
- if (channelId.isDefined) {
- info(s"Removed Event Store for this channel ID: ${channelId.get}")
- } else {
- info(s"Removed Event Store for this app ID: ${app.id}")
- }
- 0
- } else {
- if (channelId.isDefined) {
- error(s"Error removing Event Store for this channel.")
- } else {
- error(s"Error removing Event Store for this app.")
- }
- 1
- }
- // re-create table
- val dbInit = events.init(app.id, channelId)
- val r2 = if (dbInit) {
- if (channelId.isDefined) {
- info(s"Initialized Event Store for this channel ID: ${channelId.get}.")
- } else {
- info(s"Initialized Event Store for this app ID: ${app.id}.")
- }
- 0
- } else {
- if (channelId.isDefined) {
- error(s"Unable to initialize Event Store for this channel ID:" +
- s" ${channelId.get}.")
- } else {
- error(s"Unable to initialize Event Store for this appId:" +
- s" ${app.id}.")
- }
- 1
- }
- events.close()
- info("Done.")
- r1 + r2
- }
- case _ =>
- info("Aborted.")
- 0
- }
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- 1
- }
- }
-
- def dataDeleteAll(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps
- val channels = storage.Storage.getMetaDataChannels
- val events = storage.Storage.getLEvents()
- val status = apps.getByName(ca.app.name) map { app =>
- info(s"All data of the app (including default and all channels) will be deleted." +
- " Are you sure?")
- info(s" App Name: ${app.name}")
- info(s" App ID: ${app.id}")
- info(s" Description: ${app.description}")
- val chans = channels.getByAppid(app.id)
- var firstChan = true
- val titleName = "Channel Name"
- val titleID = "Channel ID"
- chans.foreach { ch =>
- if (firstChan) {
- info(f" Channels: ${titleName}%16s | ${titleID}%10s ")
- firstChan = false
- }
- info(f" ${ch.name}%16s | ${ch.id}%10s")
- }
-
- val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
- choice match {
- case "YES" => {
- // delete channels
- val delChannelStatus: Seq[Int] = chans.map { ch =>
- val r1 = if (events.remove(app.id, Some(ch.id))) {
- info(s"Removed Event Store of the channel ID: ${ch.id}")
- 0
- } else {
- error(s"Error removing Event Store of the channel ID: ${ch.id}.")
- 1
- }
- // re-create table
- val dbInit = events.init(app.id, Some(ch.id))
- val r2 = if (dbInit) {
- info(s"Initialized Event Store of the channel ID: ${ch.id}")
- 0
- } else {
- error(s"Unable to initialize Event Store of the channel ID: ${ch.id}.")
- 1
- }
- r1 + r2
- }
-
- if (delChannelStatus.filter(_ != 0).isEmpty) {
- val r1 = if (events.remove(app.id)) {
- info(s"Removed Event Store for this app ID: ${app.id}")
- 0
- } else {
- error(s"Error removing Event Store for this app.")
- 1
- }
-
- val dbInit = events.init(app.id)
- val r2 = if (dbInit) {
- info(s"Initialized Event Store for this app ID: ${app.id}.")
- 0
- } else {
- error(s"Unable to initialize Event Store for this appId: ${app.id}.")
- 1
- }
- info("Done.")
- r1 + r2
- } else 1
- }
- case _ =>
- info("Aborted.")
- 0
- }
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- 1
- }
- events.close()
- status
- }
-
- def channelNew(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps
- val channels = storage.Storage.getMetaDataChannels
- val events = storage.Storage.getLEvents()
- val newChannel = ca.app.channel
- val status = apps.getByName(ca.app.name) map { app =>
- val channelMap = channels.getByAppid(app.id).map(c => (c.name, c.id)).toMap
- if (channelMap.contains(newChannel)) {
- error(s"Unable to create new channel.")
- error(s"Channel ${newChannel} already exists.")
- 1
- } else if (!storage.Channel.isValidName(newChannel)) {
- error(s"Unable to create new channel.")
- error(s"The channel name ${newChannel} is invalid.")
- error(s"${storage.Channel.nameConstraint}")
- 1
- } else {
-
- val channelId = channels.insert(storage.Channel(
- id = 0, // new id will be assigned
- appid = app.id,
- name = newChannel
- ))
- channelId.map { chanId =>
- info(s"Updated Channel meta-data.")
- // initialize storage
- val dbInit = events.init(app.id, Some(chanId))
- if (dbInit) {
- info(s"Initialized Event Store for the channel: ${newChannel}.")
- info(s"Created new channel:")
- info(s" Channel Name: ${newChannel}")
- info(s" Channel ID: ${chanId}")
- info(s" App ID: ${app.id}")
- 0
- } else {
- error(s"Unable to create new channel.")
- error(s"Failed to initalize Event Store.")
- // reverted back the meta data
- try {
- channels.delete(chanId)
- 0
- } catch {
- case e: Exception =>
- error(s"Failed to revert back the Channel meta-data change.", e)
- error(s"The channel ${newChannel} CANNOT be used!")
- error(s"Please run 'pio app channel-delete ${app.name} ${newChannel}' " +
- "to delete this channel!")
- 1
- }
- }
- }.getOrElse {
- error(s"Unable to create new channel.")
- error(s"Failed to update Channel meta-data.")
- 1
- }
- }
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- 1
- }
- events.close()
- status
- }
-
- def channelDelete(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps
- val channels = storage.Storage.getMetaDataChannels
- val events = storage.Storage.getLEvents()
- val deleteChannel = ca.app.channel
- val status = apps.getByName(ca.app.name) map { app =>
- val channelMap = channels.getByAppid(app.id).map(c => (c.name, c.id)).toMap
- if (!channelMap.contains(deleteChannel)) {
- error(s"Unable to delete channel.")
- error(s"Channel ${deleteChannel} doesn't exist.")
- 1
- } else {
- info(s"The following channel will be deleted. Are you sure?")
- info(s" Channel Name: ${deleteChannel}")
- info(s" Channel ID: ${channelMap(deleteChannel)}")
- info(s" App Name: ${app.name}")
- info(s" App ID: ${app.id}")
- val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
- choice match {
- case "YES" => {
- // NOTE: remove storage first before remove meta data (in case remove storage failed)
- val dbRemoved = events.remove(app.id, Some(channelMap(deleteChannel)))
- if (dbRemoved) {
- info(s"Removed Event Store for this channel: ${deleteChannel}")
- try {
- channels.delete(channelMap(deleteChannel))
- info(s"Deleted channel: ${deleteChannel}.")
- 0
- } catch {
- case e: Exception =>
- error(s"Unable to delete channel.", e)
- error(s"Failed to update Channel meta-data.")
- error(s"The channel ${deleteChannel} CANNOT be used!")
- error(s"Please run 'pio app channel-delete ${app.name} ${deleteChannel}' " +
- "to delete this channel again!")
- 1
- }
- } else {
- error(s"Unable to delete channel.")
- error(s"Error removing Event Store for this channel.")
- 1
- }
- }
- case _ =>
- info("Aborted.")
- 0
- }
- }
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- 1
- }
- events.close()
- status
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
index 5646467..195740b 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
@@ -29,10 +29,16 @@ import org.apache.predictionio.data.api.EventServerConfig
import org.apache.predictionio.data.storage
import org.apache.predictionio.data.storage.EngineManifest
import org.apache.predictionio.data.storage.EngineManifestSerializer
-import org.apache.predictionio.data.storage.hbase.upgrade.Upgrade_0_8_3
import org.apache.predictionio.tools.RegisterEngine
import org.apache.predictionio.tools.RunServer
import org.apache.predictionio.tools.RunWorkflow
+import org.apache.predictionio.tools.Common
+import org.apache.predictionio.tools.commands.{
+ DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
+ BuildArgs, EngineArgs}
+import org.apache.predictionio.tools.{
+ EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
+import org.apache.predictionio.tools.EventServerArgs
import org.apache.predictionio.tools.admin.AdminServer
import org.apache.predictionio.tools.admin.AdminServerConfig
import org.apache.predictionio.tools.dashboard.Dashboard
@@ -40,6 +46,7 @@ import org.apache.predictionio.tools.dashboard.DashboardConfig
import org.apache.predictionio.workflow.JsonExtractorOption
import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.tools.commands
import org.apache.commons.io.FileUtils
import org.json4s._
import org.json4s.native.JsonMethods._
@@ -54,82 +61,39 @@ import scala.util.Random
import scalaj.http.Http
case class ConsoleArgs(
- common: CommonArgs = CommonArgs(),
build: BuildArgs = BuildArgs(),
app: AppArgs = AppArgs(),
+ spark: SparkArgs = SparkArgs(),
+ engine: EngineArgs = EngineArgs(),
+ workflow: WorkflowArgs = WorkflowArgs(),
accessKey: AccessKeyArgs = AccessKeyArgs(),
deploy: DeployArgs = DeployArgs(),
eventServer: EventServerArgs = EventServerArgs(),
adminServer: AdminServerArgs = AdminServerArgs(),
dashboard: DashboardArgs = DashboardArgs(),
- upgrade: UpgradeArgs = UpgradeArgs(),
- template: TemplateArgs = TemplateArgs(),
export: ExportArgs = ExportArgs(),
imprt: ImportArgs = ImportArgs(),
commands: Seq[String] = Seq(),
- metricsClass: Option[String] = None,
metricsParamsJsonPath: Option[String] = None,
paramsPath: String = "params",
engineInstanceId: Option[String] = None,
- mainClass: Option[String] = None)
-
-case class CommonArgs(
- batch: String = "",
- sparkPassThrough: Seq[String] = Seq(),
+ mainClass: Option[String] = None,
driverPassThrough: Seq[String] = Seq(),
pioHome: Option[String] = None,
- sparkHome: Option[String] = None,
- engineId: Option[String] = None,
- engineVersion: Option[String] = None,
- engineFactory: Option[String] = None,
- engineParamsKey: Option[String] = None,
- evaluation: Option[String] = None,
- engineParamsGenerator: Option[String] = None,
- variantJson: File = new File("engine.json"),
- manifestJson: File = new File("manifest.json"),
- stopAfterRead: Boolean = false,
- stopAfterPrepare: Boolean = false,
- skipSanityCheck: Boolean = false,
- verbose: Boolean = false,
- verbosity: Int = 0,
- sparkKryo: Boolean = false,
- scratchUri: Option[URI] = None,
- jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
-
-case class BuildArgs(
- sbt: Option[File] = None,
- sbtExtra: Option[String] = None,
- sbtAssemblyPackageDependency: Boolean = true,
- sbtClean: Boolean = false,
- uberJar: Boolean = false,
- forceGeneratePIOSbt: Boolean = false)
-
-case class DeployArgs(
- ip: String = "0.0.0.0",
- port: Int = 8000,
- logUrl: Option[String] = None,
- logPrefix: Option[String] = None)
-
-case class EventServerArgs(
- enabled: Boolean = false,
- ip: String = "0.0.0.0",
- port: Int = 7070,
- stats: Boolean = false)
-
-case class AdminServerArgs(
-ip: String = "127.0.0.1",
-port: Int = 7071)
+ verbose: Boolean = false)
-case class DashboardArgs(
- ip: String = "127.0.0.1",
- port: Int = 9000)
+case class AppArgs(
+ id: Option[Int] = None,
+ name: String = "",
+ channel: String = "",
+ dataDeleteChannel: Option[String] = None,
+ all: Boolean = false,
+ force: Boolean = false,
+ description: Option[String] = None)
-case class UpgradeArgs(
- from: String = "0.0.0",
- to: String = "0.0.0",
- oldAppId: Int = 0,
- newAppId: Int = 0
-)
+case class AccessKeyArgs(
+ accessKey: String = "",
+ events: Seq[String] = Seq())
object Console extends Logging {
def main(args: Array[String]): Unit = {
@@ -145,27 +109,27 @@ object Console extends Logging {
"for each command for more information.\n\n" +
"The following options are common to all commands:\n")
opt[String]("pio-home") action { (x, c) =>
- c.copy(common = c.common.copy(pioHome = Some(x)))
+ c.copy(pioHome = Some(x))
} text("Root directory of a PredictionIO installation.\n" +
" Specify this if automatic discovery fail.")
opt[String]("spark-home") action { (x, c) =>
- c.copy(common = c.common.copy(sparkHome = Some(x)))
+ c.copy(spark = c.spark.copy(sparkHome = Some(x)))
} text("Root directory of an Apache Spark installation.\n" +
" If not specified, will try to use the SPARK_HOME\n" +
" environmental variable. If this fails as well, default to\n" +
" current directory.")
opt[String]("engine-id") abbr("ei") action { (x, c) =>
- c.copy(common = c.common.copy(engineId = Some(x)))
+ c.copy(engine = c.engine.copy(engineId = Some(x)))
} text("Specify an engine ID. Usually used by distributed deployment.")
opt[String]("engine-version") abbr("ev") action { (x, c) =>
- c.copy(common = c.common.copy(engineVersion = Some(x)))
+ c.copy(engine = c.engine.copy(engineVersion = Some(x)))
} text("Specify an engine version. Usually used by distributed " +
"deployment.")
opt[File]("variant") abbr("v") action { (x, c) =>
- c.copy(common = c.common.copy(variantJson = x))
+ c.copy(workflow = c.workflow.copy(variantJson = x))
}
opt[File]("manifest") abbr("m") action { (x, c) =>
- c.copy(common = c.common.copy(manifestJson = x))
+ c.copy(engine = c.engine.copy(manifestJson = x))
}
opt[File]("sbt") action { (x, c) =>
c.copy(build = c.build.copy(sbt = Some(x)))
@@ -177,13 +141,13 @@ object Console extends Logging {
}
} text("Path to sbt. Default: sbt")
opt[Unit]("verbose") action { (x, c) =>
- c.copy(common = c.common.copy(verbose = true))
+ c.copy(verbose = true)
}
opt[Unit]("spark-kryo") abbr("sk") action { (x, c) =>
- c.copy(common = c.common.copy(sparkKryo = true))
+ c.copy(spark = c.spark.copy(sparkKryo = true))
}
opt[String]("scratch-uri") action { (x, c) =>
- c.copy(common = c.common.copy(scratchUri = Some(new URI(x))))
+ c.copy(spark = c.spark.copy(scratchUri = Some(new URI(x))))
}
note("")
cmd("version").
@@ -237,7 +201,7 @@ object Console extends Logging {
c.copy(commands = c.commands :+ "train")
} children(
opt[String]("batch") action { (x, c) =>
- c.copy(common = c.common.copy(batch = x))
+ c.copy(workflow = c.workflow.copy(batch = x))
} text("Batch label of the run."),
opt[String]("params-path") action { (x, c) =>
c.copy(paramsPath = x)
@@ -247,28 +211,28 @@ object Console extends Logging {
} text("Metrics parameters JSON file. Will try to use\n" +
" metrics.json in the base path."),
opt[Unit]("skip-sanity-check") abbr("ssc") action { (x, c) =>
- c.copy(common = c.common.copy(skipSanityCheck = true))
+ c.copy(workflow = c.workflow.copy(skipSanityCheck = true))
},
opt[Unit]("stop-after-read") abbr("sar") action { (x, c) =>
- c.copy(common = c.common.copy(stopAfterRead = true))
+ c.copy(workflow = c.workflow.copy(stopAfterRead = true))
},
opt[Unit]("stop-after-prepare") abbr("sap") action { (x, c) =>
- c.copy(common = c.common.copy(stopAfterPrepare = true))
+ c.copy(workflow = c.workflow.copy(stopAfterPrepare = true))
},
opt[Unit]("uber-jar") action { (x, c) =>
c.copy(build = c.build.copy(uberJar = true))
},
opt[Int]("verbosity") action { (x, c) =>
- c.copy(common = c.common.copy(verbosity = x))
+ c.copy(workflow = c.workflow.copy(verbosity = x))
},
opt[String]("engine-factory") action { (x, c) =>
- c.copy(common = c.common.copy(engineFactory = Some(x)))
+ c.copy(workflow = c.workflow.copy(engineFactory = Some(x)))
},
opt[String]("engine-params-key") action { (x, c) =>
- c.copy(common = c.common.copy(engineParamsKey = Some(x)))
+ c.copy(workflow = c.workflow.copy(engineParamsKey = Some(x)))
},
opt[String]("json-extractor") action { (x, c) =>
- c.copy(common = c.common.copy(jsonExtractor = JsonExtractorOption.withName(x)))
+ c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
} validate { x =>
if (JsonExtractorOption.values.map(_.toString).contains(x)) {
success
@@ -287,16 +251,16 @@ object Console extends Logging {
c.copy(commands = c.commands :+ "eval")
} children(
arg[String]("<evaluation-class>") action { (x, c) =>
- c.copy(common = c.common.copy(evaluation = Some(x)))
+ c.copy(workflow = c.workflow.copy(evaluation = Some(x)))
},
arg[String]("[<engine-parameters-generator-class>]") optional() action { (x, c) =>
- c.copy(common = c.common.copy(engineParamsGenerator = Some(x)))
+ c.copy(workflow = c.workflow.copy(engineParamsGenerator = Some(x)))
} text("Optional engine parameters generator class, overriding the first argument"),
opt[String]("batch") action { (x, c) =>
- c.copy(common = c.common.copy(batch = x))
+ c.copy(workflow = c.workflow.copy(batch = x))
} text("Batch label of the run."),
opt[String]("json-extractor") action { (x, c) =>
- c.copy(common = c.common.copy(jsonExtractor = JsonExtractorOption.withName(x)))
+ c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
} validate { x =>
if (JsonExtractorOption.values.map(_.toString).contains(x)) {
success
@@ -315,7 +279,7 @@ object Console extends Logging {
c.copy(commands = c.commands :+ "deploy")
} children(
opt[String]("batch") action { (x, c) =>
- c.copy(common = c.common.copy(batch = x))
+ c.copy(workflow = c.workflow.copy(batch = x))
} text("Batch label of the deployment."),
opt[String]("engine-instance-id") action { (x, c) =>
c.copy(engineInstanceId = Some(x))
@@ -354,7 +318,7 @@ object Console extends Logging {
c.copy(deploy = c.deploy.copy(logPrefix = Some(x)))
},
opt[String]("json-extractor") action { (x, c) =>
- c.copy(common = c.common.copy(jsonExtractor = JsonExtractorOption.withName(x)))
+ c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
} validate { x =>
if (JsonExtractorOption.values.map(_.toString).contains(x)) {
success
@@ -449,23 +413,10 @@ object Console extends Logging {
}
note("")
cmd("upgrade").
- text("Upgrade tool").
+ text("No longer supported!").
action { (_, c) =>
c.copy(commands = c.commands :+ "upgrade")
- } children(
- arg[String]("<from version>") action { (x, c) =>
- c.copy(upgrade = c.upgrade.copy(from = x))
- } text("The version upgraded from."),
- arg[String]("<to version>") action { (x, c) =>
- c.copy(upgrade = c.upgrade.copy(to = x))
- } text("The version upgraded to."),
- arg[Int]("<old App ID>") action { (x, c) =>
- c.copy(upgrade = c.upgrade.copy(oldAppId = x))
- } text("Old App ID."),
- arg[Int]("<new App ID>") action { (x, c) =>
- c.copy(upgrade = c.upgrade.copy(newAppId = x))
- } text("New App ID.")
- )
+ }
note("")
cmd("app").
text("Manage apps.\n").
@@ -616,29 +567,12 @@ object Console extends Logging {
c.copy(commands = c.commands :+ "template")
} children(
cmd("get").
+ text("No longer supported! Use git clone to download a template").
action { (_, c) =>
c.copy(commands = c.commands :+ "get")
- } children(
- arg[String]("<template ID>") required() action { (x, c) =>
- c.copy(template = c.template.copy(repository = x))
- },
- arg[String]("<new engine directory>") action { (x, c) =>
- c.copy(template = c.template.copy(directory = x))
- },
- opt[String]("version") action { (x, c) =>
- c.copy(template = c.template.copy(version = Some(x)))
- },
- opt[String]("name") action { (x, c) =>
- c.copy(template = c.template.copy(name = Some(x)))
- },
- opt[String]("package") action { (x, c) =>
- c.copy(template = c.template.copy(packageName = Some(x)))
- },
- opt[String]("email") action { (x, c) =>
- c.copy(template = c.template.copy(email = Some(x)))
- }
- ),
+ },
cmd("list").
+ text("No longer supported! Use git to manage your templates").
action { (_, c) =>
c.copy(commands = c.commands :+ "list")
}
@@ -694,78 +628,95 @@ object Console extends Logging {
}
parser.parse(consoleArgs, ConsoleArgs()) map { pca =>
- val ca = pca.copy(common = pca.common.copy(
- sparkPassThrough = sparkPassThroughArgs,
- driverPassThrough = driverPassThroughArgs))
- WorkflowUtils.modifyLogging(ca.common.verbose)
+ val ca = pca.copy(
+ spark = pca.spark.copy(sparkPassThrough = sparkPassThroughArgs),
+ driverPassThrough = driverPassThroughArgs)
+ WorkflowUtils.modifyLogging(ca.verbose)
val rv: Int = ca.commands match {
case Seq("") =>
System.err.println(help())
1
case Seq("version") =>
- version(ca)
- 0
+ Pio.version()
case Seq("build") =>
- regenerateManifestJson(ca.common.manifestJson)
- build(ca)
+ Pio.build(
+ ca.build, ca.pioHome.get, ca.engine.manifestJson, ca.verbose)
case Seq("unregister") =>
- unregister(ca)
- 0
+ Pio.unregister(ca.engine.manifestJson)
case Seq("train") =>
- regenerateManifestJson(ca.common.manifestJson)
- train(ca)
+ Pio.train(
+ ca.engine, ca.workflow, ca.spark, ca.pioHome.get, ca.verbose)
case Seq("eval") =>
- regenerateManifestJson(ca.common.manifestJson)
- train(ca)
+ Pio.eval(
+ ca.engine, ca.workflow, ca.spark, ca.pioHome.get, ca.verbose)
case Seq("deploy") =>
- deploy(ca)
+ Pio.deploy(
+ ca.engine,
+ ca.engineInstanceId,
+ ServerArgs(
+ ca.deploy,
+ ca.eventServer,
+ ca.workflow.batch,
+ ca.accessKey.accessKey,
+ ca.workflow.variantJson,
+ ca.workflow.jsonExtractor),
+ ca.spark,
+ ca.pioHome.get,
+ ca.verbose)
case Seq("undeploy") =>
- undeploy(ca)
+ Pio.undeploy(ca.deploy)
case Seq("dashboard") =>
- dashboard(ca)
- 0
+ Pio.dashboard(ca.dashboard)
case Seq("eventserver") =>
- eventserver(ca)
- 0
+ Pio.eventserver(ca.eventServer)
case Seq("adminserver") =>
- adminserver(ca)
- 0
+ Pio.adminserver(ca.adminServer)
case Seq("run") =>
- generateManifestJson(ca.common.manifestJson)
- run(ca)
+ Pio.run(
+ ca.mainClass.get,
+ ca.driverPassThrough,
+ ca.engine.manifestJson,
+ ca.build,
+ ca.spark,
+ ca.pioHome.get,
+ ca.verbose)
case Seq("status") =>
- status(ca)
+ Pio.status(ca.pioHome, ca.spark.sparkHome)
case Seq("upgrade") =>
- upgrade(ca)
- 0
+ error("Upgrade is no longer supported")
+ 1
case Seq("app", "new") =>
- App.create(ca)
+ Pio.App.create(
+ ca.app.name, ca.app.id, ca.app.description, ca.accessKey.accessKey)
case Seq("app", "list") =>
- App.list(ca)
+ Pio.App.list()
case Seq("app", "show") =>
- App.show(ca)
+ Pio.App.show(ca.app.name)
case Seq("app", "delete") =>
- App.delete(ca)
+ Pio.App.delete(ca.app.name, ca.app.force)
case Seq("app", "data-delete") =>
- App.dataDelete(ca)
+ Pio.App.dataDelete(
+ ca.app.name, ca.app.dataDeleteChannel, ca.app.all, ca.app.force)
case Seq("app", "channel-new") =>
- App.channelNew(ca)
+ Pio.App.channelNew(ca.app.name, ca.app.channel)
case Seq("app", "channel-delete") =>
- App.channelDelete(ca)
+ Pio.App.channelDelete(ca.app.name, ca.app.channel, ca.app.force)
case Seq("accesskey", "new") =>
- AccessKey.create(ca)
+ Pio.AccessKey.create(
+ ca.app.name, ca.accessKey.accessKey, ca.accessKey.events)
case Seq("accesskey", "list") =>
- AccessKey.list(ca)
+ Pio.AccessKey.list(
+ if (ca.app.name == "") None else Some(ca.app.name))
case Seq("accesskey", "delete") =>
- AccessKey.delete(ca)
- case Seq("template", "get") =>
- Template.get(ca)
- case Seq("template", "list") =>
- Template.list(ca)
+ Pio.AccessKey.delete(ca.accessKey.accessKey)
+ case Seq("template", _) =>
+ error("template commands are no longer supported.")
+ error("Please use git to get and manage your templates.")
+ 1
case Seq("export") =>
- Export.eventsToFile(ca)
+ Pio.export(ca.export, ca.spark, ca.pioHome.get)
case Seq("import") =>
- Import.fileToEvents(ca)
+ Pio.imprt(ca.imprt, ca.spark, ca.pioHome.get)
case _ =>
System.err.println(help(ca.commands))
1
@@ -809,465 +760,4 @@ object Console extends Logging {
"run" -> txt.run().toString,
"eval" -> txt.eval().toString,
"dashboard" -> txt.dashboard().toString)
-
- def version(ca: ConsoleArgs): Unit = println(BuildInfo.version)
-
- def build(ca: ConsoleArgs): Int = {
- Template.verifyTemplateMinVersion(new File("template.json"))
- compile(ca)
- info("Looking for an engine...")
- val jarFiles = jarFilesForScala
- if (jarFiles.isEmpty) {
- error("No engine found. Your build might have failed. Aborting.")
- return 1
- }
- jarFiles foreach { f => info(s"Found ${f.getName}")}
- RegisterEngine.registerEngine(
- ca.common.manifestJson,
- jarFiles,
- false)
- info("Your engine is ready for training.")
- 0
- }
-
- def unregister(ca: ConsoleArgs): Unit = {
- RegisterEngine.unregisterEngine(ca.common.manifestJson)
- }
-
- def train(ca: ConsoleArgs): Int = {
- Template.verifyTemplateMinVersion(new File("template.json"))
- withRegisteredManifest(
- ca.common.manifestJson,
- ca.common.engineId,
- ca.common.engineVersion) { em =>
- RunWorkflow.newRunWorkflow(ca, em)
- }
- }
-
- def deploy(ca: ConsoleArgs): Int = {
- Template.verifyTemplateMinVersion(new File("template.json"))
- withRegisteredManifest(
- ca.common.manifestJson,
- ca.common.engineId,
- ca.common.engineVersion) { em =>
- val variantJson = parse(Source.fromFile(ca.common.variantJson).mkString)
- val variantId = variantJson \ "id" match {
- case JString(s) => s
- case _ =>
- error("Unable to read engine variant ID from " +
- s"${ca.common.variantJson.getCanonicalPath}. Aborting.")
- return 1
- }
- val engineInstances = storage.Storage.getMetaDataEngineInstances
- val engineInstance = ca.engineInstanceId map { eid =>
- engineInstances.get(eid)
- } getOrElse {
- engineInstances.getLatestCompleted(em.id, em.version, variantId)
- }
- engineInstance map { r =>
- RunServer.newRunServer(ca, em, r.id)
- } getOrElse {
- ca.engineInstanceId map { eid =>
- error(
- s"Invalid engine instance ID ${ca.engineInstanceId}. Aborting.")
- } getOrElse {
- error(
- s"No valid engine instance found for engine ${em.id} " +
- s"${em.version}.\nTry running 'train' before 'deploy'. Aborting.")
- }
- 1
- }
- }
- }
-
- def dashboard(ca: ConsoleArgs): Unit = {
- info(s"Creating dashboard at ${ca.dashboard.ip}:${ca.dashboard.port}")
- Dashboard.createDashboard(DashboardConfig(
- ip = ca.dashboard.ip,
- port = ca.dashboard.port))
- }
-
- def eventserver(ca: ConsoleArgs): Unit = {
- info(
- s"Creating Event Server at ${ca.eventServer.ip}:${ca.eventServer.port}")
- EventServer.createEventServer(EventServerConfig(
- ip = ca.eventServer.ip,
- port = ca.eventServer.port,
- stats = ca.eventServer.stats))
- }
-
- def adminserver(ca: ConsoleArgs): Unit = {
- info(
- s"Creating Admin Server at ${ca.adminServer.ip}:${ca.adminServer.port}")
- AdminServer.createAdminServer(AdminServerConfig(
- ip = ca.adminServer.ip,
- port = ca.adminServer.port
- ))
- }
-
- def undeploy(ca: ConsoleArgs): Int = {
- val serverUrl = s"http://${ca.deploy.ip}:${ca.deploy.port}"
- info(
- s"Undeploying any existing engine instance at ${serverUrl}")
- try {
- val code = Http(s"${serverUrl}/stop").asString.code
- code match {
- case 200 => 0
- case 404 =>
- error(s"Another process is using ${serverUrl}. Unable to undeploy.")
- 1
- case _ =>
- error(s"Another process is using ${serverUrl}, or an existing " +
- s"engine server is not responding properly (HTTP ${code}). " +
- "Unable to undeploy.")
- 1
- }
- } catch {
- case e: java.net.ConnectException =>
- warn(s"Nothing at ${serverUrl}")
- 0
- case _: Throwable =>
- error("Another process might be occupying " +
- s"${ca.deploy.ip}:${ca.deploy.port}. Unable to undeploy.")
- 1
- }
- }
-
- def compile(ca: ConsoleArgs): Unit = {
- // only add pioVersion to sbt if project/pio.sbt exists
- if (new File("project", "pio-build.sbt").exists || ca.build.forceGeneratePIOSbt) {
- FileUtils.writeLines(
- new File("pio.sbt"),
- Seq(
- "// Generated automatically by pio build.",
- "// Changes in this file will be overridden.",
- "",
- "pioVersion := \"" + BuildInfo.version + "\""))
- }
- implicit val formats = Utils.json4sDefaultFormats
-
- val sbt = detectSbt(ca)
- info(s"Using command '${sbt}' at the current working directory to build.")
- info("If the path above is incorrect, this process will fail.")
- val asm =
- if (ca.build.sbtAssemblyPackageDependency) {
- " assemblyPackageDependency"
- } else {
- ""
- }
- val clean = if (ca.build.sbtClean) " clean" else ""
- val buildCmd = s"${sbt} ${ca.build.sbtExtra.getOrElse("")}${clean} " +
- (if (ca.build.uberJar) "assembly" else s"package${asm}")
- val core = new File(s"pio-assembly-${BuildInfo.version}.jar")
- if (ca.build.uberJar) {
- info(s"Uber JAR enabled. Putting ${core.getName} in lib.")
- val dst = new File("lib")
- dst.mkdir()
- FileUtils.copyFileToDirectory(
- coreAssembly(ca.common.pioHome.get),
- dst,
- true)
- } else {
- if (new File("engine.json").exists()) {
- info(s"Uber JAR disabled. Making sure lib/${core.getName} is absent.")
- new File("lib", core.getName).delete()
- } else {
- info("Uber JAR disabled, but current working directory does not look " +
- s"like an engine project directory. Please delete lib/${core.getName} manually.")
- }
- }
- info(s"Going to run: ${buildCmd}")
- try {
- val r =
- if (ca.common.verbose) {
- buildCmd.!(ProcessLogger(line => info(line), line => error(line)))
- } else {
- buildCmd.!(ProcessLogger(
- line => outputSbtError(line),
- line => outputSbtError(line)))
- }
- if (r != 0) {
- error(s"Return code of previous step is ${r}. Aborting.")
- sys.exit(1)
- }
- info("Build finished successfully.")
- } catch {
- case e: java.io.IOException =>
- error(s"${e.getMessage}")
- sys.exit(1)
- }
- }
-
- private def outputSbtError(line: String): Unit = {
- """\[.*error.*\]""".r findFirstIn line foreach { _ => error(line) }
- }
-
- def run(ca: ConsoleArgs): Int = {
- compile(ca)
-
- val extraFiles = WorkflowUtils.thirdPartyConfFiles
-
- val jarFiles = jarFilesForScala
- jarFiles foreach { f => info(s"Found JAR: ${f.getName}") }
- val allJarFiles = jarFiles.map(_.getCanonicalPath)
- val cmd = s"${getSparkHome(ca.common.sparkHome)}/bin/spark-submit --jars " +
- s"${allJarFiles.mkString(",")} " +
- (if (extraFiles.size > 0) {
- s"--files ${extraFiles.mkString(",")} "
- } else {
- ""
- }) +
- "--class " +
- s"${ca.mainClass.get} ${ca.common.sparkPassThrough.mkString(" ")} " +
- coreAssembly(ca.common.pioHome.get) + " " +
- ca.common.driverPassThrough.mkString(" ")
- val proc = Process(
- cmd,
- None,
- "SPARK_YARN_USER_ENV" -> sys.env.filter(kv => kv._1.startsWith("PIO_")).
- map(kv => s"${kv._1}=${kv._2}").mkString(","))
- info(s"Submission command: ${cmd}")
- val r = proc.!
- if (r != 0) {
- error(s"Return code of previous step is ${r}. Aborting.")
- return 1
- }
- r
- }
-
- def status(ca: ConsoleArgs): Int = {
- info("Inspecting PredictionIO...")
- ca.common.pioHome map { pioHome =>
- info(s"PredictionIO ${BuildInfo.version} is installed at $pioHome")
- } getOrElse {
- error("Unable to locate PredictionIO installation. Aborting.")
- return 1
- }
- info("Inspecting Apache Spark...")
- val sparkHome = getSparkHome(ca.common.sparkHome)
- if (new File(s"$sparkHome/bin/spark-submit").exists) {
- info(s"Apache Spark is installed at $sparkHome")
- val sparkMinVersion = "1.3.0"
- val sparkReleaseFile = new File(s"$sparkHome/RELEASE")
- if (sparkReleaseFile.exists) {
- val sparkReleaseStrings =
- Source.fromFile(sparkReleaseFile).mkString.split(' ')
- if (sparkReleaseStrings.length < 2) {
- warn(stripMarginAndNewlines(
- s"""|Apache Spark version information cannot be found (RELEASE file
- |is empty). This is a known issue for certain vendors (e.g.
- |Cloudera). Please make sure you are using a version of at least
- |$sparkMinVersion."""))
- } else {
- val sparkReleaseVersion = sparkReleaseStrings(1)
- val parsedMinVersion = Version.apply(sparkMinVersion)
- val parsedCurrentVersion = Version.apply(sparkReleaseVersion)
- if (parsedCurrentVersion >= parsedMinVersion) {
- info(stripMarginAndNewlines(
- s"""|Apache Spark $sparkReleaseVersion detected (meets minimum
- |requirement of $sparkMinVersion)"""))
- } else {
- error(stripMarginAndNewlines(
- s"""|Apache Spark $sparkReleaseVersion detected (does not meet
- |minimum requirement. Aborting."""))
- }
- }
- } else {
- warn(stripMarginAndNewlines(
- s"""|Apache Spark version information cannot be found. If you are
- |using a developmental tree, please make sure you are using a
- |version of at least $sparkMinVersion."""))
- }
- } else {
- error("Unable to locate a proper Apache Spark installation. Aborting.")
- return 1
- }
- info("Inspecting storage backend connections...")
- try {
- storage.Storage.verifyAllDataObjects()
- } catch {
- case e: Throwable =>
- error("Unable to connect to all storage backends successfully. The " +
- "following shows the error message from the storage backend.")
- error(s"${e.getMessage} (${e.getClass.getName})", e)
- error("Dumping configuration of initialized storage backend sources. " +
- "Please make sure they are correct.")
- storage.Storage.config.get("sources") map { src =>
- src foreach { case (s, p) =>
- error(s"Source Name: $s; Type: ${p.getOrElse("type", "(error)")}; " +
- s"Configuration: ${p.getOrElse("config", "(error)")}")
- }
- } getOrElse {
- error("No properly configured storage backend sources.")
- }
- return 1
- }
- info("(sleeping 5 seconds for all messages to show up...)")
- Thread.sleep(5000)
- info("Your system is all ready to go.")
- 0
- }
-
- def upgrade(ca: ConsoleArgs): Unit = {
- (ca.upgrade.from, ca.upgrade.to) match {
- case ("0.8.2", "0.8.3") => {
- Upgrade_0_8_3.runMain(ca.upgrade.oldAppId, ca.upgrade.newAppId)
- }
- case _ =>
- println(s"Upgrade from version ${ca.upgrade.from} to ${ca.upgrade.to}"
- + s" is not supported.")
- }
- }
-
- def coreAssembly(pioHome: String): File = {
- val core = s"pio-assembly-${BuildInfo.version}.jar"
- val coreDir =
- if (new File(pioHome + File.separator + "RELEASE").exists) {
- new File(pioHome + File.separator + "lib")
- } else {
- new File(pioHome + File.separator + "assembly")
- }
- val coreFile = new File(coreDir, core)
- if (coreFile.exists) {
- coreFile
- } else {
- error(s"PredictionIO Core Assembly (${coreFile.getCanonicalPath}) does " +
- "not exist. Aborting.")
- sys.exit(1)
- }
- }
-
- val manifestAutogenTag = "pio-autogen-manifest"
-
- def regenerateManifestJson(json: File): Unit = {
- val cwd = sys.props("user.dir")
- val ha = java.security.MessageDigest.getInstance("SHA-1").
- digest(cwd.getBytes).map("%02x".format(_)).mkString
- if (json.exists) {
- val em = readManifestJson(json)
- if (em.description == Some(manifestAutogenTag) && ha != em.version) {
- warn("This engine project directory contains an auto-generated " +
- "manifest that has been copied/moved from another location. ")
- warn("Regenerating the manifest to reflect the updated location. " +
- "This will dissociate with all previous engine instances.")
- generateManifestJson(json)
- } else {
- info(s"Using existing engine manifest JSON at ${json.getCanonicalPath}")
- }
- } else {
- generateManifestJson(json)
- }
- }
-
- def generateManifestJson(json: File): Unit = {
- val cwd = sys.props("user.dir")
- implicit val formats = Utils.json4sDefaultFormats +
- new EngineManifestSerializer
- val rand = Random.alphanumeric.take(32).mkString
- val ha = java.security.MessageDigest.getInstance("SHA-1").
- digest(cwd.getBytes).map("%02x".format(_)).mkString
- val em = EngineManifest(
- id = rand,
- version = ha,
- name = new File(cwd).getName,
- description = Some(manifestAutogenTag),
- files = Seq(),
- engineFactory = "")
- try {
- FileUtils.writeStringToFile(json, write(em), "ISO-8859-1")
- } catch {
- case e: java.io.IOException =>
- error(s"Cannot generate ${json} automatically (${e.getMessage}). " +
- "Aborting.")
- sys.exit(1)
- }
- }
-
- def readManifestJson(json: File): EngineManifest = {
- implicit val formats = Utils.json4sDefaultFormats +
- new EngineManifestSerializer
- try {
- read[EngineManifest](Source.fromFile(json).mkString)
- } catch {
- case e: java.io.FileNotFoundException =>
- error(s"${json.getCanonicalPath} does not exist. Aborting.")
- sys.exit(1)
- case e: MappingException =>
- error(s"${json.getCanonicalPath} has invalid content: " +
- e.getMessage)
- sys.exit(1)
- }
- }
-
- def withRegisteredManifest(
- json: File,
- engineId: Option[String],
- engineVersion: Option[String])(
- op: EngineManifest => Int): Int = {
- val ej = readManifestJson(json)
- val id = engineId getOrElse ej.id
- val version = engineVersion getOrElse ej.version
- storage.Storage.getMetaDataEngineManifests.get(id, version) map {
- op
- } getOrElse {
- error(s"Engine ${id} ${version} cannot be found in the system.")
- error("Possible reasons:")
- error("- the engine is not yet built by the 'build' command;")
- error("- the meta data store is offline.")
- 1
- }
- }
-
- def jarFilesAt(path: File): Array[File] = recursiveListFiles(path) filter {
- _.getName.toLowerCase.endsWith(".jar")
- }
-
- def jarFilesForScala: Array[File] = {
- val libFiles = jarFilesForScalaFilter(jarFilesAt(new File("lib")))
- val targetFiles = jarFilesForScalaFilter(jarFilesAt(new File("target" +
- File.separator + s"scala-${scalaVersionNoPatch}")))
- // Use libFiles is target is empty.
- if (targetFiles.size > 0) targetFiles else libFiles
- }
-
- def jarFilesForScalaFilter(jars: Array[File]): Array[File] =
- jars.filterNot { f =>
- f.getName.toLowerCase.endsWith("-javadoc.jar") ||
- f.getName.toLowerCase.endsWith("-sources.jar")
- }
-
- def recursiveListFiles(f: File): Array[File] = {
- Option(f.listFiles) map { these =>
- these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
- } getOrElse Array[File]()
- }
-
- def getSparkHome(sparkHome: Option[String]): String = {
- sparkHome getOrElse {
- sys.env.getOrElse("SPARK_HOME", ".")
- }
- }
-
- def versionNoPatch(fullVersion: String): String = {
- val v = """^(\d+\.\d+)""".r
- val versionNoPatch = for {
- v(np) <- v findFirstIn fullVersion
- } yield np
- versionNoPatch.getOrElse(fullVersion)
- }
-
- def scalaVersionNoPatch: String = versionNoPatch(BuildInfo.scalaVersion)
-
- def detectSbt(ca: ConsoleArgs): String = {
- ca.build.sbt map {
- _.getCanonicalPath
- } getOrElse {
- val f = new File(Seq(ca.common.pioHome.get, "sbt", "sbt").mkString(
- File.separator))
- if (f.exists) f.getCanonicalPath else "sbt"
- }
- }
-
- def stripMarginAndNewlines(string: String): String =
- string.stripMargin.replaceAll("\n", " ")
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala
deleted file mode 100644
index 20c9e02..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.tools.Runner
-
-case class ExportArgs(
- appId: Int = 0,
- channel: Option[String] = None,
- outputPath: String = "",
- format: String = "json")
-
-object Export {
- def eventsToFile(ca: ConsoleArgs): Int = {
- val channelArg = ca.export.channel
- .map(ch => Seq("--channel", ch)).getOrElse(Nil)
- Runner.runOnSpark(
- "org.apache.predictionio.tools.export.EventsToFile",
- Seq(
- "--appid",
- ca.export.appId.toString,
- "--output",
- ca.export.outputPath,
- "--format",
- ca.export.format) ++ channelArg,
- ca,
- Nil)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala
deleted file mode 100644
index b808574..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.tools.Runner
-
-case class ImportArgs(
- appId: Int = 0,
- channel: Option[String] = None,
- inputPath: String = "")
-
-object Import {
- def fileToEvents(ca: ConsoleArgs): Int = {
- val channelArg = ca.imprt.channel
- .map(ch => Seq("--channel", ch)).getOrElse(Nil)
- Runner.runOnSpark(
- "org.apache.predictionio.tools.imprt.FileToEvents",
- Seq(
- "--appid",
- ca.imprt.appId.toString,
- "--input",
- ca.imprt.inputPath) ++ channelArg,
- ca,
- Nil)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
new file mode 100644
index 0000000..77075a7
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.console
+
+import org.apache.predictionio.tools.{
+ EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
+import org.apache.predictionio.tools.commands.Management
+import org.apache.predictionio.tools.commands.{
+ DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
+ BuildArgs, EngineArgs, AppDescription}
+import org.apache.predictionio.tools.commands.Engine
+import org.apache.predictionio.tools.commands.{
+ App => AppCmd, AccessKey => AccessKeysCmd}
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.tools.commands.Import
+import org.apache.predictionio.tools.commands.Export
+
+import grizzled.slf4j.Logging
+import scala.concurrent.{Future, ExecutionContext, Await}
+import scala.concurrent.duration._
+import scala.language.implicitConversions
+import scala.sys.process._
+import java.io.File
+
+import akka.actor.ActorSystem
+
+object Pio extends Logging {
+
+ private implicit def eitherToInt[A, B](result: Either[A, B]): Int = {
+ result fold (_ => 1, _ => 0)
+ }
+
+ private def doOnSuccess[A, B](result: Either[A, B])(f: B => Int): Int = {
+ result match {
+ case Left(_) => 1
+ case Right(res) => f(res)
+ }
+ }
+
+ private def processAwaitAndClean(maybeProc: Expected[(Process, () => Unit)]) = {
+ maybeProc match {
+ case Left(_) => 1
+
+ case Right((proc, cleanup)) =>
+ Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
+ def run(): Unit = {
+ cleanup()
+ proc.destroy()
+ }
+ }))
+ val returnVal = proc.exitValue()
+ cleanup()
+ returnVal
+ }
+ }
+
+ def version(): Int = {
+ println(Management.version)
+ 0
+ }
+
+ def build(
+ buildArgs: BuildArgs,
+ pioHome: String,
+ manifestJson: File,
+ verbose: Boolean = false): Int = {
+
+ doOnSuccess(Engine.build(buildArgs, pioHome, manifestJson, verbose)) {
+ _ => info("Your engine is ready for training.")
+ 0
+ }
+ }
+
+ def unregister(manifestJson: File): Int = Engine.unregister(manifestJson)
+
+ def train(
+ ea: EngineArgs,
+ wa: WorkflowArgs,
+ sa: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Int =
+ processAwaitAndClean(Engine.train(ea, wa, sa, pioHome, verbose))
+
+ def eval(
+ ea: EngineArgs,
+ wa: WorkflowArgs,
+ sa: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Int =
+ processAwaitAndClean(Engine.train(ea, wa, sa, pioHome, verbose))
+
+ def deploy(
+ ea: EngineArgs,
+ engineInstanceId: Option[String],
+ serverArgs: ServerArgs,
+ sparkArgs: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Int =
+ processAwaitAndClean(Engine.deploy(
+ ea, engineInstanceId, serverArgs, sparkArgs, pioHome, verbose))
+
+ def undeploy(da: DeployArgs): Int = Engine.undeploy(da)
+
+ def dashboard(da: DashboardArgs): Int = {
+ Management.dashboard(da).awaitTermination
+ 0
+ }
+
+ def eventserver(ea: EventServerArgs): Int = {
+ Management.eventserver(ea).awaitTermination
+ 0
+ }
+
+ def adminserver(aa: AdminServerArgs): Int = {
+ Management.adminserver(aa).awaitTermination
+ 0
+ }
+
+ def run(
+ mainClass: String,
+ driverArguments: Seq[String],
+ manifestJson: File,
+ buildArgs: BuildArgs,
+ sparkArgs: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Int =
+ doOnSuccess(Engine.run(
+ mainClass, driverArguments, manifestJson,
+ buildArgs, sparkArgs, pioHome, verbose)) { proc =>
+
+ val r = proc.exitValue()
+ if (r != 0) {
+ error(s"Return code of previous step is ${r}. Aborting.")
+ return 1
+ }
+ r
+ }
+
+
+ def status(pioHome: Option[String], sparkHome: Option[String]): Int = {
+ Management.status(pioHome, sparkHome)
+ }
+
+ def imprt(ia: ImportArgs, sa: SparkArgs, pioHome: String): Int = {
+ processAwaitAndClean(Import.fileToEvents(ia, sa, pioHome))
+ }
+
+ def export(ea: ExportArgs, sa: SparkArgs, pioHome: String): Int = {
+ processAwaitAndClean(Export.eventsToFile(ea, sa, pioHome))
+ }
+
+ object App {
+
+ def create(
+ name: String,
+ id: Option[Int] = None,
+ description: Option[String] = None,
+ accessKey: String = ""): Int =
+ doOnSuccess(AppCmd.create(name, id, description, accessKey)) { appDesc =>
+ info("Created a new app:")
+ info(s" Name: ${appDesc.app.name}")
+ info(s" ID: ${appDesc.app.id}")
+ info(s"Access Key: ${appDesc.keys.head.key}")
+ 0
+ }
+
+ def list(): Int = {
+ val title = "Name"
+ val ak = "Access Key"
+ val apps = AppCmd.list
+ info(f"$title%20s | ID | $ak%64s | Allowed Event(s)")
+ apps foreach { appDesc =>
+ appDesc.keys foreach { k =>
+ val events =
+ if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
+ info(f"${appDesc.app.name}%20s | ${appDesc.app.id}%4d | ${k.key}%64s | $events%s")
+ }
+ }
+ info(s"Finished listing ${apps.size} app(s).")
+ 0
+ }
+
+ def show(appName: String): Int =
+ doOnSuccess(AppCmd.show(appName)) { case (appDesc, chans) =>
+ info(s" App Name: ${appDesc.app.name}")
+ info(s" App ID: ${appDesc.app.id}")
+ info(s" Description: ${appDesc.app.description.getOrElse("")}")
+
+ var firstKey = true
+ appDesc.keys foreach { k =>
+ val events =
+ if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
+ if (firstKey) {
+ info(f" Access Key: ${k.key}%s | ${events}%s")
+ firstKey = false
+ } else {
+ info(f" ${k.key}%s | ${events}%s")
+ }
+ }
+ var firstChan = true
+ val titleName = "Channel Name"
+ val titleID = "Channel ID"
+ chans.foreach { ch =>
+ if (firstChan) {
+ info(f" Channels: ${titleName}%16s | ${titleID}%10s ")
+ firstChan = false
+ }
+ info(f" ${ch.name}%16s | ${ch.id}%10s")
+ }
+ 0
+ }
+
+ def delete(name: String, force: Boolean = false): Int =
+ doOnSuccess(AppCmd.show(name)) { case (appDesc, chans) =>
+ info(s"The following app (including all channels) will be deleted. Are you sure?")
+ info(s" App Name: ${appDesc.app.name}")
+ info(s" App ID: ${appDesc.app.id}")
+ info(s" Description: ${appDesc.app.description.getOrElse("")}")
+ var firstChan = true
+ val titleName = "Channel Name"
+ val titleID = "Channel ID"
+ chans.foreach { ch =>
+ if (firstChan) {
+ info(f" Channels: ${titleName}%16s | ${titleID}%10s ")
+ firstChan = false
+ }
+ info(f" ${ch.name}%16s | ${ch.id}%10s")
+ }
+
+ val choice = if(force) "YES" else readLine("Enter 'YES' to proceed: ")
+ choice match {
+ case "YES" =>
+ AppCmd.delete(name)
+ case _ =>
+ info("Aborted.")
+ 0
+ }
+ }
+
+ def dataDelete(
+ name: String,
+ channel: Option[String] = None,
+ all: Boolean = false,
+ force: Boolean = false): Int =
+ doOnSuccess(AppCmd.show(name)) { case (appDesc, chans) =>
+
+ val channelId = channel.map { ch =>
+ val channelMap = chans.map(c => (c.name, c.id)).toMap
+ if (!channelMap.contains(ch)) {
+ error(s"Unable to delete data for channel.")
+ error(s"Channel ${ch} doesn't exist.")
+ return 1
+ }
+ channelMap(ch)
+ }
+ if (all) {
+ info(s"All data of the app (including default and all channels) will be deleted." +
+ " Are you sure?")
+ } else if (channelId.isDefined) {
+ info(s"Data of the following channel will be deleted. Are you sure?")
+ info(s"Channel Name: ${channel.get}")
+ info(s" Channel ID: ${channelId.get}")
+ } else {
+ info(s"Data of the following app (default channel only) will be deleted. Are you sure?")
+ }
+ info(s" App Name: ${appDesc.app.name}")
+ info(s" App ID: ${appDesc.app.id}")
+ info(s" Description: ${appDesc.app.description}")
+
+ val choice = if(force) "YES" else readLine("Enter 'YES' to proceed: ")
+ choice match {
+ case "YES" =>
+ AppCmd.dataDelete(name, channel, all)
+ case _ =>
+ info("Aborted.")
+ 0
+ }
+ }
+
+ def channelNew(appName: String, newChannel: String): Int =
+ AppCmd.channelNew(appName, newChannel)
+
+ def channelDelete(
+ appName: String,
+ deleteChannel: String,
+ force: Boolean = false): Int =
+ doOnSuccess(AppCmd.show(appName)) { case (appDesc, chans) =>
+ chans.find(chan => chan.name == deleteChannel) match {
+ case None =>
+ error(s"Unable to delete channel.")
+ error(s"Channel ${deleteChannel} doesn't exist.")
+ 1
+ case Some(chan) =>
+ info(s"The following channel will be deleted. Are you sure?")
+ info(s" Channel Name: ${deleteChannel}")
+ info(s" Channel ID: ${chan.id}")
+ info(s" App Name: ${appDesc.app.name}")
+ info(s" App ID: ${appDesc.app.id}")
+ val choice = if(force) "YES" else readLine("Enter 'YES' to proceed: ")
+ choice match {
+ case "YES" =>
+ AppCmd.channelDelete(appName, deleteChannel)
+ case _ =>
+ info("Aborted.")
+ 0
+ }
+ }
+ }
+
+ }
+
+ object AccessKey {
+
+ def create(
+ appName: String,
+ key: String,
+ events: Seq[String]): Int =
+ AccessKeysCmd.create(appName, key, events)
+
+ def list(app: Option[String]): Int =
+ doOnSuccess(AccessKeysCmd.list(app)) { keys =>
+ val title = "Access Key(s)"
+ info(f"$title%64s | App ID | Allowed Event(s)")
+ keys.sortBy(k => k.appid) foreach { k =>
+ val events =
+ if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
+ info(f"${k.key}%64s | ${k.appid}%6d | $events%s")
+ }
+ info(s"Finished listing ${keys.size} access key(s).")
+ 0
+ }
+
+ def delete(key: String): Int = AccessKeysCmd.delete(key)
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala
deleted file mode 100644
index 0136d25..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala
+++ /dev/null
@@ -1,432 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import java.io.BufferedInputStream
-import java.io.BufferedOutputStream
-import java.io.File
-import java.io.FileInputStream
-import java.io.FileOutputStream
-import java.net.ConnectException
-import java.net.URI
-import java.util.zip.ZipInputStream
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.controller.Utils
-import org.apache.predictionio.core.BuildInfo
-import org.apache.commons.io.FileUtils
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-import semverfi._
-
-import scala.io.Source
-import scala.sys.process._
-import scalaj.http._
-
-case class TemplateArgs(
- directory: String = "",
- repository: String = "",
- version: Option[String] = None,
- name: Option[String] = None,
- packageName: Option[String] = None,
- email: Option[String] = None)
-
-case class GitHubTag(
- name: String,
- zipball_url: String,
- tarball_url: String,
- commit: GitHubCommit)
-
-case class GitHubCommit(
- sha: String,
- url: String)
-
-case class GitHubCache(
- headers: Map[String, String],
- body: String)
-
-case class TemplateEntry(
- repo: String)
-
-case class TemplateMetaData(
- pioVersionMin: Option[String] = None)
-
-object Template extends Logging {
- implicit val formats = Utils.json4sDefaultFormats
-
- def templateMetaData(templateJson: File): TemplateMetaData = {
- if (!templateJson.exists) {
- warn(s"$templateJson does not exist. Template metadata will not be available. " +
- "(This is safe to ignore if you are not working on a template.)")
- TemplateMetaData()
- } else {
- val jsonString = Source.fromFile(templateJson)(scala.io.Codec.ISO8859).mkString
- val json = try {
- parse(jsonString)
- } catch {
- case e: org.json4s.ParserUtil.ParseException =>
- warn(s"$templateJson cannot be parsed. Template metadata will not be available.")
- return TemplateMetaData()
- }
- val pioVersionMin = json \ "pio" \ "version" \ "min"
- pioVersionMin match {
- case JString(s) => TemplateMetaData(pioVersionMin = Some(s))
- case _ => TemplateMetaData()
- }
- }
- }
-
- /** Creates a wrapper that provides the functionality of scalaj.http.Http()
- * with automatic proxy settings handling. The proxy settings will first
- * come from "git" followed by system properties "http.proxyHost" and
- * "http.proxyPort".
- *
- * @param url URL to be connected
- * @return
- */
- def httpOptionalProxy(url: String): HttpRequest = {
- val gitProxy = try {
- Some(Process("git config --global http.proxy").lines.toList(0))
- } catch {
- case e: Throwable => None
- }
-
- val (host, port) = gitProxy map { p =>
- val proxyUri = new URI(p)
- (Option(proxyUri.getHost),
- if (proxyUri.getPort == -1) None else Some(proxyUri.getPort))
- } getOrElse {
- (sys.props.get("http.proxyHost"),
- sys.props.get("http.proxyPort").map { p =>
- try {
- Some(p.toInt)
- } catch {
- case e: NumberFormatException => None
- }
- } getOrElse None)
- }
-
- (host, port) match {
- case (Some(h), Some(p)) => Http(url).proxy(h, p)
- case _ => Http(url)
- }
- }
-
- def getGitHubRepos(
- repos: Seq[String],
- apiType: String,
- repoFilename: String): Map[String, GitHubCache] = {
- val reposCache = try {
- val cache =
- Source.fromFile(repoFilename)(scala.io.Codec.ISO8859).mkString
- read[Map[String, GitHubCache]](cache)
- } catch {
- case e: Throwable => Map[String, GitHubCache]()
- }
- val newReposCache = reposCache ++ (try {
- repos.map { repo =>
- val url = s"https://api.github.com/repos/$repo/$apiType"
- val http = httpOptionalProxy(url)
- val response = reposCache.get(repo).map { cache =>
- cache.headers.get("ETag").map { etag =>
- http.header("If-None-Match", etag).asString
- } getOrElse {
- http.asString
- }
- } getOrElse {
- http.asString
- }
-
- val body = if (response.code == 304) {
- reposCache(repo).body
- } else {
- response.body
- }
-
- repo -> GitHubCache(headers = response.headers, body = body)
- }.toMap
- } catch {
- case e: ConnectException =>
- githubConnectErrorMessage(e)
- Map()
- })
- FileUtils.writeStringToFile(
- new File(repoFilename),
- write(newReposCache),
- "ISO-8859-1")
- newReposCache
- }
-
- def sub(repo: String, name: String, email: String, org: String): Unit = {
- val data = Map(
- "repo" -> repo,
- "name" -> name,
- "email" -> email,
- "org" -> org)
- try {
- httpOptionalProxy("https://update.prediction.io/templates.subscribe").
- postData("json=" + write(data)).asString
- } catch {
- case e: Throwable => error("Unable to subscribe.")
- }
- }
-
- def meta(repo: String, name: String, org: String): Unit = {
- try {
- httpOptionalProxy(
- s"https://meta.prediction.io/templates/$repo/$org/$name").asString
- } catch {
- case e: Throwable => debug("Template metadata unavailable.")
- }
- }
-
- def list(ca: ConsoleArgs): Int = {
- val templatesUrl = "https://templates.prediction.io/index.json"
- try {
- val templatesJson = Source.fromURL(templatesUrl).mkString("")
- val templates = read[List[TemplateEntry]](templatesJson)
- println("The following is a list of template IDs registered on " +
- "PredictionIO Template Gallery:")
- println()
- templates.sortBy(_.repo.toLowerCase).foreach { template =>
- println(template.repo)
- }
- println()
- println("Notice that it is possible use any GitHub repository as your " +
- "engine template ID (e.g. YourOrg/YourTemplate).")
- 0
- } catch {
- case e: Throwable =>
- error(s"Unable to list templates from $templatesUrl " +
- s"(${e.getMessage}). Aborting.")
- 1
- }
- }
-
- def githubConnectErrorMessage(e: ConnectException): Unit = {
- error(s"Unable to connect to GitHub (Reason: ${e.getMessage}). " +
- "Please check your network configuration and proxy settings.")
- }
-
- def get(ca: ConsoleArgs): Int = {
- val repos =
- getGitHubRepos(Seq(ca.template.repository), "tags", ".templates-cache")
-
- repos.get(ca.template.repository).map { repo =>
- try {
- read[List[GitHubTag]](repo.body)
- } catch {
- case e: MappingException =>
- error(s"Either ${ca.template.repository} is not a valid GitHub " +
- "repository, or it does not have any tag. Aborting.")
- return 1
- }
- } getOrElse {
- error(s"Failed to retrieve ${ca.template.repository}. Aborting.")
- return 1
- }
-
- val name = ca.template.name getOrElse {
- try {
- Process("git config --global user.name").lines.toList(0)
- } catch {
- case e: Throwable =>
- readLine("Please enter author's name: ")
- }
- }
-
- val organization = ca.template.packageName getOrElse {
- readLine(
- "Please enter the template's Scala package name (e.g. com.mycompany): ")
- }
-
- val email = ca.template.email getOrElse {
- try {
- Process("git config --global user.email").lines.toList(0)
- } catch {
- case e: Throwable =>
- readLine("Please enter author's e-mail address: ")
- }
- }
-
- println(s"Author's name: $name")
- println(s"Author's e-mail: $email")
- println(s"Author's organization: $organization")
-
- var subscribe = readLine("Would you like to be informed about new bug " +
- "fixes and security updates of this template? (Y/n) ")
- var valid = false
-
- do {
- subscribe match {
- case "" | "Y" | "y" =>
- sub(ca.template.repository, name, email, organization)
- valid = true
- case "n" | "N" =>
- meta(ca.template.repository, name, organization)
- valid = true
- case _ =>
- println("Please answer 'y' or 'n'")
- subscribe = readLine("(Y/n)? ")
- }
- } while (!valid)
-
- val repo = repos(ca.template.repository)
-
- println(s"Retrieving ${ca.template.repository}")
- val tags = read[List[GitHubTag]](repo.body)
- println(s"There are ${tags.size} tags")
-
- if (tags.size == 0) {
- println(s"${ca.template.repository} does not have any tag. Aborting.")
- return 1
- }
-
- val tag = ca.template.version.map { v =>
- tags.find(_.name == v).getOrElse {
- println(s"${ca.template.repository} does not have tag $v. Aborting.")
- return 1
- }
- } getOrElse tags.head
-
- println(s"Using tag ${tag.name}")
- val url =
- s"https://github.com/${ca.template.repository}/archive/${tag.name}.zip"
- println(s"Going to download $url")
- val trial = try {
- httpOptionalProxy(url).asBytes
- } catch {
- case e: ConnectException =>
- githubConnectErrorMessage(e)
- return 1
- }
- val finalTrial = try {
- trial.location.map { loc =>
- println(s"Redirecting to $loc")
- httpOptionalProxy(loc).asBytes
- } getOrElse trial
- } catch {
- case e: ConnectException =>
- githubConnectErrorMessage(e)
- return 1
- }
- val zipFilename =
- s"${ca.template.repository.replace('/', '-')}-${tag.name}.zip"
- FileUtils.writeByteArrayToFile(
- new File(zipFilename),
- finalTrial.body)
- val zis = new ZipInputStream(
- new BufferedInputStream(new FileInputStream(zipFilename)))
- val bufferSize = 4096
- val filesToModify = collection.mutable.ListBuffer[String]()
- var ze = zis.getNextEntry
- while (ze != null) {
- val filenameSegments = ze.getName.split(File.separatorChar)
- val destFilename = (ca.template.directory +: filenameSegments.tail).
- mkString(File.separator)
- if (ze.isDirectory) {
- new File(destFilename).mkdirs
- } else {
- val os = new BufferedOutputStream(
- new FileOutputStream(destFilename),
- bufferSize)
- val data = Array.ofDim[Byte](bufferSize)
- var count = zis.read(data, 0, bufferSize)
- while (count != -1) {
- os.write(data, 0, count)
- count = zis.read(data, 0, bufferSize)
- }
- os.flush()
- os.close()
-
- val nameOnly = new File(destFilename).getName
-
- if (organization != "" &&
- (nameOnly.endsWith(".scala") ||
- nameOnly == "build.sbt" ||
- nameOnly == "engine.json")) {
- filesToModify += destFilename
- }
- }
- ze = zis.getNextEntry
- }
- zis.close()
- new File(zipFilename).delete
-
- val engineJsonFile =
- new File(ca.template.directory, "engine.json")
-
- val engineJson = try {
- Some(parse(Source.fromFile(engineJsonFile).mkString))
- } catch {
- case e: java.io.IOException =>
- error("Unable to read engine.json. Skipping automatic package " +
- "name replacement.")
- None
- case e: MappingException =>
- error("Unable to parse engine.json. Skipping automatic package " +
- "name replacement.")
- None
- }
-
- val engineFactory = engineJson.map { ej =>
- (ej \ "engineFactory").extractOpt[String]
- } getOrElse None
-
- engineFactory.map { ef =>
- val pkgName = ef.split('.').dropRight(1).mkString(".")
- println(s"Replacing $pkgName with $organization...")
-
- filesToModify.foreach { ftm =>
- println(s"Processing $ftm...")
- val fileContent = Source.fromFile(ftm).getLines()
- val processedLines =
- fileContent.map(_.replaceAllLiterally(pkgName, organization))
- FileUtils.writeStringToFile(
- new File(ftm),
- processedLines.mkString("\n"))
- }
- } getOrElse {
- error("engineFactory is not found in engine.json. Skipping automatic " +
- "package name replacement.")
- }
-
- verifyTemplateMinVersion(new File(ca.template.directory, "template.json"))
-
- println(s"Engine template ${ca.template.repository} is now ready at " +
- ca.template.directory)
-
- 0
- }
-
- def verifyTemplateMinVersion(templateJsonFile: File): Unit = {
- val metadata = templateMetaData(templateJsonFile)
-
- metadata.pioVersionMin.foreach { pvm =>
- if (Version(BuildInfo.version) < Version(pvm)) {
- error(s"This engine template requires at least PredictionIO $pvm. " +
- s"The template may not work with PredictionIO ${BuildInfo.version}.")
- sys.exit(1)
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
index 402e853..2e9fb26 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
@@ -55,12 +55,13 @@ object Dashboard extends Logging with SSLConfiguration {
}
parser.parse(args, DashboardConfig()) map { dc =>
- createDashboard(dc)
+ createDashboard(dc).awaitTermination
}
}
- def createDashboard(dc: DashboardConfig): Unit = {
- implicit val system = ActorSystem("pio-dashboard")
+ def createDashboard(dc: DashboardConfig): ActorSystem = {
+ val systemName = "pio-dashboard"
+ implicit val system = ActorSystem(systemName)
val service =
system.actorOf(Props(classOf[DashboardActor], dc), "dashboard")
implicit val timeout = Timeout(5.seconds)
@@ -72,7 +73,7 @@ object Dashboard extends Logging with SSLConfiguration {
interface = dc.ip,
port = dc.port,
settings = Some(settings.copy(sslEncryption = sslEnforced)))
- system.awaitTermination
+ system
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
----------------------------------------------------------------------
diff --git a/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt b/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
index f97c8ce..909e15c 100644
--- a/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
+++ b/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
@@ -1,25 +1,3 @@
Usage: pio template list
-Retrieves a list of available template IDs.
-
-Usage: pio template get <template ID> <new engine directory>
- [--version <version>]
- [--name <value>] [--package <value>] [--email <value>]
-
-Seeds a directory with an engine template.
-
- <template ID>
- Engine template ID.
- <new engine directory>
- Location of the new engine.
- --version <value>
- The template version to get. By default, the most recently tagged version
- will be downloaded.
- --name <value>
- Name of the author of the new engine.
- --package <value>
- Scala package name of the new engine.
- --email <value>
- E-mail address of the author of the new engine. Specify this if you want
- to receive updates (critical bug fixes, etc) about the engine template
- that you are going to use.
+No longer supported! Please use git to download and manage your templates.
[3/3] incubator-predictionio git commit: [PIO-28] Console refactor
Posted by do...@apache.org.
[PIO-28] Console refactor
Extract logic from functions handling console commands in tools package
Closes #283
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/e4a3c0c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/e4a3c0c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/e4a3c0c9
Branch: refs/heads/develop
Commit: e4a3c0c9fc1251d7355d921acb66168226446b3f
Parents: f7c30d3
Author: Marcin Ziemi\u0144ski <zi...@gmail.com>
Authored: Tue Nov 1 10:54:07 2016 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Nov 1 10:54:07 2016 -0700
----------------------------------------------------------------------
.../predictionio/data/api/EventServer.scala | 5 +-
.../org/apache/predictionio/tools/Common.scala | 114 +++
.../predictionio/tools/RegisterEngine.scala | 18 +-
.../apache/predictionio/tools/RunServer.scala | 176 ++---
.../apache/predictionio/tools/RunWorkflow.scala | 195 +----
.../org/apache/predictionio/tools/Runner.scala | 66 +-
.../predictionio/tools/admin/AdminAPI.scala | 5 +-
.../predictionio/tools/commands/AccessKey.scala | 70 ++
.../predictionio/tools/commands/App.scala | 365 +++++++++
.../predictionio/tools/commands/Engine.scala | 424 +++++++++++
.../predictionio/tools/commands/Export.scala | 54 ++
.../predictionio/tools/commands/Import.scala | 51 ++
.../tools/commands/Management.scala | 178 +++++
.../predictionio/tools/commands/Template.scala | 71 ++
.../predictionio/tools/console/AccessKey.scala | 86 ---
.../apache/predictionio/tools/console/App.scala | 540 --------------
.../predictionio/tools/console/Console.scala | 734 +++----------------
.../predictionio/tools/console/Export.scala | 45 --
.../predictionio/tools/console/Import.scala | 42 --
.../apache/predictionio/tools/console/Pio.scala | 352 +++++++++
.../predictionio/tools/console/Template.scala | 432 -----------
.../tools/dashboard/Dashboard.scala | 9 +-
.../tools/console/template.scala.txt | 24 +-
.../tools/console/upgrade.scala.txt | 14 +-
24 files changed, 1932 insertions(+), 2138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
index a0ba40f..648316e 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
@@ -611,7 +611,7 @@ case class EventServerConfig(
stats: Boolean = false)
object EventServer {
- def createEventServer(config: EventServerConfig): Unit = {
+ def createEventServer(config: EventServerConfig): ActorSystem = {
implicit val system = ActorSystem("EventServerSystem")
val eventClient = Storage.getLEvents()
@@ -630,7 +630,7 @@ object EventServer {
if (config.stats) system.actorOf(Props[StatsActor], "StatsActor")
system.actorOf(Props[PluginsActor], "PluginsActor")
serverActor ! StartServer(config.ip, config.port)
- system.awaitTermination()
+ system
}
}
@@ -639,5 +639,6 @@ object Run {
EventServer.createEventServer(EventServerConfig(
ip = "0.0.0.0",
port = 7070))
+ .awaitTermination
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
new file mode 100644
index 0000000..c379138
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools
+
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.tools.ReturnTypes._
+
+import grizzled.slf4j.Logging
+import java.io.File
+
+
+object ReturnTypes {
+ sealed case class Ok()
+
+ type MaybeError = Either[String, Ok]
+ type Expected[T] = Either[String, T]
+
+ val Success: MaybeError = Right(Ok())
+}
+
+trait EitherLogging extends Logging {
+ import ReturnTypes._
+
+ protected def logAndFail[T](msg: => String): Expected[T] = {
+ error(msg)
+ Left(msg)
+ }
+
+ protected def logOnFail[T](msg: => String, t: => Throwable): Expected[T] = {
+ error(msg, t)
+ Left(msg)
+ }
+
+ protected def logAndReturn[T](value: T, msg: => Any): Expected[T] = {
+ info(msg)
+ Right(value)
+ }
+
+ protected def logAndSucceed(msg: => Any): MaybeError = {
+ info(msg)
+ Success
+ }
+}
+
+object Common extends EitherLogging {
+
+ def getSparkHome(sparkHome: Option[String]): String = {
+ sparkHome getOrElse {
+ sys.env.getOrElse("SPARK_HOME", ".")
+ }
+ }
+
+ def versionNoPatch(fullVersion: String): String = {
+ val v = """^(\d+\.\d+)""".r
+ val versionNoPatch = for {
+ v(np) <- v findFirstIn fullVersion
+ } yield np
+ versionNoPatch.getOrElse(fullVersion)
+ }
+
+ def jarFilesForScala: Array[File] = {
+ def recursiveListFiles(f: File): Array[File] = {
+ Option(f.listFiles) map { these =>
+ these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
+ } getOrElse Array[File]()
+ }
+ def jarFilesForScalaFilter(jars: Array[File]): Array[File] =
+ jars.filterNot { f =>
+ f.getName.toLowerCase.endsWith("-javadoc.jar") ||
+ f.getName.toLowerCase.endsWith("-sources.jar")
+ }
+ def jarFilesAt(path: File): Array[File] = recursiveListFiles(path) filter {
+ _.getName.toLowerCase.endsWith(".jar")
+ }
+ val libFiles = jarFilesForScalaFilter(jarFilesAt(new File("lib")))
+ val scalaVersionNoPatch = Common.versionNoPatch(BuildInfo.scalaVersion)
+ val targetFiles = jarFilesForScalaFilter(jarFilesAt(new File("target" +
+ File.separator + s"scala-${scalaVersionNoPatch}")))
+ // Use libFiles is target is empty.
+ if (targetFiles.size > 0) targetFiles else libFiles
+ }
+
+ def coreAssembly(pioHome: String): Expected[File] = {
+ val core = s"pio-assembly-${BuildInfo.version}.jar"
+ val coreDir =
+ if (new File(pioHome + File.separator + "RELEASE").exists) {
+ new File(pioHome + File.separator + "lib")
+ } else {
+ new File(pioHome + File.separator + "assembly")
+ }
+ val coreFile = new File(coreDir, core)
+ if (coreFile.exists) {
+ Right(coreFile)
+ } else {
+ logAndFail(s"PredictionIO Core Assembly (${coreFile.getCanonicalPath}) does " +
+ "not exist. Aborting.")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala b/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
index 189e5a8..334981c 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
@@ -24,6 +24,7 @@ import grizzled.slf4j.Logging
import org.apache.predictionio.data.storage.EngineManifest
import org.apache.predictionio.data.storage.EngineManifestSerializer
import org.apache.predictionio.data.storage.Storage
+import org.apache.predictionio.tools.ReturnTypes._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
@@ -32,35 +33,34 @@ import org.json4s.native.Serialization.read
import scala.io.Source
-object RegisterEngine extends Logging {
+object RegisterEngine extends EitherLogging {
val engineManifests = Storage.getMetaDataEngineManifests
implicit val formats = DefaultFormats + new EngineManifestSerializer
def registerEngine(
jsonManifest: File,
engineFiles: Seq[File],
- copyLocal: Boolean = false): Unit = {
+ copyLocal: Boolean = false): MaybeError = {
val jsonString = try {
Source.fromFile(jsonManifest).mkString
} catch {
case e: java.io.FileNotFoundException =>
- error(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
- sys.exit(1)
+ return logAndFail(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
}
val engineManifest = read[EngineManifest](jsonString)
info(s"Registering engine ${engineManifest.id} ${engineManifest.version}")
engineManifests.update(
engineManifest.copy(files = engineFiles.map(_.toURI.toString)), true)
+ Success
}
- def unregisterEngine(jsonManifest: File): Unit = {
+ def unregisterEngine(jsonManifest: File): MaybeError = {
val jsonString = try {
Source.fromFile(jsonManifest).mkString
} catch {
case e: java.io.FileNotFoundException =>
- error(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
- sys.exit(1)
+ return logAndFail(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
}
val fileEngineManifest = read[EngineManifest](jsonString)
val engineManifest = engineManifests.get(
@@ -78,9 +78,9 @@ object RegisterEngine extends Logging {
}
engineManifests.delete(em.id, em.version)
- info(s"Unregistered engine ${em.id} ${em.version}")
+ logAndSucceed(s"Unregistered engine ${em.id} ${em.version}")
} getOrElse {
- error(s"${fileEngineManifest.id} ${fileEngineManifest.version} is not " +
+ logAndFail(s"${fileEngineManifest.id} ${fileEngineManifest.version} is not " +
"registered.")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
index d091654..1431432 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
@@ -23,159 +23,75 @@ import java.net.URI
import grizzled.slf4j.Logging
import org.apache.predictionio.data.storage.EngineManifest
+import org.apache.predictionio.tools.ReturnTypes._
import org.apache.predictionio.tools.console.ConsoleArgs
import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.workflow.JsonExtractorOption
+import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
import scala.sys.process._
-object RunServer extends Logging {
- def runServer(
- ca: ConsoleArgs,
- core: File,
- em: EngineManifest,
- engineInstanceId: String): Int = {
- val pioEnvVars = sys.env.filter(kv => kv._1.startsWith("PIO_")).map(kv =>
- s"${kv._1}=${kv._2}"
- ).mkString(",")
-
- val sparkHome = ca.common.sparkHome.getOrElse(
- sys.env.getOrElse("SPARK_HOME", "."))
-
- val extraFiles = WorkflowUtils.thirdPartyConfFiles
-
- val driverClassPathIndex =
- ca.common.sparkPassThrough.indexOf("--driver-class-path")
- val driverClassPathPrefix =
- if (driverClassPathIndex != -1) {
- Seq(ca.common.sparkPassThrough(driverClassPathIndex + 1))
- } else {
- Seq()
- }
- val extraClasspaths =
- driverClassPathPrefix ++ WorkflowUtils.thirdPartyClasspaths
+case class DeployArgs(
+ ip: String = "0.0.0.0",
+ port: Int = 8000,
+ logUrl: Option[String] = None,
+ logPrefix: Option[String] = None)
- val deployModeIndex =
- ca.common.sparkPassThrough.indexOf("--deploy-mode")
- val deployMode = if (deployModeIndex != -1) {
- ca.common.sparkPassThrough(deployModeIndex + 1)
- } else {
- "client"
- }
+case class EventServerArgs(
+ enabled: Boolean = false,
+ ip: String = "0.0.0.0",
+ port: Int = 7070,
+ stats: Boolean = false)
- val mainJar =
- if (ca.build.uberJar) {
- if (deployMode == "cluster") {
- em.files.filter(_.startsWith("hdfs")).head
- } else {
- em.files.filterNot(_.startsWith("hdfs")).head
- }
- } else {
- if (deployMode == "cluster") {
- em.files.filter(_.contains("pio-assembly")).head
- } else {
- core.getCanonicalPath
- }
- }
-
- val jarFiles = (em.files ++ Option(new File(ca.common.pioHome.get, "plugins")
- .listFiles()).getOrElse(Array.empty[File]).map(_.getAbsolutePath)).mkString(",")
-
- val sparkSubmit =
- Seq(Seq(sparkHome, "bin", "spark-submit").mkString(File.separator)) ++
- ca.common.sparkPassThrough ++
- Seq(
- "--class",
- "org.apache.predictionio.workflow.CreateServer",
- "--name",
- s"PredictionIO Engine Instance: ${engineInstanceId}") ++
- (if (!ca.build.uberJar) {
- Seq("--jars", jarFiles)
- } else Seq()) ++
- (if (extraFiles.size > 0) {
- Seq("--files", extraFiles.mkString(","))
- } else {
- Seq()
- }) ++
- (if (extraClasspaths.size > 0) {
- Seq("--driver-class-path", extraClasspaths.mkString(":"))
- } else {
- Seq()
- }) ++
- (if (ca.common.sparkKryo) {
- Seq(
- "--conf",
- "spark.serializer=org.apache.spark.serializer.KryoSerializer")
- } else {
- Seq()
- }) ++
- Seq(
- mainJar,
- "--engineInstanceId",
- engineInstanceId,
- "--ip",
- ca.deploy.ip,
- "--port",
- ca.deploy.port.toString,
- "--event-server-ip",
- ca.eventServer.ip,
- "--event-server-port",
- ca.eventServer.port.toString) ++
- (if (ca.accessKey.accessKey != "") {
- Seq("--accesskey", ca.accessKey.accessKey)
- } else {
- Seq()
- }) ++
- (if (ca.eventServer.enabled) Seq("--feedback") else Seq()) ++
- (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
- (if (ca.common.verbose) Seq("--verbose") else Seq()) ++
- ca.deploy.logUrl.map(x => Seq("--log-url", x)).getOrElse(Seq()) ++
- ca.deploy.logPrefix.map(x => Seq("--log-prefix", x)).getOrElse(Seq()) ++
- Seq("--json-extractor", ca.common.jsonExtractor.toString)
+case class ServerArgs(
+ deploy: DeployArgs = DeployArgs(),
+ eventServer: EventServerArgs = EventServerArgs(),
+ batch: String = "",
+ accessKey: String = "",
+ variantJson: File = new File("engine.json"),
+ jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
- info(s"Submission command: ${sparkSubmit.mkString(" ")}")
- val proc =
- Process(sparkSubmit, None, "CLASSPATH" -> "", "SPARK_YARN_USER_ENV" -> pioEnvVars).run()
- Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
- def run(): Unit = {
- proc.destroy()
- }
- }))
- proc.exitValue()
- }
+object RunServer extends Logging {
- def newRunServer(
- ca: ConsoleArgs,
+ def runServer(
+ engineInstanceId: String,
+ serverArgs: ServerArgs,
+ sparkArgs: SparkArgs,
em: EngineManifest,
- engineInstanceId: String): Int = {
+ pioHome: String,
+ verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
val jarFiles = em.files.map(new URI(_)) ++
- Option(new File(ca.common.pioHome.get, "plugins").listFiles())
+ Option(new File(pioHome, "plugins").listFiles())
.getOrElse(Array.empty[File]).map(_.toURI)
val args = Seq(
"--engineInstanceId",
engineInstanceId,
"--engine-variant",
- ca.common.variantJson.toURI.toString,
+ serverArgs.variantJson.toURI.toString,
"--ip",
- ca.deploy.ip,
+ serverArgs.deploy.ip,
"--port",
- ca.deploy.port.toString,
+ serverArgs.deploy.port.toString,
"--event-server-ip",
- ca.eventServer.ip,
+ serverArgs.eventServer.ip,
"--event-server-port",
- ca.eventServer.port.toString) ++
- (if (ca.accessKey.accessKey != "") {
- Seq("--accesskey", ca.accessKey.accessKey)
+ serverArgs.eventServer.port.toString) ++
+ (if (serverArgs.accessKey != "") {
+ Seq("--accesskey", serverArgs.accessKey)
} else {
Nil
}) ++
- (if (ca.eventServer.enabled) Seq("--feedback") else Nil) ++
- (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Nil) ++
- (if (ca.common.verbose) Seq("--verbose") else Nil) ++
- ca.deploy.logUrl.map(x => Seq("--log-url", x)).getOrElse(Nil) ++
- ca.deploy.logPrefix.map(x => Seq("--log-prefix", x)).getOrElse(Nil) ++
- Seq("--json-extractor", ca.common.jsonExtractor.toString)
-
- Runner.runOnSpark("org.apache.predictionio.workflow.CreateServer", args, ca, jarFiles)
+ (if (serverArgs.eventServer.enabled) Seq("--feedback") else Nil) ++
+ (if (serverArgs.batch != "") Seq("--batch", serverArgs.batch) else Nil) ++
+ (if (verbose) Seq("--verbose") else Nil) ++
+ serverArgs.deploy.logUrl.map(x => Seq("--log-url", x)).getOrElse(Nil) ++
+ serverArgs.deploy.logPrefix.map(x => Seq("--log-prefix", x)).getOrElse(Nil) ++
+ Seq("--json-extractor", serverArgs.jsonExtractor.toString)
+
+ Runner.runOnSpark(
+ "org.apache.predictionio.workflow.CreateServer",
+ args, sparkArgs, jarFiles, pioHome, verbose)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
index 097eb83..8b8d769 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
@@ -24,156 +24,39 @@ import java.net.URI
import grizzled.slf4j.Logging
import org.apache.predictionio.data.storage.EngineManifest
import org.apache.predictionio.tools.console.ConsoleArgs
+import org.apache.predictionio.tools.ReturnTypes._
import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.workflow.JsonExtractorOption
+import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
+
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import scala.sys.process._
-object RunWorkflow extends Logging {
- def runWorkflow(
- ca: ConsoleArgs,
- core: File,
- em: EngineManifest,
- variantJson: File): Int = {
- // Collect and serialize PIO_* environmental variables
- val pioEnvVars = sys.env.filter(kv => kv._1.startsWith("PIO_")).map(kv =>
- s"${kv._1}=${kv._2}"
- ).mkString(",")
-
- val sparkHome = ca.common.sparkHome.getOrElse(
- sys.env.getOrElse("SPARK_HOME", "."))
-
- val hadoopConf = new Configuration
- val hdfs = FileSystem.get(hadoopConf)
-
- val driverClassPathIndex =
- ca.common.sparkPassThrough.indexOf("--driver-class-path")
- val driverClassPathPrefix =
- if (driverClassPathIndex != -1) {
- Seq(ca.common.sparkPassThrough(driverClassPathIndex + 1))
- } else {
- Seq()
- }
- val extraClasspaths =
- driverClassPathPrefix ++ WorkflowUtils.thirdPartyClasspaths
-
- val deployModeIndex =
- ca.common.sparkPassThrough.indexOf("--deploy-mode")
- val deployMode = if (deployModeIndex != -1) {
- ca.common.sparkPassThrough(deployModeIndex + 1)
- } else {
- "client"
- }
-
- val extraFiles = WorkflowUtils.thirdPartyConfFiles
+case class WorkflowArgs(
+ batch: String = "",
+ variantJson: File = new File("engine.json"),
+ verbosity: Int = 0,
+ engineParamsKey: Option[String] = None,
+ engineFactory: Option[String] = None,
+ evaluation: Option[String] = None,
+ engineParamsGenerator: Option[String] = None,
+ stopAfterRead: Boolean = false,
+ stopAfterPrepare: Boolean = false,
+ skipSanityCheck: Boolean = false,
+ jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
- val mainJar =
- if (ca.build.uberJar) {
- if (deployMode == "cluster") {
- em.files.filter(_.startsWith("hdfs")).head
- } else {
- em.files.filterNot(_.startsWith("hdfs")).head
- }
- } else {
- if (deployMode == "cluster") {
- em.files.filter(_.contains("pio-assembly")).head
- } else {
- core.getCanonicalPath
- }
- }
-
- val workMode =
- ca.common.evaluation.map(_ => "Evaluation").getOrElse("Training")
-
- val engineLocation = Seq(
- sys.env("PIO_FS_ENGINESDIR"),
- em.id,
- em.version)
-
- if (deployMode == "cluster") {
- val dstPath = new Path(engineLocation.mkString(Path.SEPARATOR))
- info("Cluster deploy mode detected. Trying to copy " +
- s"${variantJson.getCanonicalPath} to " +
- s"${hdfs.makeQualified(dstPath).toString}.")
- hdfs.copyFromLocalFile(new Path(variantJson.toURI), dstPath)
- }
-
- val sparkSubmit =
- Seq(Seq(sparkHome, "bin", "spark-submit").mkString(File.separator)) ++
- ca.common.sparkPassThrough ++
- Seq(
- "--class",
- "org.apache.predictionio.workflow.CreateWorkflow",
- "--name",
- s"PredictionIO $workMode: ${em.id} ${em.version} (${ca.common.batch})") ++
- (if (!ca.build.uberJar) {
- Seq("--jars", em.files.mkString(","))
- } else Seq()) ++
- (if (extraFiles.size > 0) {
- Seq("--files", extraFiles.mkString(","))
- } else {
- Seq()
- }) ++
- (if (extraClasspaths.size > 0) {
- Seq("--driver-class-path", extraClasspaths.mkString(":"))
- } else {
- Seq()
- }) ++
- (if (ca.common.sparkKryo) {
- Seq(
- "--conf",
- "spark.serializer=org.apache.spark.serializer.KryoSerializer")
- } else {
- Seq()
- }) ++
- Seq(
- mainJar,
- "--env",
- pioEnvVars,
- "--engine-id",
- em.id,
- "--engine-version",
- em.version,
- "--engine-variant",
- if (deployMode == "cluster") {
- hdfs.makeQualified(new Path(
- (engineLocation :+ variantJson.getName).mkString(Path.SEPARATOR))).
- toString
- } else {
- variantJson.getCanonicalPath
- },
- "--verbosity",
- ca.common.verbosity.toString) ++
- ca.common.engineFactory.map(
- x => Seq("--engine-factory", x)).getOrElse(Seq()) ++
- ca.common.engineParamsKey.map(
- x => Seq("--engine-params-key", x)).getOrElse(Seq()) ++
- (if (deployMode == "cluster") Seq("--deploy-mode", "cluster") else Seq()) ++
- (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
- (if (ca.common.verbose) Seq("--verbose") else Seq()) ++
- (if (ca.common.skipSanityCheck) Seq("--skip-sanity-check") else Seq()) ++
- (if (ca.common.stopAfterRead) Seq("--stop-after-read") else Seq()) ++
- (if (ca.common.stopAfterPrepare) {
- Seq("--stop-after-prepare")
- } else {
- Seq()
- }) ++
- ca.common.evaluation.map(x => Seq("--evaluation-class", x)).
- getOrElse(Seq()) ++
- // If engineParamsGenerator is specified, it overrides the evaluation.
- ca.common.engineParamsGenerator.orElse(ca.common.evaluation)
- .map(x => Seq("--engine-params-generator-class", x))
- .getOrElse(Seq()) ++
- (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
- Seq("--json-extractor", ca.common.jsonExtractor.toString)
+object RunWorkflow extends Logging {
- info(s"Submission command: ${sparkSubmit.mkString(" ")}")
- Process(sparkSubmit, None, "CLASSPATH" -> "", "SPARK_YARN_USER_ENV" -> pioEnvVars).!
- }
+ def runWorkflow(
+ wa: WorkflowArgs,
+ sa: SparkArgs,
+ em: EngineManifest,
+ pioHome: String,
+ verbose: Boolean = false): Expected[(Process, () => Unit)] = {
- def newRunWorkflow(ca: ConsoleArgs, em: EngineManifest): Int = {
val jarFiles = em.files.map(new URI(_))
val args = Seq(
"--engine-id",
@@ -181,35 +64,37 @@ object RunWorkflow extends Logging {
"--engine-version",
em.version,
"--engine-variant",
- ca.common.variantJson.toURI.toString,
+ wa.variantJson.toURI.toString,
"--verbosity",
- ca.common.verbosity.toString) ++
- ca.common.engineFactory.map(
+ wa.verbosity.toString) ++
+ wa.engineFactory.map(
x => Seq("--engine-factory", x)).getOrElse(Seq()) ++
- ca.common.engineParamsKey.map(
+ wa.engineParamsKey.map(
x => Seq("--engine-params-key", x)).getOrElse(Seq()) ++
- (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
- (if (ca.common.verbose) Seq("--verbose") else Seq()) ++
- (if (ca.common.skipSanityCheck) Seq("--skip-sanity-check") else Seq()) ++
- (if (ca.common.stopAfterRead) Seq("--stop-after-read") else Seq()) ++
- (if (ca.common.stopAfterPrepare) {
+ (if (wa.batch != "") Seq("--batch", wa.batch) else Seq()) ++
+ (if (verbose) Seq("--verbose") else Seq()) ++
+ (if (wa.skipSanityCheck) Seq("--skip-sanity-check") else Seq()) ++
+ (if (wa.stopAfterRead) Seq("--stop-after-read") else Seq()) ++
+ (if (wa.stopAfterPrepare) {
Seq("--stop-after-prepare")
} else {
Seq()
}) ++
- ca.common.evaluation.map(x => Seq("--evaluation-class", x)).
+ wa.evaluation.map(x => Seq("--evaluation-class", x)).
getOrElse(Seq()) ++
// If engineParamsGenerator is specified, it overrides the evaluation.
- ca.common.engineParamsGenerator.orElse(ca.common.evaluation)
+ wa.engineParamsGenerator.orElse(wa.evaluation)
.map(x => Seq("--engine-params-generator-class", x))
.getOrElse(Seq()) ++
- (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
- Seq("--json-extractor", ca.common.jsonExtractor.toString)
+ (if (wa.batch != "") Seq("--batch", wa.batch) else Seq()) ++
+ Seq("--json-extractor", wa.jsonExtractor.toString)
Runner.runOnSpark(
"org.apache.predictionio.workflow.CreateWorkflow",
args,
- ca,
- jarFiles)
+ sa,
+ jarFiles,
+ pioHome,
+ verbose)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index b3ec51c..d9752df 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -21,16 +21,22 @@ package org.apache.predictionio.tools
import java.io.File
import java.net.URI
-import grizzled.slf4j.Logging
import org.apache.predictionio.tools.console.ConsoleArgs
import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.tools.ReturnTypes._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import scala.sys.process._
-object Runner extends Logging {
+case class SparkArgs(
+ sparkHome: Option[String] = None,
+ sparkPassThrough: Seq[String] = Seq(),
+ sparkKryo: Boolean = false,
+ scratchUri: Option[URI] = None)
+
+object Runner extends EitherLogging {
def envStringToMap(env: String): Map[String, String] =
env.split(',').flatMap(p =>
p.split('=') match {
@@ -95,26 +101,27 @@ object Runner extends Logging {
def runOnSpark(
className: String,
classArgs: Seq[String],
- ca: ConsoleArgs,
- extraJars: Seq[URI]): Int = {
+ sa: SparkArgs,
+ extraJars: Seq[URI],
+ pioHome: String,
+ verbose: Boolean = false): Expected[(Process, () => Unit)] = {
// Return error for unsupported cases
val deployMode =
- argumentValue(ca.common.sparkPassThrough, "--deploy-mode").getOrElse("client")
+ argumentValue(sa.sparkPassThrough, "--deploy-mode").getOrElse("client")
val master =
- argumentValue(ca.common.sparkPassThrough, "--master").getOrElse("local")
+ argumentValue(sa.sparkPassThrough, "--master").getOrElse("local")
- (ca.common.scratchUri, deployMode, master) match {
+ (sa.scratchUri, deployMode, master) match {
case (Some(u), "client", m) if m != "yarn-cluster" =>
- error("--scratch-uri cannot be set when deploy mode is client")
- return 1
+ return logAndFail("--scratch-uri cannot be set when deploy mode is client")
case (_, "cluster", m) if m.startsWith("spark://") =>
- error("Using cluster deploy mode with Spark standalone cluster is not supported")
- return 1
+ return logAndFail(
+ "Using cluster deploy mode with Spark standalone cluster is not supported")
case _ => Unit
}
// Initialize HDFS API for scratch URI
- val fs = ca.common.scratchUri map { uri =>
+ val fs = sa.scratchUri map { uri =>
FileSystem.get(uri, new Configuration())
}
@@ -124,18 +131,18 @@ object Runner extends Logging {
).mkString(",")
// Location of Spark
- val sparkHome = ca.common.sparkHome.getOrElse(
+ val sparkHome = sa.sparkHome.getOrElse(
sys.env.getOrElse("SPARK_HOME", "."))
// Local path to PredictionIO assembly JAR
- val mainJar = handleScratchFile(
- fs,
- ca.common.scratchUri,
- console.Console.coreAssembly(ca.common.pioHome.get))
+ val mainJar = Common.coreAssembly(pioHome) fold(
+ errStr => return Left(errStr),
+ assembly => handleScratchFile(fs, sa.scratchUri, assembly)
+ )
// Extra JARs that are needed by the driver
val driverClassPathPrefix =
- argumentValue(ca.common.sparkPassThrough, "--driver-class-path") map { v =>
+ argumentValue(sa.sparkPassThrough, "--driver-class-path") map { v =>
Seq(v)
} getOrElse {
Nil
@@ -146,11 +153,11 @@ object Runner extends Logging {
// Extra files that are needed to be passed to --files
val extraFiles = WorkflowUtils.thirdPartyConfFiles map { f =>
- handleScratchFile(fs, ca.common.scratchUri, new File(f))
+ handleScratchFile(fs, sa.scratchUri, new File(f))
}
val deployedJars = extraJars map { j =>
- handleScratchFile(fs, ca.common.scratchUri, new File(j))
+ handleScratchFile(fs, sa.scratchUri, new File(j))
}
val sparkSubmitCommand =
@@ -174,7 +181,7 @@ object Runner extends Logging {
Nil
}
- val sparkSubmitKryo = if (ca.common.sparkKryo) {
+ val sparkSubmitKryo = if (sa.sparkKryo) {
Seq(
"--conf",
"spark.serializer=org.apache.spark.serializer.KryoSerializer")
@@ -182,33 +189,26 @@ object Runner extends Logging {
Nil
}
- val verbose = if (ca.common.verbose) Seq("--verbose") else Nil
+ val verboseArg = if (verbose) Seq("--verbose") else Nil
val sparkSubmit = Seq(
sparkSubmitCommand,
- ca.common.sparkPassThrough,
+ sa.sparkPassThrough,
Seq("--class", className),
sparkSubmitJars,
sparkSubmitFiles,
sparkSubmitExtraClasspaths,
sparkSubmitKryo,
Seq(mainJar),
- detectFilePaths(fs, ca.common.scratchUri, classArgs),
+ detectFilePaths(fs, sa.scratchUri, classArgs),
Seq("--env", pioEnvVars),
- verbose).flatten.filter(_ != "")
+ verboseArg).flatten.filter(_ != "")
info(s"Submission command: ${sparkSubmit.mkString(" ")}")
val proc = Process(
sparkSubmit,
None,
"CLASSPATH" -> "",
"SPARK_YARN_USER_ENV" -> pioEnvVars).run()
- Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
- def run(): Unit = {
- cleanup(fs, ca.common.scratchUri)
- proc.destroy()
- }
- }))
- cleanup(fs, ca.common.scratchUri)
- proc.exitValue()
+ Right((proc, () => cleanup(fs, sa.scratchUri)))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala b/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
index f53d84c..bbe39a5 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
@@ -133,7 +133,7 @@ case class AdminServerConfig(
)
object AdminServer {
- def createAdminServer(config: AdminServerConfig): Unit = {
+ def createAdminServer(config: AdminServerConfig): ActorSystem = {
implicit val system = ActorSystem("AdminServerSystem")
val commandClient = new CommandClient(
@@ -146,7 +146,7 @@ object AdminServer {
Props(classOf[AdminServerActor], commandClient),
"AdminServerActor")
serverActor ! StartServer(config.ip, config.port)
- system.awaitTermination
+ system
}
}
@@ -155,5 +155,6 @@ object AdminRun {
AdminServer.createAdminServer(AdminServerConfig(
ip = "localhost",
port = 7071))
+ .awaitTermination
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
new file mode 100644
index 0000000..715cf8f
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.tools
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.ReturnTypes._
+
+import grizzled.slf4j.Logging
+import scala.util.Either
+
+object AccessKey extends EitherLogging {
+
+ def create(
+ appName: String,
+ key: String,
+ events: Seq[String]): Expected[storage.AccessKey] = {
+
+ val apps = storage.Storage.getMetaDataApps
+ apps.getByName(appName) map { app =>
+ val accessKeys = storage.Storage.getMetaDataAccessKeys
+ val newKey = storage.AccessKey(
+ key = key,
+ appid = app.id,
+ events = events)
+ accessKeys.insert(newKey) map { k =>
+ info(s"Created new access key: ${k}")
+ Right(newKey.copy(key = k))
+ } getOrElse {
+ logAndFail(s"Unable to create new access key.")
+ }
+ } getOrElse {
+ logAndFail(s"App ${appName} does not exist. Aborting.")
+ }
+ }
+
+ def list(app: Option[String]): Expected[Seq[storage.AccessKey]] =
+ app map { appName =>
+ App.show(appName).right map { appChansPair => appChansPair._1.keys }
+ } getOrElse {
+ Right(storage.Storage.getMetaDataAccessKeys.getAll)
+ }
+
+ def delete(key: String): MaybeError = {
+ try {
+ storage.Storage.getMetaDataAccessKeys.delete(key)
+ logAndSucceed(s"Deleted access key ${key}.")
+ } catch {
+ case e: Exception =>
+ error(s"Error deleting access key ${key}.", e)
+ Left(s"Error deleting access key ${key}.")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
new file mode 100644
index 0000000..44fa667
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.data.storage.Channel
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.ReturnTypes._
+
+sealed case class AppDescription(
+ app: storage.App,
+ keys: Seq[storage.AccessKey])
+
+object App extends EitherLogging {
+
+ def create(
+ name: String,
+ id: Option[Int] = None,
+ description: Option[String] = None,
+ accessKey: String = "") : Expected[AppDescription] = {
+
+ val apps = storage.Storage.getMetaDataApps()
+ // get the client in the beginning so error exit right away if can't access client
+ val events = storage.Storage.getLEvents()
+ var errStr = ""
+
+ apps.getByName(name) map { app =>
+ errStr = s"App ${name} already exists. Aborting."
+ error(errStr)
+ errStr
+ } orElse {
+ id.flatMap { id =>
+ apps.get(id) map { app =>
+ errStr = s"App ID ${id} already exists and maps to the app '${app.name}'. " +
+ "Aborting."
+ error(errStr)
+ errStr
+ }
+ }
+ } map {err => Left(err)} getOrElse {
+ val newApp = storage.App(
+ id = id.getOrElse(0),
+ name = name,
+ description = description)
+ val appid = apps.insert(newApp)
+
+ appid map { id =>
+ val dbInit = events.init(id)
+ val r = if (dbInit) {
+ info(s"Initialized Event Store for this app ID: ${id}.")
+ val accessKeys = storage.Storage.getMetaDataAccessKeys
+ val newKey = storage.AccessKey(
+ key = accessKey,
+ appid = id,
+ events = Seq())
+ accessKeys.insert(newKey)
+ .map { k =>
+ Right(AppDescription(
+ app = newApp.copy(id = id),
+ keys = Seq(newKey.copy(key = k))))
+ } getOrElse {
+ logAndFail(s"Unable to create new access key.")
+ }
+ } else {
+ errStr = s"Unable to initialize Event Store for this app ID: ${id}."
+ try {
+ apps.delete(id)
+ } catch {
+ case e: Exception =>
+ errStr += s"""
+ |Failed to revert back the App meta-data change.
+ |The app ${name} CANNOT be used!
+ |Please run 'pio app delete ${name}' to delete this app!"""
+ }
+ logAndFail(errStr)
+ }
+ events.close()
+ r
+ } getOrElse {
+ logAndFail(s"Unable to create new app.")
+ }
+ }
+ }
+
+ def list: Seq[AppDescription] = {
+ val apps = storage.Storage.getMetaDataApps.getAll().sortBy(_.name)
+ val accessKeys = storage.Storage.getMetaDataAccessKeys
+
+ apps map { app =>
+ AppDescription(
+ app = app,
+ keys = accessKeys.getByAppid(app.id))
+ }
+ }
+
+ def show(appName: String): Expected[(AppDescription, Seq[Channel])] = {
+ val apps = storage.Storage.getMetaDataApps
+ val accessKeys = storage.Storage.getMetaDataAccessKeys
+ val channels = storage.Storage.getMetaDataChannels
+
+ apps.getByName(appName) map { app =>
+ Right(
+ (AppDescription(
+ app = app,
+ keys = accessKeys.getByAppid(app.id)),
+ channels.getByAppid(app.id))
+ )
+ } getOrElse {
+ logAndFail(s"App ${appName} does not exist. Aborting.")
+ }
+ }
+
+ def delete(name: String): MaybeError = {
+ val events = storage.Storage.getLEvents()
+ try {
+ show(name).right.flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+
+ val delChannelStatus: MaybeError =
+ channels.map { ch =>
+ if (events.remove(appDesc.app.id, Some(ch.id))) {
+ info(s"Removed Event Store of the channel ID: ${ch.id}")
+ try {
+ storage.Storage.getMetaDataChannels.delete(ch.id)
+ info(s"Deleted channel ${ch.name}")
+ None
+ } catch {
+ case e: Exception =>
+ val errStr = s"Error deleting channel ${ch.name}."
+ error(errStr, e)
+ Some(errStr)
+ }
+ } else {
+ val errStr = s"Error removing Event Store of the channel ID: ${ch.id}."
+ error(errStr)
+ Some(errStr)
+ }
+ }
+ .flatten
+ .reduceOption(_ + "\n" + _)
+ .map(Left(_)) getOrElse Success
+
+ if (delChannelStatus.isLeft) {
+ return delChannelStatus
+ }
+
+ try {
+ events.remove(appDesc.app.id)
+ info(s"Removed Event Store for this app ID: ${appDesc.app.id}")
+ } catch {
+ case e: Exception =>
+ logAndFail(s"Error removing Event Store for this app. Aborting.")
+ }
+
+ appDesc.keys foreach { key =>
+ try {
+ storage.Storage.getMetaDataAccessKeys.delete(key.key)
+ info(s"Removed access key ${key.key}")
+ } catch {
+ case e: Exception =>
+ logAndFail(s"Error removing access key ${key.key}. Aborting.")
+ }
+ }
+
+ try {
+ storage.Storage.getMetaDataApps.delete(appDesc.app.id)
+ info(s"Deleted app ${appDesc.app.name}.")
+ } catch {
+ case e: Exception =>
+ logAndFail(s"Error deleting app ${appDesc.app.name}. Aborting.")
+ }
+ logAndSucceed("Done.")
+ }
+
+ } finally {
+ events.close()
+ }
+ }
+
+ def dataDelete(
+ name: String,
+ channel: Option[String] = None,
+ all: Boolean = false): MaybeError = {
+
+ var errStr = ""
+ val events = storage.Storage.getLEvents()
+ try {
+ show(name).right.flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+
+ val chanIdsToRemove: Seq[Option[Int]] =
+ if (all) {
+ channels.map(ch => Some(ch.id)) :+ None // remove default channel too
+ } else {
+ channel.map { chName =>
+ channels.find(ch => ch.name == chName) match {
+ case None =>
+ return logAndFail(s"""Unable to delete data for channel.
+ |Channel ${chName} doesn't exist.""")
+ case Some(ch) => Seq(Some(ch.id))
+ }
+ } getOrElse {
+ Seq(None) // for default channel
+ }
+ }
+
+ chanIdsToRemove.map { chId: Option[Int] =>
+
+ val r1 = if (events.remove(appDesc.app.id, chId)) {
+ if (chId.isDefined) {
+ info(s"Removed Event Store for the channel ID: ${chId.get}")
+ } else {
+ info(s"Removed Event Store for the app ID: ${appDesc.app.id}")
+ }
+ None
+ } else {
+ errStr =
+ if (chId.isDefined) s"Error removing Event Store for the channel ID: ${chId.get}."
+ else s"Error removing Event Store for the app ID: ${appDesc.app.id}."
+ error(errStr)
+ Some(errStr)
+ }
+ // re-create table
+ val dbInit = events.init(appDesc.app.id, chId)
+ val r2 = if (dbInit) {
+ if (chId.isDefined) {
+ info(s"Initialized Event Store for the channel ID: ${chId.get}")
+ } else {
+ info(s"Initialized Event Store for the app ID: ${appDesc.app.id}")
+ }
+ None
+ } else {
+ errStr =
+ if (chId.isDefined) {
+ s"Unable to initialize Event Store for the channel ID: ${chId.get}."
+ }
+ else {
+ s"Unable to initialize Event tore for the app ID: ${appDesc.app.id}."
+ }
+ error(errStr)
+ Some(errStr)
+ }
+ Seq(r1, r2)
+ }
+ .flatten.flatten
+ .reduceOption(_ + "\n" + _)
+ .toLeft(Ok())
+ }
+ } finally {
+ events.close()
+ }
+ }
+
+ def channelNew(appName: String, newChannel: String): Expected[Channel] = {
+ val events = storage.Storage.getLEvents()
+ val chanStorage = storage.Storage.getMetaDataChannels
+ var errStr = ""
+ try {
+ show(appName).right flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+ if (channels.find(ch => ch.name == newChannel).isDefined) {
+ logAndFail(s"""Channel ${newChannel} already exists.
+ |Unable to create new channel.""")
+ } else if (!storage.Channel.isValidName(newChannel)) {
+ logAndFail(s"""Unable to create new channel.
+ |The channel name ${newChannel} is invalid.
+ |${storage.Channel.nameConstraint}""")
+ } else {
+
+ val channel = Channel(
+ id = 0,
+ appid = appDesc.app.id,
+ name = newChannel)
+
+ chanStorage.insert(channel) map { chanId =>
+
+ info(s"Updated Channel meta-data.")
+
+ // initialize storage
+ val dbInit = events.init(appDesc.app.id, Some(chanId))
+ if (dbInit) {
+ info(s"Initialized Event Store for the channel: ${newChannel}.")
+ info(s"Created new channel:")
+ info(s" Channel Name: ${newChannel}")
+ info(s" Channel ID: ${chanId}")
+ info(s" App ID: ${appDesc.app.id}")
+ Right(channel.copy(id = chanId))
+ } else {
+ errStr = s"""Unable to create new channel.
+ |Failed to initalize Event Store."""
+ error(errStr)
+ // reverted back the meta data
+ try {
+ chanStorage.delete(chanId)
+ Left(errStr)
+ } catch {
+ case e: Exception =>
+ val nextErrStr = s"""
+ |Failed to revert back the Channel meta-data change.
+ |The channel ${newChannel} CANNOT be used!
+ |Please run 'pio app channel-delete ${appName} ${newChannel}'""" +
+ " to delete this channel!"
+ logAndFail(errStr + nextErrStr)
+ }
+ }
+ } getOrElse {
+ logAndFail(s"""Unable to create new channel.
+ |Failed to update Channel meta-data.""")
+ }
+ }
+ }
+ } finally {
+ events.close()
+ }
+ }
+
+ def channelDelete(appName: String, deleteChannel: String): MaybeError = {
+ val chanStorage = storage.Storage.getMetaDataChannels
+ val events = storage.Storage.getLEvents()
+ var errStr = ""
+ try {
+ show(appName).right.flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+ val foundChannel = channels.find(ch => ch.name == deleteChannel)
+ if (foundChannel.isEmpty) {
+ logAndFail(s"""Unable to delete channel
+ |Channel ${deleteChannel} doesn't exists.""")
+ } else {
+ val chId = foundChannel.get.id
+ val dbRemoved = events.remove(appDesc.app.id, Some(chId))
+ if (dbRemoved) {
+ info(s"Removed Event Store for this channel: ${deleteChannel}")
+ try {
+ chanStorage.delete(chId)
+ logAndSucceed(s"Deleted channel: ${deleteChannel}.")
+ } catch {
+ case e: Exception =>
+ logAndFail(s"""Unable to delete channel.
+ |Failed to update Channel meta-data.
+ |The channel ${deleteChannel} CANNOT be used!
+ |Please run 'pio app channel-delete ${appDesc.app.name} ${deleteChannel}'""" +
+ " to delete this channel again!")
+ }
+ } else {
+ logAndFail(s"""Unable to delete channel.
+ |Error removing Event Store for this channel.""")
+ }
+ }
+ }
+ } finally {
+ events.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
new file mode 100644
index 0000000..6fd8977
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.controller.Utils
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.data.storage.EngineManifest
+import org.apache.predictionio.data.storage.EngineManifestSerializer
+import org.apache.predictionio.tools.RegisterEngine
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.{RunWorkflow, RunServer}
+import org.apache.predictionio.tools.{DeployArgs, WorkflowArgs, SparkArgs, ServerArgs}
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.tools.Common._
+import org.apache.predictionio.workflow.WorkflowUtils
+
+import org.apache.commons.io.FileUtils
+import org.json4s.native.Serialization.read
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+import scala.io.Source
+import scala.util.Random
+import scala.collection.JavaConversions._
+import scala.sys.process._
+import scalaj.http.Http
+import java.io.File
+
+case class BuildArgs(
+ sbt: Option[File] = None,
+ sbtExtra: Option[String] = None,
+ sbtAssemblyPackageDependency: Boolean = true,
+ sbtClean: Boolean = false,
+ uberJar: Boolean = false,
+ forceGeneratePIOSbt: Boolean = false)
+
+case class EngineArgs(
+ manifestJson: File = new File("manifest.json"),
+ engineId: Option[String] = None,
+ engineVersion: Option[String] = None)
+
+object Engine extends EitherLogging {
+
+ private val manifestAutogenTag = "pio-autogen-manifest"
+
+ private def readManifestJson(json: File): Expected[EngineManifest] = {
+ implicit val formats = Utils.json4sDefaultFormats +
+ new EngineManifestSerializer
+ try {
+ Right(read[EngineManifest](Source.fromFile(json).mkString))
+ } catch {
+ case e: java.io.FileNotFoundException =>
+ logAndFail(s"${json.getCanonicalPath} does not exist. Aborting.")
+ case e: MappingException =>
+ logAndFail(s"${json.getCanonicalPath} has invalid content: " +
+ e.getMessage)
+ }
+ }
+
+ private def withRegisteredManifest[T](ea: EngineArgs)(
+ op: EngineManifest => Expected[T]): Expected[T] = {
+ val res: Expected[Expected[T]] = for {
+ ej <- readManifestJson(ea.manifestJson).right
+ id <- Right(ea.engineId getOrElse ej.id).right
+ version <- Right(ea.engineVersion getOrElse ej.version).right
+ manifest <- storage.Storage.getMetaDataEngineManifests.get(id, version)
+ .toRight {
+ val errStr =
+ s"""Engine ${id} ${version} cannot be found in the system.")
+ |Possible reasons:
+ |- the engine is not yet built by the 'build' command;
+ |- the meta data store is offline."""
+ error(errStr)
+ errStr
+ }.right
+ } yield {
+ op(manifest)
+ }
+ res.joinRight
+ }
+
+ private def generateManifestJson(json: File): MaybeError = {
+ val cwd = sys.props("user.dir")
+ implicit val formats = Utils.json4sDefaultFormats +
+ new EngineManifestSerializer
+ val rand = Random.alphanumeric.take(32).mkString
+ val ha = java.security.MessageDigest.getInstance("SHA-1").
+ digest(cwd.getBytes).map("%02x".format(_)).mkString
+ val em = EngineManifest(
+ id = rand,
+ version = ha,
+ name = new File(cwd).getName,
+ description = Some(manifestAutogenTag),
+ files = Seq(),
+ engineFactory = "")
+ try {
+ FileUtils.writeStringToFile(json, write(em), "ISO-8859-1")
+ Success
+ } catch {
+ case e: java.io.IOException =>
+ logAndFail(s"Cannot generate ${json} automatically (${e.getMessage}). " +
+ "Aborting.")
+ }
+ }
+
+ private def regenerateManifestJson(json: File): MaybeError = {
+ val cwd = sys.props("user.dir")
+ val ha = java.security.MessageDigest.getInstance("SHA-1").
+ digest(cwd.getBytes).map("%02x".format(_)).mkString
+ if (json.exists) {
+ readManifestJson(json).right.flatMap { em =>
+ if (em.description == Some(manifestAutogenTag) && ha != em.version) {
+ warn("This engine project directory contains an auto-generated " +
+ "manifest that has been copied/moved from another location. ")
+ warn("Regenerating the manifest to reflect the updated location. " +
+ "This will dissociate with all previous engine instances.")
+ generateManifestJson(json)
+ } else {
+ logAndSucceed(s"Using existing engine manifest JSON at " +
+ "${json.getCanonicalPath}")
+ }
+ }
+ } else {
+ generateManifestJson(json)
+ }
+ }
+
+ private def detectSbt(sbt: Option[File], pioHome: String): String = {
+ sbt map {
+ _.getCanonicalPath
+ } getOrElse {
+ val f = new File(Seq(pioHome, "sbt", "sbt").mkString(File.separator))
+ if (f.exists) f.getCanonicalPath else "sbt"
+ }
+ }
+
+ private def outputSbtError(line: String): Unit = {
+ """\[.*error.*\]""".r findFirstIn line foreach { _ => error(line) }
+ }
+
+ private def compile(
+ buildArgs: BuildArgs, pioHome: String, verbose: Boolean): MaybeError = {
+ // only add pioVersion to sbt if project/pio.sbt exists
+ if (new File("project", "pio-build.sbt").exists || buildArgs.forceGeneratePIOSbt) {
+ FileUtils.writeLines(
+ new File("pio.sbt"),
+ Seq(
+ "// Generated automatically by pio build.",
+ "// Changes in this file will be overridden.",
+ "",
+ "pioVersion := \"" + BuildInfo.version + "\""))
+ }
+ implicit val formats = Utils.json4sDefaultFormats
+
+ val sbt = detectSbt(buildArgs.sbt, pioHome)
+ info(s"Using command '${sbt}' at the current working directory to build.")
+ info("If the path above is incorrect, this process will fail.")
+ val asm =
+ if (buildArgs.sbtAssemblyPackageDependency) {
+ " assemblyPackageDependency"
+ } else {
+ ""
+ }
+ val clean = if (buildArgs.sbtClean) " clean" else ""
+ val buildCmd = s"${sbt} ${buildArgs.sbtExtra.getOrElse("")}${clean} " +
+ (if (buildArgs.uberJar) "assembly" else s"package${asm}")
+ val core = new File(s"pio-assembly-${BuildInfo.version}.jar")
+ if (buildArgs.uberJar) {
+ info(s"Uber JAR enabled. Putting ${core.getName} in lib.")
+ val dst = new File("lib")
+ dst.mkdir()
+ coreAssembly(pioHome) match {
+ case Right(coreFile) =>
+ FileUtils.copyFileToDirectory(
+ coreFile,
+ dst,
+ true)
+ case Left(errStr) => return Left(errStr)
+ }
+ } else {
+ if (new File("engine.json").exists()) {
+ info(s"Uber JAR disabled. Making sure lib/${core.getName} is absent.")
+ new File("lib", core.getName).delete()
+ } else {
+ info("Uber JAR disabled, but current working directory does not look " +
+ s"like an engine project directory. Please delete lib/${core.getName} manually.")
+ }
+ }
+ info(s"Going to run: ${buildCmd}")
+ try {
+ val r =
+ if (verbose) {
+ buildCmd.!(ProcessLogger(line => info(line), line => error(line)))
+ } else {
+ buildCmd.!(ProcessLogger(
+ line => outputSbtError(line),
+ line => outputSbtError(line)))
+ }
+ if (r != 0) {
+ logAndFail(s"Return code of build command: ${buildCmd} is ${r}. Aborting.")
+ } else {
+ logAndSucceed("Compilation finished successfully.")
+ }
+ } catch {
+ case e: java.io.IOException =>
+ logAndFail(s"Exception during compilation: ${e.getMessage}")
+ }
+ }
+
+ def build(
+ buildArgs: BuildArgs,
+ pioHome: String,
+ manifestJson: File,
+ verbose: Boolean): MaybeError = {
+
+ regenerateManifestJson(manifestJson) match {
+ case Left(err) => return Left(err)
+ case _ => Unit
+ }
+
+ Template.verifyTemplateMinVersion(new File("template.json")) match {
+ case Left(err) => return Left(err)
+ case Right(_) =>
+ compile(buildArgs, pioHome, verbose)
+ info("Looking for an engine...")
+ val jarFiles = jarFilesForScala
+ if (jarFiles.isEmpty) {
+ return logAndFail("No engine found. Your build might have failed. Aborting.")
+ }
+ jarFiles foreach { f => info(s"Found ${f.getName}")}
+ RegisterEngine.registerEngine(
+ manifestJson,
+ jarFiles,
+ false)
+ }
+ }
+
+ /** Training an engine.
+ * The function starts a training process to bu run concurrenlty.
+ *
+ * @param ea An instance of [[EngineArgs]]
+ * @param wa An instance of [[WorkflowArgs]] for running a single training.
+ * @param sa An instance of [[SparkArgs]]
+ * @param pioHome [[String]] with a path to PIO installation
+ * @param verbose A [[Boolean]]
+ * @return An instance of [[Expected]] contaning either [[Left]]
+ * with an error message or [[Right]] with a handle to a process
+ * responsible for training and a function () => Unit,
+ * that must be called when the process is complete
+ */
+ def train(
+ ea: EngineArgs,
+ wa: WorkflowArgs,
+ sa: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
+ regenerateManifestJson(ea.manifestJson) match {
+ case Left(err) => return Left(err)
+ case _ => Unit
+ }
+
+ Template.verifyTemplateMinVersion(new File("template.json")).right.flatMap {
+ _ =>
+ withRegisteredManifest(ea) { em =>
+ RunWorkflow.runWorkflow(wa, sa, em, pioHome, verbose)
+ }
+ }
+ }
+
+ /** Deploying an engine.
+ * The function starts a new process to be run concerrently.
+ *
+ * @param ea An instance of [[EngineArgs]]
+ * @param engineInstanceId An instance of [[engineInstanceId]]
+ * @param serverArgs An instance of [[ServerArgs]]
+ * @param sparkArgs An instance of [[SparkArgs]]
+ * @param pioHome [[String]] with a path to PIO installation
+ * @param verbose A [[Boolean]]
+ * @return An instance of [[Expected]] contaning either [[Left]]
+ * with an error message or [[Right]] with a handle to process
+ * of a running angine and a function () => Unit,
+ * that must be called when the process is complete
+ */
+ def deploy(
+ ea: EngineArgs,
+ engineInstanceId: Option[String],
+ serverArgs: ServerArgs,
+ sparkArgs: SparkArgs,
+ pioHome: String,
+ verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
+ val verifyResult = Template.verifyTemplateMinVersion(new File("template.json"))
+ if (verifyResult.isLeft) {
+ return Left(verifyResult.left.get)
+ }
+ withRegisteredManifest(ea) { em =>
+ val variantJson = parse(Source.fromFile(serverArgs.variantJson).mkString)
+ val variantId = variantJson \ "id" match {
+ case JString(s) => s
+ case _ =>
+ return logAndFail("Unable to read engine variant ID from " +
+ s"${serverArgs.variantJson.getCanonicalPath}. Aborting.")
+ }
+ val engineInstances = storage.Storage.getMetaDataEngineInstances
+ val engineInstance = engineInstanceId map { eid =>
+ engineInstances.get(eid)
+ } getOrElse {
+ engineInstances.getLatestCompleted(em.id, em.version, variantId)
+ }
+ engineInstance map { r =>
+ RunServer.runServer(r.id, serverArgs, sparkArgs, em, pioHome, verbose)
+ } getOrElse {
+ engineInstanceId map { eid =>
+ logAndFail(s"Invalid engine instance ID ${eid}. Aborting.")
+ } getOrElse {
+ logAndFail(s"No valid engine instance found for engine ${em.id} " +
+ s"${em.version}.\nTry running 'train' before 'deploy'. Aborting.")
+ }
+ }
+ }
+ }
+
+ def undeploy(da: DeployArgs): MaybeError = {
+
+ val serverUrl = s"http://${da.ip}:${da.port}"
+ info(
+ s"Undeploying any existing engine instance at ${serverUrl}")
+ try {
+ val code = Http(s"${serverUrl}/stop").asString.code
+ code match {
+ case 200 => Success
+ case 404 =>
+ logAndFail(s"Another process is using ${serverUrl}. Unable to undeploy.")
+ case _ =>
+ logAndFail(s"Another process is using ${serverUrl}, or an existing " +
+ s"engine server is not responding properly (HTTP ${code}). " +
+ "Unable to undeploy.")
+ }
+ } catch {
+ case e: java.net.ConnectException =>
+ logAndFail(s"Nothing at ${serverUrl}")
+ case _: Throwable =>
+ logAndFail("Another process might be occupying " +
+ s"${da.ip}:${da.port}. Unable to undeploy.")
+ }
+ }
+
+ /** Running a driver on spark.
+ * The function starts a process and returns immediately
+ *
+ * @param mainClass A [[String]] with the class containing a main functionto run
+ * @param driverArguments Arguments to be passed to the main function
+ * @param manifestJson An instance of [[File]] for running a single training.
+ * @param buildArgs An instance of [[BuildArgs]]
+ * @param sparkArgs an instance of [[SparkArgs]]
+ * @param pioHome [[String]] with a path to PIO installation
+ * @param verbose A [[Boolean]]
+ * @return An instance of [[Expected]] contaning either [[Left]]
+ * with an error message or [[Right]] with a handle to a process
+ * of a running driver
+ */
+ def run(
+ mainClass: String,
+ driverArguments: Seq[String],
+ manifestJson: File,
+ buildArgs: BuildArgs,
+ sparkArgs: SparkArgs,
+ pioHome: String,
+ verbose: Boolean): Expected[Process] = {
+
+ generateManifestJson(manifestJson) match {
+ case Left(err) => return Left(err)
+ case _ => Unit
+ }
+
+ compile(buildArgs, pioHome, verbose)
+
+ val extraFiles = WorkflowUtils.thirdPartyConfFiles
+
+ val jarFiles = jarFilesForScala
+ jarFiles foreach { f => info(s"Found JAR: ${f.getName}") }
+ val allJarFiles = jarFiles.map(_.getCanonicalPath)
+ val cmd = s"${getSparkHome(sparkArgs.sparkHome)}/bin/spark-submit --jars " +
+ s"${allJarFiles.mkString(",")} " +
+ (if (extraFiles.size > 0) {
+ s"--files ${extraFiles.mkString(",")} "
+ } else {
+ ""
+ }) +
+ "--class " +
+ s"${mainClass} ${sparkArgs.sparkPassThrough.mkString(" ")} " +
+ coreAssembly(pioHome) + " " +
+ driverArguments.mkString(" ")
+ info(s"Submission command: ${cmd}")
+ Right(Process(
+ cmd,
+ None,
+ "SPARK_YARN_USER_ENV" -> sys.env.filter(kv => kv._1.startsWith("PIO_")).
+ map(kv => s"${kv._1}=${kv._2}").mkString(",")).run())
+ }
+
+ def unregister(jsonManifest: File): MaybeError = {
+ RegisterEngine.unregisterEngine(jsonManifest)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala
new file mode 100644
index 0000000..ed6b487
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.tools.Runner
+import org.apache.predictionio.tools.SparkArgs
+import org.apache.predictionio.tools.ReturnTypes._
+
+import scala.sys.process._
+
+case class ExportArgs(
+ appId: Int = 0,
+ channel: Option[String] = None,
+ outputPath: String = "",
+ format: String = "json")
+
+object Export {
+ def eventsToFile(
+ ea: ExportArgs,
+ sa: SparkArgs,
+ pioHome: String): Expected[(Process, () => Unit)] = {
+
+ val channelArg = ea.channel
+ .map(ch => Seq("--channel", ch)).getOrElse(Nil)
+ Runner.runOnSpark(
+ "org.apache.predictionio.tools.export.EventsToFile",
+ Seq(
+ "--appid",
+ ea.appId.toString,
+ "--output",
+ ea.outputPath,
+ "--format",
+ ea.format) ++ channelArg,
+ sa,
+ Nil,
+ pioHome)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala
new file mode 100644
index 0000000..9fac559
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.tools.Runner
+import org.apache.predictionio.tools.SparkArgs
+import org.apache.predictionio.tools.ReturnTypes._
+
+import scala.sys.process._
+
+case class ImportArgs(
+ appId: Int = 0,
+ channel: Option[String] = None,
+ inputPath: String = "")
+
+object Import {
+ def fileToEvents(
+ ia: ImportArgs,
+ sa: SparkArgs,
+ pioHome: String): Expected[(Process, () => Unit)] = {
+
+ val channelArg = ia.channel
+ .map(ch => Seq("--channel", ch)).getOrElse(Nil)
+ Runner.runOnSpark(
+ "org.apache.predictionio.tools.imprt.FileToEvents",
+ Seq(
+ "--appid",
+ ia.appId.toString,
+ "--input",
+ ia.inputPath) ++ channelArg,
+ sa,
+ Nil,
+ pioHome)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
new file mode 100644
index 0000000..10aca41
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.data.api.EventServer
+import org.apache.predictionio.data.api.EventServerConfig
+import org.apache.predictionio.tools.EventServerArgs
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.Common
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.tools.dashboard.Dashboard
+import org.apache.predictionio.tools.dashboard.DashboardConfig
+import org.apache.predictionio.tools.admin.AdminServer
+import org.apache.predictionio.tools.admin.AdminServerConfig
+
+import akka.actor.ActorSystem
+import java.io.File
+import scala.io.Source
+import semverfi._
+
+case class DashboardArgs(
+ ip: String = "127.0.0.1",
+ port: Int = 9000)
+
+case class AdminServerArgs(
+ ip: String = "127.0.0.1",
+ port: Int = 7071)
+
+case class PioStatus(
+ version: String = "",
+ pioHome: String = "",
+ sparkHome: String = "",
+ sparkVersion: String = "",
+ sparkMinVersion: String = "",
+ warnings: Seq[String] = Seq())
+
+object Management extends EitherLogging {
+
+ def version(): String = BuildInfo.version
+
+ /** Starts a dashboard server and returns immediately
+ *
+ * @param da An instance of [[DashboardArgs]]
+ * @return An instance of [[ActorSystem]] in which the server is being executed
+ */
+ def dashboard(da: DashboardArgs): ActorSystem = {
+ info(s"Creating dashboard at ${da.ip}:${da.port}")
+ Dashboard.createDashboard(DashboardConfig(
+ ip = da.ip,
+ port = da.port))
+ }
+
+ /** Starts an eventserver server and returns immediately
+ *
+ * @param ea An instance of [[EventServerArgs]]
+ * @return An instance of [[ActorSystem]] in which the server is being executed
+ */
+ def eventserver(ea: EventServerArgs): ActorSystem = {
+ info(s"Creating Event Server at ${ea.ip}:${ea.port}")
+ EventServer.createEventServer(EventServerConfig(
+ ip = ea.ip,
+ port = ea.port,
+ stats = ea.stats))
+ }
+
+ /** Starts an adminserver server and returns immediately
+ *
+ * @param aa An instance of [[AdminServerArgs]]
+ * @return An instance of [[ActorSystem]] in which the server is being executed
+ */
+ def adminserver(aa: AdminServerArgs): ActorSystem = {
+ info(s"Creating Admin Server at ${aa.ip}:${aa.port}")
+ AdminServer.createAdminServer(AdminServerConfig(
+ ip = aa.ip,
+ port = aa.port
+ ))
+ }
+
+ private def stripMarginAndNewlines(string: String): String =
+ string.stripMargin.replaceAll("\n", " ")
+
+ def status(pioHome: Option[String], sparkHome: Option[String]): Expected[PioStatus] = {
+ var pioStatus = PioStatus()
+ info("Inspecting PredictionIO...")
+ pioHome map { pioHome =>
+ info(s"PredictionIO ${BuildInfo.version} is installed at $pioHome")
+ pioStatus = pioStatus.copy(version = version(), pioHome = pioHome)
+ } getOrElse {
+ return logAndFail("Unable to locate PredictionIO installation. Aborting.")
+ }
+ info("Inspecting Apache Spark...")
+ val sparkHomePath = Common.getSparkHome(sparkHome)
+ if (new File(s"$sparkHomePath/bin/spark-submit").exists) {
+ info(s"Apache Spark is installed at $sparkHome")
+ val sparkMinVersion = "1.3.0"
+ pioStatus = pioStatus.copy(
+ sparkHome = sparkHomePath,
+ sparkMinVersion = sparkMinVersion)
+ val sparkReleaseFile = new File(s"$sparkHomePath/RELEASE")
+ if (sparkReleaseFile.exists) {
+ val sparkReleaseStrings =
+ Source.fromFile(sparkReleaseFile).mkString.split(' ')
+ if (sparkReleaseStrings.length < 2) {
+ val warning = (stripMarginAndNewlines(
+ s"""|Apache Spark version information cannot be found (RELEASE file
+ |is empty). This is a known issue for certain vendors (e.g.
+ |Cloudera). Please make sure you are using a version of at least
+ |$sparkMinVersion."""))
+ warn(warning)
+ pioStatus = pioStatus.copy(warnings = pioStatus.warnings :+ warning)
+ } else {
+ val sparkReleaseVersion = sparkReleaseStrings(1)
+ val parsedMinVersion = Version.apply(sparkMinVersion)
+ val parsedCurrentVersion = Version.apply(sparkReleaseVersion)
+ if (parsedCurrentVersion >= parsedMinVersion) {
+ info(stripMarginAndNewlines(
+ s"""|Apache Spark $sparkReleaseVersion detected (meets minimum
+ |requirement of $sparkMinVersion)"""))
+ pioStatus = pioStatus.copy(sparkVersion = sparkReleaseVersion)
+ } else {
+ return logAndFail(stripMarginAndNewlines(
+ s"""|Apache Spark $sparkReleaseVersion detected (does not meet
+ |minimum requirement. Aborting."""))
+ }
+ }
+ } else {
+ val warning = (stripMarginAndNewlines(
+ s"""|Apache Spark version information cannot be found. If you are
+ |using a developmental tree, please make sure you are using a
+ |version of at least $sparkMinVersion."""))
+ warn(warning)
+ pioStatus = pioStatus.copy(warnings = pioStatus.warnings :+ warning)
+ }
+ } else {
+ return logAndFail("Unable to locate a proper Apache Spark installation. Aborting.")
+ }
+ info("Inspecting storage backend connections...")
+ try {
+ storage.Storage.verifyAllDataObjects()
+ } catch {
+ case e: Throwable =>
+ val errStr = s"""Unable to connect to all storage backends successfully.
+ |The following shows the error message from the storage backend.
+ |${e.getMessage} (${e.getClass.getName})", e)
+ |Dumping configuration of initialized storage backend sources.
+ |"Please make sure they are correct.
+ |"""
+ val sources = storage.Storage.config.get("sources") map { src =>
+ src map { case (s, p) =>
+ s"Source Name: $s; Type: ${p.getOrElse("type", "(error)")}; " +
+ s"Configuration: ${p.getOrElse("config", "(error)")}"
+ } mkString("\n")
+ } getOrElse {
+ "No properly configured storage backend sources."
+ }
+ return logAndFail(errStr + sources)
+ }
+ info("Your system is all ready to go.")
+ Right(pioStatus)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
new file mode 100644
index 0000000..1476598
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import java.io.File
+
+import scala.io.Source
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.ReturnTypes._
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.{write, read}
+import semverfi._
+
+case class TemplateMetaData(
+ pioVersionMin: Option[String] = None)
+
+object Template extends EitherLogging {
+
+ def templateMetaData(templateJson: File): TemplateMetaData = {
+ if (!templateJson.exists) {
+ warn(s"$templateJson does not exist. Template metadata will not be available. " +
+ "(This is safe to ignore if you are not working on a template.)")
+ TemplateMetaData()
+ } else {
+ val jsonString = Source.fromFile(templateJson)(scala.io.Codec.ISO8859).mkString
+ val json = try {
+ parse(jsonString)
+ } catch {
+ case e: org.json4s.ParserUtil.ParseException =>
+ warn(s"$templateJson cannot be parsed. Template metadata will not be available.")
+ return TemplateMetaData()
+ }
+ val pioVersionMin = json \ "pio" \ "version" \ "min"
+ pioVersionMin match {
+ case JString(s) => TemplateMetaData(pioVersionMin = Some(s))
+ case _ => TemplateMetaData()
+ }
+ }
+ }
+
+ def verifyTemplateMinVersion(templateJsonFile: File): MaybeError = {
+ val metadata = templateMetaData(templateJsonFile)
+
+ for (pvm <- metadata.pioVersionMin) {
+ if (Version(BuildInfo.version) < Version(pvm)) {
+ return logAndFail(s"This engine template requires at least PredictionIO $pvm. " +
+ s"The template may not work with PredictionIO ${BuildInfo.version}.")
+ }
+ }
+ Success
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala
deleted file mode 100644
index fcb9608..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.data.storage
-
-import grizzled.slf4j.Logging
-
-case class AccessKeyArgs(
- accessKey: String = "",
- events: Seq[String] = Seq())
-
-object AccessKey extends Logging {
- def create(ca: ConsoleArgs): Int = {
- val apps = storage.Storage.getMetaDataApps
- apps.getByName(ca.app.name) map { app =>
- val accessKeys = storage.Storage.getMetaDataAccessKeys
- val accessKey = accessKeys.insert(storage.AccessKey(
- key = ca.accessKey.accessKey,
- appid = app.id,
- events = ca.accessKey.events))
- accessKey map { k =>
- info(s"Created new access key: ${k}")
- 0
- } getOrElse {
- error(s"Unable to create new access key.")
- 1
- }
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- 1
- }
- }
-
- def list(ca: ConsoleArgs): Int = {
- val keys =
- if (ca.app.name == "") {
- storage.Storage.getMetaDataAccessKeys.getAll
- } else {
- val apps = storage.Storage.getMetaDataApps
- apps.getByName(ca.app.name) map { app =>
- storage.Storage.getMetaDataAccessKeys.getByAppid(app.id)
- } getOrElse {
- error(s"App ${ca.app.name} does not exist. Aborting.")
- return 1
- }
- }
- val title = "Access Key(s)"
- info(f"$title%64s | App ID | Allowed Event(s)")
- keys.sortBy(k => k.appid) foreach { k =>
- val events =
- if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
- info(f"${k.key}%64s | ${k.appid}%6d | $events%s")
- }
- info(s"Finished listing ${keys.size} access key(s).")
- 0
- }
-
- def delete(ca: ConsoleArgs): Int = {
- try {
- storage.Storage.getMetaDataAccessKeys.delete(ca.accessKey.accessKey)
- info(s"Deleted access key ${ca.accessKey.accessKey}.")
- 0
- } catch {
- case e: Exception =>
- error(s"Error deleting access key ${ca.accessKey.accessKey}.", e)
- 1
- }
- }
-}