You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2016/11/01 17:56:19 UTC

[1/3] incubator-predictionio git commit: [PIO-28] Console refactor

Repository: incubator-predictionio
Updated Branches:
  refs/heads/develop f7c30d37b -> e4a3c0c9f


http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
----------------------------------------------------------------------
diff --git a/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt b/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
index 48ea23e..e660116 100644
--- a/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
+++ b/tools/src/main/twirl/org/apache/predictionio/tools/console/upgrade.scala.txt
@@ -1,15 +1,3 @@
 Usage: pio upgrade <from version> <to version> <old App ID> <new app ID>
 
-Migrate the event data from old App ID to new App ID which can be used with <to version> of PredictionIO.
-
-  <from version>
-    The version upgraded from.
-
-  <to version>
-    The version upgraded to.
-
-  <old App ID>
-    Old App ID.
-
-  <new app ID>
-    Mew App ID.
+No longer supported!


[2/3] incubator-predictionio git commit: [PIO-28] Console refactor

Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala
deleted file mode 100644
index 53b1b40..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/App.scala
+++ /dev/null
@@ -1,540 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.data.storage
-
-import grizzled.slf4j.Logging
-
-case class AppArgs(
-  id: Option[Int] = None,
-  name: String = "",
-  channel: String = "",
-  dataDeleteChannel: Option[String] = None,
-  all: Boolean = false,
-  force: Boolean = false,
-  description: Option[String] = None)
-
-object App extends Logging {
-  def create(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps()
-    // get the client in the beginning so error exit right away if can't access client
-    val events = storage.Storage.getLEvents()
-    apps.getByName(ca.app.name) map { app =>
-      error(s"App ${ca.app.name} already exists. Aborting.")
-      1
-    } getOrElse {
-      ca.app.id.map { id =>
-        apps.get(id) map { app =>
-          error(
-            s"App ID ${id} already exists and maps to the app '${app.name}'. " +
-            "Aborting.")
-          return 1
-        }
-      }
-      val appid = apps.insert(storage.App(
-        id = ca.app.id.getOrElse(0),
-        name = ca.app.name,
-        description = ca.app.description))
-      appid map { id =>
-        val dbInit = events.init(id)
-        val r = if (dbInit) {
-          info(s"Initialized Event Store for this app ID: ${id}.")
-          val accessKeys = storage.Storage.getMetaDataAccessKeys
-          val accessKey = accessKeys.insert(storage.AccessKey(
-            key = ca.accessKey.accessKey,
-            appid = id,
-            events = Seq()))
-          accessKey map { k =>
-            info("Created new app:")
-            info(s"      Name: ${ca.app.name}")
-            info(s"        ID: ${id}")
-            info(s"Access Key: ${k}")
-            0
-          } getOrElse {
-            error(s"Unable to create new access key.")
-            1
-          }
-        } else {
-          error(s"Unable to initialize Event Store for this app ID: ${id}.")
-          // revert back the meta data change
-          try {
-            apps.delete(id)
-            0
-          } catch {
-            case e: Exception =>
-              error(s"Failed to revert back the App meta-data change.", e)
-              error(s"The app ${ca.app.name} CANNOT be used!")
-              error(s"Please run 'pio app delete ${ca.app.name}' " +
-                "to delete this app!")
-              1
-          }
-        }
-        events.close()
-        r
-      } getOrElse {
-        error(s"Unable to create new app.")
-        1
-      }
-    }
-  }
-
-  def list(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps.getAll().sortBy(_.name)
-    val accessKeys = storage.Storage.getMetaDataAccessKeys
-    val title = "Name"
-    val ak = "Access Key"
-    info(f"$title%20s |   ID | $ak%64s | Allowed Event(s)")
-    apps foreach { app =>
-      val keys = accessKeys.getByAppid(app.id)
-      keys foreach { k =>
-        val events =
-          if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
-        info(f"${app.name}%20s | ${app.id}%4d | ${k.key}%64s | $events%s")
-      }
-    }
-    info(s"Finished listing ${apps.size} app(s).")
-    0
-  }
-
-  def show(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps
-    val accessKeys = storage.Storage.getMetaDataAccessKeys
-    val channels = storage.Storage.getMetaDataChannels
-    apps.getByName(ca.app.name) map { app =>
-      info(s"    App Name: ${app.name}")
-      info(s"      App ID: ${app.id}")
-      info(s" Description: ${app.description.getOrElse("")}")
-      val keys = accessKeys.getByAppid(app.id)
-
-      var firstKey = true
-      keys foreach { k =>
-        val events =
-          if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
-        if (firstKey) {
-          info(f"  Access Key: ${k.key}%s | ${events}%s")
-          firstKey = false
-        } else {
-          info(f"              ${k.key}%s | ${events}%s")
-        }
-      }
-
-      val chans = channels.getByAppid(app.id)
-      var firstChan = true
-      val titleName = "Channel Name"
-      val titleID = "Channel ID"
-      chans.foreach { ch =>
-        if (firstChan) {
-          info(f"    Channels: ${titleName}%16s | ${titleID}%10s ")
-          firstChan = false
-        }
-        info(f"              ${ch.name}%16s | ${ch.id}%10s")
-      }
-      0
-    } getOrElse {
-      error(s"App ${ca.app.name} does not exist. Aborting.")
-      1
-    }
-  }
-
-  def delete(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps
-    val accesskeys = storage.Storage.getMetaDataAccessKeys
-    val channels = storage.Storage.getMetaDataChannels
-    val events = storage.Storage.getLEvents()
-    val status = apps.getByName(ca.app.name) map { app =>
-      info(s"The following app (including all channels) will be deleted. Are you sure?")
-      info(s"    App Name: ${app.name}")
-      info(s"      App ID: ${app.id}")
-      info(s" Description: ${app.description.getOrElse("")}")
-      val chans = channels.getByAppid(app.id)
-      var firstChan = true
-      val titleName = "Channel Name"
-      val titleID = "Channel ID"
-      chans.foreach { ch =>
-        if (firstChan) {
-          info(f"    Channels: ${titleName}%16s | ${titleID}%10s ")
-          firstChan = false
-        }
-        info(f"              ${ch.name}%16s | ${ch.id}%10s")
-      }
-
-      val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
-      choice match {
-        case "YES" => {
-          // delete channels
-          val delChannelStatus: Seq[Int] = chans.map { ch =>
-            if (events.remove(app.id, Some(ch.id))) {
-              info(s"Removed Event Store of the channel ID: ${ch.id}")
-              try {
-                channels.delete(ch.id)
-                info(s"Deleted channel ${ch.name}")
-                0
-              } catch {
-                case e: Exception =>
-                  error(s"Error deleting channel ${ch.name}.", e)
-                  1
-              }
-            } else {
-              error(s"Error removing Event Store of the channel ID: ${ch.id}.")
-              return 1
-            }
-          }
-
-          if (delChannelStatus.exists(_ != 0)) {
-            error("Error occurred while deleting channels. Aborting.")
-            return 1
-          }
-
-          try {
-            events.remove(app.id)
-            info(s"Removed Event Store for this app ID: ${app.id}")
-          } catch {
-            case e: Exception =>
-              error(s"Error removing Event Store for this app. Aborting.", e)
-              return 1
-          }
-
-          accesskeys.getByAppid(app.id) foreach { key =>
-            try {
-              accesskeys.delete(key.key)
-              info(s"Removed access key ${key.key}")
-            } catch {
-              case e: Exception =>
-                error(s"Error removing access key ${key.key}. Aborting.", e)
-                return 1
-            }
-          }
-
-          try {
-            apps.delete(app.id)
-            info(s"Deleted app ${app.name}.")
-          } catch {
-            case e: Exception =>
-              error(s"Error deleting app ${app.name}. Aborting.", e)
-              return 1
-          }
-
-          info("Done.")
-          0
-        }
-        case _ =>
-          info("Aborted.")
-          0
-      }
-    } getOrElse {
-      error(s"App ${ca.app.name} does not exist. Aborting.")
-      1
-    }
-    events.close()
-    status
-  }
-
-  def dataDelete(ca: ConsoleArgs): Int = {
-    if (ca.app.all) {
-      dataDeleteAll(ca)
-    } else {
-      dataDeleteOne(ca)
-    }
-  }
-
-  def dataDeleteOne(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps
-    val channels = storage.Storage.getMetaDataChannels
-    apps.getByName(ca.app.name) map { app =>
-
-      val channelId = ca.app.dataDeleteChannel.map { ch =>
-        val channelMap = channels.getByAppid(app.id).map(c => (c.name, c.id)).toMap
-        if (!channelMap.contains(ch)) {
-          error(s"Unable to delete data for channel.")
-          error(s"Channel ${ch} doesn't exist.")
-          return 1
-        }
-
-        channelMap(ch)
-      }
-
-      if (channelId.isDefined) {
-        info(s"Data of the following channel will be deleted. Are you sure?")
-        info(s"Channel Name: ${ca.app.dataDeleteChannel.get}")
-        info(s"  Channel ID: ${channelId.get}")
-        info(s"    App Name: ${app.name}")
-        info(s"      App ID: ${app.id}")
-        info(s" Description: ${app.description}")
-      } else {
-        info(s"Data of the following app (default channel only) will be deleted. Are you sure?")
-        info(s"    App Name: ${app.name}")
-        info(s"      App ID: ${app.id}")
-        info(s" Description: ${app.description}")
-      }
-
-      val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
-
-      choice match {
-        case "YES" => {
-          val events = storage.Storage.getLEvents()
-          // remove table
-          val r1 = if (events.remove(app.id, channelId)) {
-            if (channelId.isDefined) {
-              info(s"Removed Event Store for this channel ID: ${channelId.get}")
-            } else {
-              info(s"Removed Event Store for this app ID: ${app.id}")
-            }
-            0
-          } else {
-            if (channelId.isDefined) {
-              error(s"Error removing Event Store for this channel.")
-            } else {
-              error(s"Error removing Event Store for this app.")
-            }
-            1
-          }
-          // re-create table
-          val dbInit = events.init(app.id, channelId)
-          val r2 = if (dbInit) {
-            if (channelId.isDefined) {
-              info(s"Initialized Event Store for this channel ID: ${channelId.get}.")
-            } else {
-              info(s"Initialized Event Store for this app ID: ${app.id}.")
-            }
-            0
-          } else {
-            if (channelId.isDefined) {
-              error(s"Unable to initialize Event Store for this channel ID:" +
-                s" ${channelId.get}.")
-            } else {
-              error(s"Unable to initialize Event Store for this appId:" +
-                s" ${app.id}.")
-            }
-            1
-          }
-          events.close()
-          info("Done.")
-          r1 + r2
-        }
-        case _ =>
-          info("Aborted.")
-          0
-      }
-    } getOrElse {
-      error(s"App ${ca.app.name} does not exist. Aborting.")
-      1
-    }
-  }
-
-  def dataDeleteAll(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps
-    val channels = storage.Storage.getMetaDataChannels
-    val events = storage.Storage.getLEvents()
-    val status = apps.getByName(ca.app.name) map { app =>
-      info(s"All data of the app (including default and all channels) will be deleted." +
-        " Are you sure?")
-      info(s"    App Name: ${app.name}")
-      info(s"      App ID: ${app.id}")
-      info(s" Description: ${app.description}")
-      val chans = channels.getByAppid(app.id)
-      var firstChan = true
-      val titleName = "Channel Name"
-      val titleID = "Channel ID"
-      chans.foreach { ch =>
-        if (firstChan) {
-          info(f"    Channels: ${titleName}%16s | ${titleID}%10s ")
-          firstChan = false
-        }
-        info(f"              ${ch.name}%16s | ${ch.id}%10s")
-      }
-
-      val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
-      choice match {
-        case "YES" => {
-          // delete channels
-          val delChannelStatus: Seq[Int] = chans.map { ch =>
-            val r1 = if (events.remove(app.id, Some(ch.id))) {
-              info(s"Removed Event Store of the channel ID: ${ch.id}")
-              0
-            } else {
-              error(s"Error removing Event Store of the channel ID: ${ch.id}.")
-              1
-            }
-            // re-create table
-            val dbInit = events.init(app.id, Some(ch.id))
-            val r2 = if (dbInit) {
-              info(s"Initialized Event Store of the channel ID: ${ch.id}")
-              0
-            } else {
-              error(s"Unable to initialize Event Store of the channel ID: ${ch.id}.")
-              1
-            }
-            r1 + r2
-          }
-
-          if (delChannelStatus.filter(_ != 0).isEmpty) {
-            val r1 = if (events.remove(app.id)) {
-              info(s"Removed Event Store for this app ID: ${app.id}")
-              0
-            } else {
-              error(s"Error removing Event Store for this app.")
-              1
-            }
-
-            val dbInit = events.init(app.id)
-            val r2 = if (dbInit) {
-              info(s"Initialized Event Store for this app ID: ${app.id}.")
-              0
-            } else {
-              error(s"Unable to initialize Event Store for this appId: ${app.id}.")
-              1
-            }
-            info("Done.")
-            r1 + r2
-          } else 1
-        }
-        case _ =>
-          info("Aborted.")
-          0
-      }
-    } getOrElse {
-      error(s"App ${ca.app.name} does not exist. Aborting.")
-      1
-    }
-    events.close()
-    status
-  }
-
-  def channelNew(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps
-    val channels = storage.Storage.getMetaDataChannels
-    val events = storage.Storage.getLEvents()
-    val newChannel = ca.app.channel
-    val status = apps.getByName(ca.app.name) map { app =>
-      val channelMap = channels.getByAppid(app.id).map(c => (c.name, c.id)).toMap
-      if (channelMap.contains(newChannel)) {
-        error(s"Unable to create new channel.")
-        error(s"Channel ${newChannel} already exists.")
-        1
-      } else if (!storage.Channel.isValidName(newChannel)) {
-        error(s"Unable to create new channel.")
-        error(s"The channel name ${newChannel} is invalid.")
-        error(s"${storage.Channel.nameConstraint}")
-        1
-      } else {
-
-        val channelId = channels.insert(storage.Channel(
-          id = 0, // new id will be assigned
-          appid = app.id,
-          name = newChannel
-        ))
-        channelId.map { chanId =>
-          info(s"Updated Channel meta-data.")
-          // initialize storage
-          val dbInit = events.init(app.id, Some(chanId))
-          if (dbInit) {
-            info(s"Initialized Event Store for the channel: ${newChannel}.")
-            info(s"Created new channel:")
-            info(s"    Channel Name: ${newChannel}")
-            info(s"      Channel ID: ${chanId}")
-            info(s"          App ID: ${app.id}")
-            0
-          } else {
-            error(s"Unable to create new channel.")
-            error(s"Failed to initalize Event Store.")
-            // reverted back the meta data
-            try {
-              channels.delete(chanId)
-              0
-            } catch {
-              case e: Exception =>
-                error(s"Failed to revert back the Channel meta-data change.", e)
-                error(s"The channel ${newChannel} CANNOT be used!")
-                error(s"Please run 'pio app channel-delete ${app.name} ${newChannel}' " +
-                  "to delete this channel!")
-                1
-            }
-          }
-        }.getOrElse {
-          error(s"Unable to create new channel.")
-          error(s"Failed to update Channel meta-data.")
-          1
-        }
-      }
-    } getOrElse {
-      error(s"App ${ca.app.name} does not exist. Aborting.")
-      1
-    }
-    events.close()
-    status
-  }
-
-  def channelDelete(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps
-    val channels = storage.Storage.getMetaDataChannels
-    val events = storage.Storage.getLEvents()
-    val deleteChannel = ca.app.channel
-    val status = apps.getByName(ca.app.name) map { app =>
-      val channelMap = channels.getByAppid(app.id).map(c => (c.name, c.id)).toMap
-      if (!channelMap.contains(deleteChannel)) {
-        error(s"Unable to delete channel.")
-        error(s"Channel ${deleteChannel} doesn't exist.")
-        1
-      } else {
-        info(s"The following channel will be deleted. Are you sure?")
-        info(s"    Channel Name: ${deleteChannel}")
-        info(s"      Channel ID: ${channelMap(deleteChannel)}")
-        info(s"        App Name: ${app.name}")
-        info(s"          App ID: ${app.id}")
-        val choice = if(ca.app.force) "YES" else readLine("Enter 'YES' to proceed: ")
-        choice match {
-          case "YES" => {
-            // NOTE: remove storage first before remove meta data (in case remove storage failed)
-            val dbRemoved = events.remove(app.id, Some(channelMap(deleteChannel)))
-            if (dbRemoved) {
-              info(s"Removed Event Store for this channel: ${deleteChannel}")
-              try {
-                channels.delete(channelMap(deleteChannel))
-                info(s"Deleted channel: ${deleteChannel}.")
-                0
-              } catch {
-                case e: Exception =>
-                  error(s"Unable to delete channel.", e)
-                  error(s"Failed to update Channel meta-data.")
-                  error(s"The channel ${deleteChannel} CANNOT be used!")
-                  error(s"Please run 'pio app channel-delete ${app.name} ${deleteChannel}' " +
-                    "to delete this channel again!")
-                  1
-              }
-            } else {
-              error(s"Unable to delete channel.")
-              error(s"Error removing Event Store for this channel.")
-              1
-            }
-          }
-          case _ =>
-            info("Aborted.")
-            0
-        }
-      }
-    } getOrElse {
-      error(s"App ${ca.app.name} does not exist. Aborting.")
-      1
-    }
-    events.close()
-    status
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
index 5646467..195740b 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
@@ -29,10 +29,16 @@ import org.apache.predictionio.data.api.EventServerConfig
 import org.apache.predictionio.data.storage
 import org.apache.predictionio.data.storage.EngineManifest
 import org.apache.predictionio.data.storage.EngineManifestSerializer
-import org.apache.predictionio.data.storage.hbase.upgrade.Upgrade_0_8_3
 import org.apache.predictionio.tools.RegisterEngine
 import org.apache.predictionio.tools.RunServer
 import org.apache.predictionio.tools.RunWorkflow
+import org.apache.predictionio.tools.Common
+import org.apache.predictionio.tools.commands.{
+  DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
+  BuildArgs, EngineArgs}
+import org.apache.predictionio.tools.{
+  EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
+import org.apache.predictionio.tools.EventServerArgs
 import org.apache.predictionio.tools.admin.AdminServer
 import org.apache.predictionio.tools.admin.AdminServerConfig
 import org.apache.predictionio.tools.dashboard.Dashboard
@@ -40,6 +46,7 @@ import org.apache.predictionio.tools.dashboard.DashboardConfig
 import org.apache.predictionio.workflow.JsonExtractorOption
 import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
 import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.tools.commands
 import org.apache.commons.io.FileUtils
 import org.json4s._
 import org.json4s.native.JsonMethods._
@@ -54,82 +61,39 @@ import scala.util.Random
 import scalaj.http.Http
 
 case class ConsoleArgs(
-  common: CommonArgs = CommonArgs(),
   build: BuildArgs = BuildArgs(),
   app: AppArgs = AppArgs(),
+  spark: SparkArgs = SparkArgs(),
+  engine: EngineArgs = EngineArgs(),
+  workflow: WorkflowArgs = WorkflowArgs(),
   accessKey: AccessKeyArgs = AccessKeyArgs(),
   deploy: DeployArgs = DeployArgs(),
   eventServer: EventServerArgs = EventServerArgs(),
   adminServer: AdminServerArgs = AdminServerArgs(),
   dashboard: DashboardArgs = DashboardArgs(),
-  upgrade: UpgradeArgs = UpgradeArgs(),
-  template: TemplateArgs = TemplateArgs(),
   export: ExportArgs = ExportArgs(),
   imprt: ImportArgs = ImportArgs(),
   commands: Seq[String] = Seq(),
-  metricsClass: Option[String] = None,
   metricsParamsJsonPath: Option[String] = None,
   paramsPath: String = "params",
   engineInstanceId: Option[String] = None,
-  mainClass: Option[String] = None)
-
-case class CommonArgs(
-  batch: String = "",
-  sparkPassThrough: Seq[String] = Seq(),
+  mainClass: Option[String] = None,
   driverPassThrough: Seq[String] = Seq(),
   pioHome: Option[String] = None,
-  sparkHome: Option[String] = None,
-  engineId: Option[String] = None,
-  engineVersion: Option[String] = None,
-  engineFactory: Option[String] = None,
-  engineParamsKey: Option[String] = None,
-  evaluation: Option[String] = None,
-  engineParamsGenerator: Option[String] = None,
-  variantJson: File = new File("engine.json"),
-  manifestJson: File = new File("manifest.json"),
-  stopAfterRead: Boolean = false,
-  stopAfterPrepare: Boolean = false,
-  skipSanityCheck: Boolean = false,
-  verbose: Boolean = false,
-  verbosity: Int = 0,
-  sparkKryo: Boolean = false,
-  scratchUri: Option[URI] = None,
-  jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
-
-case class BuildArgs(
-  sbt: Option[File] = None,
-  sbtExtra: Option[String] = None,
-  sbtAssemblyPackageDependency: Boolean = true,
-  sbtClean: Boolean = false,
-  uberJar: Boolean = false,
-  forceGeneratePIOSbt: Boolean = false)
-
-case class DeployArgs(
-  ip: String = "0.0.0.0",
-  port: Int = 8000,
-  logUrl: Option[String] = None,
-  logPrefix: Option[String] = None)
-
-case class EventServerArgs(
-  enabled: Boolean = false,
-  ip: String = "0.0.0.0",
-  port: Int = 7070,
-  stats: Boolean = false)
-
-case class AdminServerArgs(
-ip: String = "127.0.0.1",
-port: Int = 7071)
+  verbose: Boolean = false)
 
-case class DashboardArgs(
-  ip: String = "127.0.0.1",
-  port: Int = 9000)
+case class AppArgs(
+  id: Option[Int] = None,
+  name: String = "",
+  channel: String = "",
+  dataDeleteChannel: Option[String] = None,
+  all: Boolean = false,
+  force: Boolean = false,
+  description: Option[String] = None)
 
-case class UpgradeArgs(
-  from: String = "0.0.0",
-  to: String = "0.0.0",
-  oldAppId: Int = 0,
-  newAppId: Int = 0
-)
+case class AccessKeyArgs(
+  accessKey: String = "",
+  events: Seq[String] = Seq())
 
 object Console extends Logging {
   def main(args: Array[String]): Unit = {
@@ -145,27 +109,27 @@ object Console extends Logging {
         "for each command for more information.\n\n" +
         "The following options are common to all commands:\n")
       opt[String]("pio-home") action { (x, c) =>
-        c.copy(common = c.common.copy(pioHome = Some(x)))
+        c.copy(pioHome = Some(x))
       } text("Root directory of a PredictionIO installation.\n" +
         "        Specify this if automatic discovery fail.")
       opt[String]("spark-home") action { (x, c) =>
-        c.copy(common = c.common.copy(sparkHome = Some(x)))
+        c.copy(spark = c.spark.copy(sparkHome = Some(x)))
       } text("Root directory of an Apache Spark installation.\n" +
         "        If not specified, will try to use the SPARK_HOME\n" +
         "        environmental variable. If this fails as well, default to\n" +
         "        current directory.")
       opt[String]("engine-id") abbr("ei") action { (x, c) =>
-        c.copy(common = c.common.copy(engineId = Some(x)))
+        c.copy(engine = c.engine.copy(engineId = Some(x)))
       } text("Specify an engine ID. Usually used by distributed deployment.")
       opt[String]("engine-version") abbr("ev") action { (x, c) =>
-        c.copy(common = c.common.copy(engineVersion = Some(x)))
+        c.copy(engine = c.engine.copy(engineVersion = Some(x)))
       } text("Specify an engine version. Usually used by distributed " +
         "deployment.")
       opt[File]("variant") abbr("v") action { (x, c) =>
-        c.copy(common = c.common.copy(variantJson = x))
+        c.copy(workflow = c.workflow.copy(variantJson = x))
       }
       opt[File]("manifest") abbr("m") action { (x, c) =>
-        c.copy(common = c.common.copy(manifestJson = x))
+        c.copy(engine = c.engine.copy(manifestJson = x))
       }
       opt[File]("sbt") action { (x, c) =>
         c.copy(build = c.build.copy(sbt = Some(x)))
@@ -177,13 +141,13 @@ object Console extends Logging {
         }
       } text("Path to sbt. Default: sbt")
       opt[Unit]("verbose") action { (x, c) =>
-        c.copy(common = c.common.copy(verbose = true))
+        c.copy(verbose = true)
       }
       opt[Unit]("spark-kryo") abbr("sk") action { (x, c) =>
-        c.copy(common = c.common.copy(sparkKryo = true))
+        c.copy(spark = c.spark.copy(sparkKryo = true))
       }
       opt[String]("scratch-uri") action { (x, c) =>
-        c.copy(common = c.common.copy(scratchUri = Some(new URI(x))))
+        c.copy(spark = c.spark.copy(scratchUri = Some(new URI(x))))
       }
       note("")
       cmd("version").
@@ -237,7 +201,7 @@ object Console extends Logging {
           c.copy(commands = c.commands :+ "train")
         } children(
           opt[String]("batch") action { (x, c) =>
-            c.copy(common = c.common.copy(batch = x))
+            c.copy(workflow = c.workflow.copy(batch = x))
           } text("Batch label of the run."),
           opt[String]("params-path") action { (x, c) =>
             c.copy(paramsPath = x)
@@ -247,28 +211,28 @@ object Console extends Logging {
           } text("Metrics parameters JSON file. Will try to use\n" +
             "        metrics.json in the base path."),
           opt[Unit]("skip-sanity-check") abbr("ssc") action { (x, c) =>
-            c.copy(common = c.common.copy(skipSanityCheck = true))
+            c.copy(workflow = c.workflow.copy(skipSanityCheck = true))
           },
           opt[Unit]("stop-after-read") abbr("sar") action { (x, c) =>
-            c.copy(common = c.common.copy(stopAfterRead = true))
+            c.copy(workflow = c.workflow.copy(stopAfterRead = true))
           },
           opt[Unit]("stop-after-prepare") abbr("sap") action { (x, c) =>
-            c.copy(common = c.common.copy(stopAfterPrepare = true))
+            c.copy(workflow = c.workflow.copy(stopAfterPrepare = true))
           },
           opt[Unit]("uber-jar") action { (x, c) =>
             c.copy(build = c.build.copy(uberJar = true))
           },
           opt[Int]("verbosity") action { (x, c) =>
-            c.copy(common = c.common.copy(verbosity = x))
+            c.copy(workflow = c.workflow.copy(verbosity = x))
           },
           opt[String]("engine-factory") action { (x, c) =>
-            c.copy(common = c.common.copy(engineFactory = Some(x)))
+            c.copy(workflow = c.workflow.copy(engineFactory = Some(x)))
           },
           opt[String]("engine-params-key") action { (x, c) =>
-            c.copy(common = c.common.copy(engineParamsKey = Some(x)))
+            c.copy(workflow = c.workflow.copy(engineParamsKey = Some(x)))
           },
           opt[String]("json-extractor") action { (x, c) =>
-            c.copy(common = c.common.copy(jsonExtractor = JsonExtractorOption.withName(x)))
+            c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
           } validate { x =>
               if (JsonExtractorOption.values.map(_.toString).contains(x)) {
                 success
@@ -287,16 +251,16 @@ object Console extends Logging {
           c.copy(commands = c.commands :+ "eval")
         } children(
           arg[String]("<evaluation-class>") action { (x, c) =>
-            c.copy(common = c.common.copy(evaluation = Some(x)))
+            c.copy(workflow = c.workflow.copy(evaluation = Some(x)))
           },
           arg[String]("[<engine-parameters-generator-class>]") optional() action { (x, c) =>
-            c.copy(common = c.common.copy(engineParamsGenerator = Some(x)))
+            c.copy(workflow = c.workflow.copy(engineParamsGenerator = Some(x)))
           } text("Optional engine parameters generator class, overriding the first argument"),
           opt[String]("batch") action { (x, c) =>
-            c.copy(common = c.common.copy(batch = x))
+            c.copy(workflow = c.workflow.copy(batch = x))
           } text("Batch label of the run."),
           opt[String]("json-extractor") action { (x, c) =>
-            c.copy(common = c.common.copy(jsonExtractor = JsonExtractorOption.withName(x)))
+            c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
           } validate { x =>
             if (JsonExtractorOption.values.map(_.toString).contains(x)) {
               success
@@ -315,7 +279,7 @@ object Console extends Logging {
           c.copy(commands = c.commands :+ "deploy")
         } children(
           opt[String]("batch") action { (x, c) =>
-            c.copy(common = c.common.copy(batch = x))
+            c.copy(workflow = c.workflow.copy(batch = x))
           } text("Batch label of the deployment."),
           opt[String]("engine-instance-id") action { (x, c) =>
             c.copy(engineInstanceId = Some(x))
@@ -354,7 +318,7 @@ object Console extends Logging {
             c.copy(deploy = c.deploy.copy(logPrefix = Some(x)))
           },
           opt[String]("json-extractor") action { (x, c) =>
-            c.copy(common = c.common.copy(jsonExtractor = JsonExtractorOption.withName(x)))
+            c.copy(workflow = c.workflow.copy(jsonExtractor = JsonExtractorOption.withName(x)))
           } validate { x =>
             if (JsonExtractorOption.values.map(_.toString).contains(x)) {
               success
@@ -449,23 +413,10 @@ object Console extends Logging {
         }
       note("")
       cmd("upgrade").
-        text("Upgrade tool").
+        text("No longer supported!").
         action { (_, c) =>
           c.copy(commands = c.commands :+ "upgrade")
-        } children(
-          arg[String]("<from version>") action { (x, c) =>
-            c.copy(upgrade = c.upgrade.copy(from = x))
-          } text("The version upgraded from."),
-          arg[String]("<to version>") action { (x, c) =>
-            c.copy(upgrade = c.upgrade.copy(to = x))
-          } text("The version upgraded to."),
-          arg[Int]("<old App ID>") action { (x, c) =>
-            c.copy(upgrade = c.upgrade.copy(oldAppId = x))
-          } text("Old App ID."),
-          arg[Int]("<new App ID>") action { (x, c) =>
-            c.copy(upgrade = c.upgrade.copy(newAppId = x))
-          } text("New App ID.")
-        )
+        }
       note("")
       cmd("app").
         text("Manage apps.\n").
@@ -616,29 +567,12 @@ object Console extends Logging {
           c.copy(commands = c.commands :+ "template")
         } children(
           cmd("get").
+            text("No longer supported! Use git clone to download a template").
             action { (_, c) =>
               c.copy(commands = c.commands :+ "get")
-            } children(
-              arg[String]("<template ID>") required() action { (x, c) =>
-                c.copy(template = c.template.copy(repository = x))
-              },
-              arg[String]("<new engine directory>") action { (x, c) =>
-                c.copy(template = c.template.copy(directory = x))
-              },
-              opt[String]("version") action { (x, c) =>
-                c.copy(template = c.template.copy(version = Some(x)))
-              },
-              opt[String]("name") action { (x, c) =>
-                c.copy(template = c.template.copy(name = Some(x)))
-              },
-              opt[String]("package") action { (x, c) =>
-                c.copy(template = c.template.copy(packageName = Some(x)))
-              },
-              opt[String]("email") action { (x, c) =>
-                c.copy(template = c.template.copy(email = Some(x)))
-              }
-            ),
+            },
           cmd("list").
+            text("No longer supported! Use git to manage your templates").
             action { (_, c) =>
               c.copy(commands = c.commands :+ "list")
             }
@@ -694,78 +628,95 @@ object Console extends Logging {
       }
 
     parser.parse(consoleArgs, ConsoleArgs()) map { pca =>
-      val ca = pca.copy(common = pca.common.copy(
-        sparkPassThrough = sparkPassThroughArgs,
-        driverPassThrough = driverPassThroughArgs))
-      WorkflowUtils.modifyLogging(ca.common.verbose)
+      val ca = pca.copy(
+        spark = pca.spark.copy(sparkPassThrough = sparkPassThroughArgs),
+        driverPassThrough = driverPassThroughArgs)
+      WorkflowUtils.modifyLogging(ca.verbose)
       val rv: Int = ca.commands match {
         case Seq("") =>
           System.err.println(help())
           1
         case Seq("version") =>
-          version(ca)
-          0
+          Pio.version()
         case Seq("build") =>
-          regenerateManifestJson(ca.common.manifestJson)
-          build(ca)
+          Pio.build(
+            ca.build, ca.pioHome.get, ca.engine.manifestJson, ca.verbose)
         case Seq("unregister") =>
-          unregister(ca)
-          0
+          Pio.unregister(ca.engine.manifestJson)
         case Seq("train") =>
-          regenerateManifestJson(ca.common.manifestJson)
-          train(ca)
+          Pio.train(
+            ca.engine, ca.workflow, ca.spark, ca.pioHome.get, ca.verbose)
         case Seq("eval") =>
-          regenerateManifestJson(ca.common.manifestJson)
-          train(ca)
+          Pio.eval(
+            ca.engine, ca.workflow, ca.spark, ca.pioHome.get, ca.verbose)
         case Seq("deploy") =>
-          deploy(ca)
+          Pio.deploy(
+            ca.engine,
+            ca.engineInstanceId,
+            ServerArgs(
+              ca.deploy,
+              ca.eventServer,
+              ca.workflow.batch,
+              ca.accessKey.accessKey,
+              ca.workflow.variantJson,
+              ca.workflow.jsonExtractor),
+            ca.spark,
+            ca.pioHome.get,
+            ca.verbose)
         case Seq("undeploy") =>
-          undeploy(ca)
+          Pio.undeploy(ca.deploy)
         case Seq("dashboard") =>
-          dashboard(ca)
-          0
+          Pio.dashboard(ca.dashboard)
         case Seq("eventserver") =>
-          eventserver(ca)
-          0
+          Pio.eventserver(ca.eventServer)
         case Seq("adminserver") =>
-          adminserver(ca)
-          0
+          Pio.adminserver(ca.adminServer)
         case Seq("run") =>
-          generateManifestJson(ca.common.manifestJson)
-          run(ca)
+          Pio.run(
+            ca.mainClass.get,
+            ca.driverPassThrough,
+            ca.engine.manifestJson,
+            ca.build,
+            ca.spark,
+            ca.pioHome.get,
+            ca.verbose)
         case Seq("status") =>
-          status(ca)
+          Pio.status(ca.pioHome, ca.spark.sparkHome)
         case Seq("upgrade") =>
-          upgrade(ca)
-          0
+          error("Upgrade is no longer supported")
+          1
         case Seq("app", "new") =>
-          App.create(ca)
+          Pio.App.create(
+            ca.app.name, ca.app.id, ca.app.description, ca.accessKey.accessKey)
         case Seq("app", "list") =>
-          App.list(ca)
+          Pio.App.list()
         case Seq("app", "show") =>
-          App.show(ca)
+          Pio.App.show(ca.app.name)
         case Seq("app", "delete") =>
-          App.delete(ca)
+          Pio.App.delete(ca.app.name, ca.app.force)
         case Seq("app", "data-delete") =>
-          App.dataDelete(ca)
+          Pio.App.dataDelete(
+            ca.app.name, ca.app.dataDeleteChannel, ca.app.all, ca.app.force)
         case Seq("app", "channel-new") =>
-          App.channelNew(ca)
+          Pio.App.channelNew(ca.app.name, ca.app.channel)
         case Seq("app", "channel-delete") =>
-          App.channelDelete(ca)
+          Pio.App.channelDelete(ca.app.name, ca.app.channel, ca.app.force)
         case Seq("accesskey", "new") =>
-          AccessKey.create(ca)
+          Pio.AccessKey.create(
+            ca.app.name, ca.accessKey.accessKey, ca.accessKey.events)
         case Seq("accesskey", "list") =>
-          AccessKey.list(ca)
+         Pio.AccessKey.list(
+           if (ca.app.name == "") None else Some(ca.app.name))
         case Seq("accesskey", "delete") =>
-          AccessKey.delete(ca)
-        case Seq("template", "get") =>
-          Template.get(ca)
-        case Seq("template", "list") =>
-          Template.list(ca)
+          Pio.AccessKey.delete(ca.accessKey.accessKey)
+        case Seq("template", _) =>
+          error("template commands are no longer supported.")
+          error("Please use git to get and manage your templates.")
+          1
         case Seq("export") =>
-          Export.eventsToFile(ca)
+          Pio.export(ca.export, ca.spark, ca.pioHome.get)
         case Seq("import") =>
-          Import.fileToEvents(ca)
+          Pio.imprt(ca.imprt, ca.spark, ca.pioHome.get)
         case _ =>
           System.err.println(help(ca.commands))
           1
@@ -809,465 +760,4 @@ object Console extends Logging {
     "run" -> txt.run().toString,
     "eval" -> txt.eval().toString,
     "dashboard" -> txt.dashboard().toString)
-
-  def version(ca: ConsoleArgs): Unit = println(BuildInfo.version)
-
-  def build(ca: ConsoleArgs): Int = {
-    Template.verifyTemplateMinVersion(new File("template.json"))
-    compile(ca)
-    info("Looking for an engine...")
-    val jarFiles = jarFilesForScala
-    if (jarFiles.isEmpty) {
-      error("No engine found. Your build might have failed. Aborting.")
-      return 1
-    }
-    jarFiles foreach { f => info(s"Found ${f.getName}")}
-    RegisterEngine.registerEngine(
-      ca.common.manifestJson,
-      jarFiles,
-      false)
-    info("Your engine is ready for training.")
-    0
-  }
-
-  def unregister(ca: ConsoleArgs): Unit = {
-    RegisterEngine.unregisterEngine(ca.common.manifestJson)
-  }
-
-  def train(ca: ConsoleArgs): Int = {
-    Template.verifyTemplateMinVersion(new File("template.json"))
-    withRegisteredManifest(
-      ca.common.manifestJson,
-      ca.common.engineId,
-      ca.common.engineVersion) { em =>
-      RunWorkflow.newRunWorkflow(ca, em)
-    }
-  }
-
-  def deploy(ca: ConsoleArgs): Int = {
-    Template.verifyTemplateMinVersion(new File("template.json"))
-    withRegisteredManifest(
-      ca.common.manifestJson,
-      ca.common.engineId,
-      ca.common.engineVersion) { em =>
-      val variantJson = parse(Source.fromFile(ca.common.variantJson).mkString)
-      val variantId = variantJson \ "id" match {
-        case JString(s) => s
-        case _ =>
-          error("Unable to read engine variant ID from " +
-            s"${ca.common.variantJson.getCanonicalPath}. Aborting.")
-          return 1
-      }
-      val engineInstances = storage.Storage.getMetaDataEngineInstances
-      val engineInstance = ca.engineInstanceId map { eid =>
-        engineInstances.get(eid)
-      } getOrElse {
-        engineInstances.getLatestCompleted(em.id, em.version, variantId)
-      }
-      engineInstance map { r =>
-        RunServer.newRunServer(ca, em, r.id)
-      } getOrElse {
-        ca.engineInstanceId map { eid =>
-          error(
-            s"Invalid engine instance ID ${ca.engineInstanceId}. Aborting.")
-        } getOrElse {
-          error(
-            s"No valid engine instance found for engine ${em.id} " +
-              s"${em.version}.\nTry running 'train' before 'deploy'. Aborting.")
-        }
-        1
-      }
-    }
-  }
-
-  def dashboard(ca: ConsoleArgs): Unit = {
-    info(s"Creating dashboard at ${ca.dashboard.ip}:${ca.dashboard.port}")
-    Dashboard.createDashboard(DashboardConfig(
-      ip = ca.dashboard.ip,
-      port = ca.dashboard.port))
-  }
-
-  def eventserver(ca: ConsoleArgs): Unit = {
-    info(
-      s"Creating Event Server at ${ca.eventServer.ip}:${ca.eventServer.port}")
-    EventServer.createEventServer(EventServerConfig(
-      ip = ca.eventServer.ip,
-      port = ca.eventServer.port,
-      stats = ca.eventServer.stats))
-  }
-
-  def adminserver(ca: ConsoleArgs): Unit = {
-    info(
-      s"Creating Admin Server at ${ca.adminServer.ip}:${ca.adminServer.port}")
-    AdminServer.createAdminServer(AdminServerConfig(
-      ip = ca.adminServer.ip,
-      port = ca.adminServer.port
-    ))
-  }
-
-  def undeploy(ca: ConsoleArgs): Int = {
-    val serverUrl = s"http://${ca.deploy.ip}:${ca.deploy.port}"
-    info(
-      s"Undeploying any existing engine instance at ${serverUrl}")
-    try {
-      val code = Http(s"${serverUrl}/stop").asString.code
-      code match {
-        case 200 => 0
-        case 404 =>
-          error(s"Another process is using ${serverUrl}. Unable to undeploy.")
-          1
-        case _ =>
-          error(s"Another process is using ${serverUrl}, or an existing " +
-            s"engine server is not responding properly (HTTP ${code}). " +
-            "Unable to undeploy.")
-            1
-      }
-    } catch {
-      case e: java.net.ConnectException =>
-        warn(s"Nothing at ${serverUrl}")
-        0
-      case _: Throwable =>
-        error("Another process might be occupying " +
-          s"${ca.deploy.ip}:${ca.deploy.port}. Unable to undeploy.")
-        1
-    }
-  }
-
-  def compile(ca: ConsoleArgs): Unit = {
-    // only add pioVersion to sbt if project/pio.sbt exists
-    if (new File("project", "pio-build.sbt").exists || ca.build.forceGeneratePIOSbt) {
-      FileUtils.writeLines(
-        new File("pio.sbt"),
-        Seq(
-          "// Generated automatically by pio build.",
-          "// Changes in this file will be overridden.",
-          "",
-          "pioVersion := \"" + BuildInfo.version + "\""))
-    }
-    implicit val formats = Utils.json4sDefaultFormats
-
-    val sbt = detectSbt(ca)
-    info(s"Using command '${sbt}' at the current working directory to build.")
-    info("If the path above is incorrect, this process will fail.")
-    val asm =
-      if (ca.build.sbtAssemblyPackageDependency) {
-        " assemblyPackageDependency"
-      } else {
-        ""
-      }
-    val clean = if (ca.build.sbtClean) " clean" else ""
-    val buildCmd = s"${sbt} ${ca.build.sbtExtra.getOrElse("")}${clean} " +
-      (if (ca.build.uberJar) "assembly" else s"package${asm}")
-    val core = new File(s"pio-assembly-${BuildInfo.version}.jar")
-    if (ca.build.uberJar) {
-      info(s"Uber JAR enabled. Putting ${core.getName} in lib.")
-      val dst = new File("lib")
-      dst.mkdir()
-      FileUtils.copyFileToDirectory(
-        coreAssembly(ca.common.pioHome.get),
-        dst,
-        true)
-    } else {
-      if (new File("engine.json").exists()) {
-        info(s"Uber JAR disabled. Making sure lib/${core.getName} is absent.")
-        new File("lib", core.getName).delete()
-      } else {
-        info("Uber JAR disabled, but current working directory does not look " +
-          s"like an engine project directory. Please delete lib/${core.getName} manually.")
-      }
-    }
-    info(s"Going to run: ${buildCmd}")
-    try {
-      val r =
-        if (ca.common.verbose) {
-          buildCmd.!(ProcessLogger(line => info(line), line => error(line)))
-        } else {
-          buildCmd.!(ProcessLogger(
-            line => outputSbtError(line),
-            line => outputSbtError(line)))
-        }
-      if (r != 0) {
-        error(s"Return code of previous step is ${r}. Aborting.")
-        sys.exit(1)
-      }
-      info("Build finished successfully.")
-    } catch {
-      case e: java.io.IOException =>
-        error(s"${e.getMessage}")
-        sys.exit(1)
-    }
-  }
-
-  private def outputSbtError(line: String): Unit = {
-    """\[.*error.*\]""".r findFirstIn line foreach { _ => error(line) }
-  }
-
-  def run(ca: ConsoleArgs): Int = {
-    compile(ca)
-
-    val extraFiles = WorkflowUtils.thirdPartyConfFiles
-
-    val jarFiles = jarFilesForScala
-    jarFiles foreach { f => info(s"Found JAR: ${f.getName}") }
-    val allJarFiles = jarFiles.map(_.getCanonicalPath)
-    val cmd = s"${getSparkHome(ca.common.sparkHome)}/bin/spark-submit --jars " +
-      s"${allJarFiles.mkString(",")} " +
-      (if (extraFiles.size > 0) {
-        s"--files ${extraFiles.mkString(",")} "
-      } else {
-        ""
-      }) +
-      "--class " +
-      s"${ca.mainClass.get} ${ca.common.sparkPassThrough.mkString(" ")} " +
-      coreAssembly(ca.common.pioHome.get) + " " +
-      ca.common.driverPassThrough.mkString(" ")
-    val proc = Process(
-      cmd,
-      None,
-      "SPARK_YARN_USER_ENV" -> sys.env.filter(kv => kv._1.startsWith("PIO_")).
-        map(kv => s"${kv._1}=${kv._2}").mkString(","))
-    info(s"Submission command: ${cmd}")
-    val r = proc.!
-    if (r != 0) {
-      error(s"Return code of previous step is ${r}. Aborting.")
-      return 1
-    }
-    r
-  }
-
-  def status(ca: ConsoleArgs): Int = {
-    info("Inspecting PredictionIO...")
-    ca.common.pioHome map { pioHome =>
-      info(s"PredictionIO ${BuildInfo.version} is installed at $pioHome")
-    } getOrElse {
-      error("Unable to locate PredictionIO installation. Aborting.")
-      return 1
-    }
-    info("Inspecting Apache Spark...")
-    val sparkHome = getSparkHome(ca.common.sparkHome)
-    if (new File(s"$sparkHome/bin/spark-submit").exists) {
-      info(s"Apache Spark is installed at $sparkHome")
-      val sparkMinVersion = "1.3.0"
-      val sparkReleaseFile = new File(s"$sparkHome/RELEASE")
-      if (sparkReleaseFile.exists) {
-        val sparkReleaseStrings =
-          Source.fromFile(sparkReleaseFile).mkString.split(' ')
-        if (sparkReleaseStrings.length < 2) {
-          warn(stripMarginAndNewlines(
-            s"""|Apache Spark version information cannot be found (RELEASE file
-                |is empty). This is a known issue for certain vendors (e.g.
-                |Cloudera). Please make sure you are using a version of at least
-                |$sparkMinVersion."""))
-        } else {
-          val sparkReleaseVersion = sparkReleaseStrings(1)
-          val parsedMinVersion = Version.apply(sparkMinVersion)
-          val parsedCurrentVersion = Version.apply(sparkReleaseVersion)
-          if (parsedCurrentVersion >= parsedMinVersion) {
-            info(stripMarginAndNewlines(
-              s"""|Apache Spark $sparkReleaseVersion detected (meets minimum
-                  |requirement of $sparkMinVersion)"""))
-          } else {
-            error(stripMarginAndNewlines(
-              s"""|Apache Spark $sparkReleaseVersion detected (does not meet
-                  |minimum requirement. Aborting."""))
-          }
-        }
-      } else {
-        warn(stripMarginAndNewlines(
-          s"""|Apache Spark version information cannot be found. If you are
-              |using a developmental tree, please make sure you are using a
-              |version of at least $sparkMinVersion."""))
-      }
-    } else {
-      error("Unable to locate a proper Apache Spark installation. Aborting.")
-      return 1
-    }
-    info("Inspecting storage backend connections...")
-    try {
-      storage.Storage.verifyAllDataObjects()
-    } catch {
-      case e: Throwable =>
-        error("Unable to connect to all storage backends successfully. The " +
-          "following shows the error message from the storage backend.")
-        error(s"${e.getMessage} (${e.getClass.getName})", e)
-        error("Dumping configuration of initialized storage backend sources. " +
-          "Please make sure they are correct.")
-        storage.Storage.config.get("sources") map { src =>
-          src foreach { case (s, p) =>
-            error(s"Source Name: $s; Type: ${p.getOrElse("type", "(error)")}; " +
-              s"Configuration: ${p.getOrElse("config", "(error)")}")
-          }
-        } getOrElse {
-          error("No properly configured storage backend sources.")
-        }
-        return 1
-    }
-    info("(sleeping 5 seconds for all messages to show up...)")
-    Thread.sleep(5000)
-    info("Your system is all ready to go.")
-    0
-  }
-
-  def upgrade(ca: ConsoleArgs): Unit = {
-    (ca.upgrade.from, ca.upgrade.to) match {
-      case ("0.8.2", "0.8.3") => {
-        Upgrade_0_8_3.runMain(ca.upgrade.oldAppId, ca.upgrade.newAppId)
-      }
-      case _ =>
-        println(s"Upgrade from version ${ca.upgrade.from} to ${ca.upgrade.to}"
-          + s" is not supported.")
-    }
-  }
-
-  def coreAssembly(pioHome: String): File = {
-    val core = s"pio-assembly-${BuildInfo.version}.jar"
-    val coreDir =
-      if (new File(pioHome + File.separator + "RELEASE").exists) {
-        new File(pioHome + File.separator + "lib")
-      } else {
-        new File(pioHome + File.separator + "assembly")
-      }
-    val coreFile = new File(coreDir, core)
-    if (coreFile.exists) {
-      coreFile
-    } else {
-      error(s"PredictionIO Core Assembly (${coreFile.getCanonicalPath}) does " +
-        "not exist. Aborting.")
-      sys.exit(1)
-    }
-  }
-
-  val manifestAutogenTag = "pio-autogen-manifest"
-
-  def regenerateManifestJson(json: File): Unit = {
-    val cwd = sys.props("user.dir")
-    val ha = java.security.MessageDigest.getInstance("SHA-1").
-      digest(cwd.getBytes).map("%02x".format(_)).mkString
-    if (json.exists) {
-      val em = readManifestJson(json)
-      if (em.description == Some(manifestAutogenTag) && ha != em.version) {
-        warn("This engine project directory contains an auto-generated " +
-          "manifest that has been copied/moved from another location. ")
-        warn("Regenerating the manifest to reflect the updated location. " +
-          "This will dissociate with all previous engine instances.")
-        generateManifestJson(json)
-      } else {
-        info(s"Using existing engine manifest JSON at ${json.getCanonicalPath}")
-      }
-    } else {
-      generateManifestJson(json)
-    }
-  }
-
-  def generateManifestJson(json: File): Unit = {
-    val cwd = sys.props("user.dir")
-    implicit val formats = Utils.json4sDefaultFormats +
-      new EngineManifestSerializer
-    val rand = Random.alphanumeric.take(32).mkString
-    val ha = java.security.MessageDigest.getInstance("SHA-1").
-      digest(cwd.getBytes).map("%02x".format(_)).mkString
-    val em = EngineManifest(
-      id = rand,
-      version = ha,
-      name = new File(cwd).getName,
-      description = Some(manifestAutogenTag),
-      files = Seq(),
-      engineFactory = "")
-    try {
-      FileUtils.writeStringToFile(json, write(em), "ISO-8859-1")
-    } catch {
-      case e: java.io.IOException =>
-        error(s"Cannot generate ${json} automatically (${e.getMessage}). " +
-          "Aborting.")
-        sys.exit(1)
-    }
-  }
-
-  def readManifestJson(json: File): EngineManifest = {
-    implicit val formats = Utils.json4sDefaultFormats +
-      new EngineManifestSerializer
-    try {
-      read[EngineManifest](Source.fromFile(json).mkString)
-    } catch {
-      case e: java.io.FileNotFoundException =>
-        error(s"${json.getCanonicalPath} does not exist. Aborting.")
-        sys.exit(1)
-      case e: MappingException =>
-        error(s"${json.getCanonicalPath} has invalid content: " +
-          e.getMessage)
-        sys.exit(1)
-    }
-  }
-
-  def withRegisteredManifest(
-      json: File,
-      engineId: Option[String],
-      engineVersion: Option[String])(
-      op: EngineManifest => Int): Int = {
-    val ej = readManifestJson(json)
-    val id = engineId getOrElse ej.id
-    val version = engineVersion getOrElse ej.version
-    storage.Storage.getMetaDataEngineManifests.get(id, version) map {
-      op
-    } getOrElse {
-      error(s"Engine ${id} ${version} cannot be found in the system.")
-      error("Possible reasons:")
-      error("- the engine is not yet built by the 'build' command;")
-      error("- the meta data store is offline.")
-      1
-    }
-  }
-
-  def jarFilesAt(path: File): Array[File] = recursiveListFiles(path) filter {
-    _.getName.toLowerCase.endsWith(".jar")
-  }
-
-  def jarFilesForScala: Array[File] = {
-    val libFiles = jarFilesForScalaFilter(jarFilesAt(new File("lib")))
-    val targetFiles = jarFilesForScalaFilter(jarFilesAt(new File("target" +
-      File.separator + s"scala-${scalaVersionNoPatch}")))
-    // Use libFiles is target is empty.
-    if (targetFiles.size > 0) targetFiles else libFiles
-  }
-
-  def jarFilesForScalaFilter(jars: Array[File]): Array[File] =
-    jars.filterNot { f =>
-      f.getName.toLowerCase.endsWith("-javadoc.jar") ||
-      f.getName.toLowerCase.endsWith("-sources.jar")
-    }
-
-  def recursiveListFiles(f: File): Array[File] = {
-    Option(f.listFiles) map { these =>
-      these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
-    } getOrElse Array[File]()
-  }
-
-  def getSparkHome(sparkHome: Option[String]): String = {
-    sparkHome getOrElse {
-      sys.env.getOrElse("SPARK_HOME", ".")
-    }
-  }
-
-  def versionNoPatch(fullVersion: String): String = {
-    val v = """^(\d+\.\d+)""".r
-    val versionNoPatch = for {
-      v(np) <- v findFirstIn fullVersion
-    } yield np
-    versionNoPatch.getOrElse(fullVersion)
-  }
-
-  def scalaVersionNoPatch: String = versionNoPatch(BuildInfo.scalaVersion)
-
-  def detectSbt(ca: ConsoleArgs): String = {
-    ca.build.sbt map {
-      _.getCanonicalPath
-    } getOrElse {
-      val f = new File(Seq(ca.common.pioHome.get, "sbt", "sbt").mkString(
-        File.separator))
-      if (f.exists) f.getCanonicalPath else "sbt"
-    }
-  }
-
-  def stripMarginAndNewlines(string: String): String =
-    string.stripMargin.replaceAll("\n", " ")
 }

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala
deleted file mode 100644
index 20c9e02..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Export.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.tools.Runner
-
-case class ExportArgs(
-  appId: Int = 0,
-  channel: Option[String] = None,
-  outputPath: String = "",
-  format: String = "json")
-
-object Export {
-  def eventsToFile(ca: ConsoleArgs): Int = {
-    val channelArg = ca.export.channel
-      .map(ch => Seq("--channel", ch)).getOrElse(Nil)
-    Runner.runOnSpark(
-      "org.apache.predictionio.tools.export.EventsToFile",
-      Seq(
-        "--appid",
-        ca.export.appId.toString,
-        "--output",
-        ca.export.outputPath,
-        "--format",
-        ca.export.format) ++ channelArg,
-      ca,
-      Nil)
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala
deleted file mode 100644
index b808574..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Import.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.tools.Runner
-
-case class ImportArgs(
-  appId: Int = 0,
-  channel: Option[String] = None,
-  inputPath: String = "")
-
-object Import {
-  def fileToEvents(ca: ConsoleArgs): Int = {
-    val channelArg = ca.imprt.channel
-      .map(ch => Seq("--channel", ch)).getOrElse(Nil)
-    Runner.runOnSpark(
-      "org.apache.predictionio.tools.imprt.FileToEvents",
-      Seq(
-        "--appid",
-        ca.imprt.appId.toString,
-        "--input",
-        ca.imprt.inputPath) ++ channelArg,
-      ca,
-      Nil)
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
new file mode 100644
index 0000000..77075a7
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.console
+
+import org.apache.predictionio.tools.{
+  EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
+import org.apache.predictionio.tools.commands.Management
+import org.apache.predictionio.tools.commands.{
+  DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
+  BuildArgs, EngineArgs, AppDescription}
+import org.apache.predictionio.tools.commands.Engine
+import org.apache.predictionio.tools.commands.{
+  App => AppCmd, AccessKey => AccessKeysCmd}
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.tools.commands.Import
+import org.apache.predictionio.tools.commands.Export
+
+import grizzled.slf4j.Logging
+import scala.concurrent.{Future, ExecutionContext, Await}
+import scala.concurrent.duration._
+import scala.language.implicitConversions
+import scala.sys.process._
+import java.io.File
+
+import akka.actor.ActorSystem
+
+object Pio extends Logging {
+
+  private implicit def eitherToInt[A, B](result: Either[A, B]): Int = {
+    result fold (_ => 1, _ => 0)
+  }
+
+  private def doOnSuccess[A, B](result: Either[A, B])(f: B => Int): Int = {
+    result match {
+      case Left(_) => 1
+      case Right(res) => f(res)
+    }
+  }
+
+  private def processAwaitAndClean(maybeProc: Expected[(Process, () => Unit)]) = {
+    maybeProc match {
+      case Left(_) => 1
+
+      case Right((proc, cleanup)) =>
+        Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
+          def run(): Unit = {
+            cleanup()
+            proc.destroy()
+          }
+        }))
+        val returnVal = proc.exitValue()
+        cleanup()
+        returnVal
+    }
+  }
+
+  def version(): Int = {
+    println(Management.version)
+    0
+  }
+
+  def build(
+    buildArgs: BuildArgs,
+    pioHome: String,
+    manifestJson: File,
+    verbose: Boolean = false): Int = {
+
+    doOnSuccess(Engine.build(buildArgs, pioHome, manifestJson, verbose)) {
+      _ => info("Your engine is ready for training.")
+      0
+    }
+  }
+
+  def unregister(manifestJson: File): Int = Engine.unregister(manifestJson)
+
+  def train(
+    ea: EngineArgs,
+    wa: WorkflowArgs,
+    sa: SparkArgs,
+    pioHome: String,
+    verbose: Boolean = false): Int =
+      processAwaitAndClean(Engine.train(ea, wa, sa, pioHome, verbose))
+
+  def eval(
+    ea: EngineArgs,
+    wa: WorkflowArgs,
+    sa: SparkArgs,
+    pioHome: String,
+    verbose: Boolean = false): Int =
+      processAwaitAndClean(Engine.train(ea, wa, sa, pioHome, verbose))
+
+  def deploy(
+    ea: EngineArgs,
+    engineInstanceId: Option[String],
+    serverArgs: ServerArgs,
+    sparkArgs: SparkArgs,
+    pioHome: String,
+    verbose: Boolean = false): Int =
+      processAwaitAndClean(Engine.deploy(
+        ea, engineInstanceId, serverArgs, sparkArgs, pioHome, verbose))
+
+  def undeploy(da: DeployArgs): Int = Engine.undeploy(da)
+
+  def dashboard(da: DashboardArgs): Int = {
+    Management.dashboard(da).awaitTermination
+    0
+  }
+
+  def eventserver(ea: EventServerArgs): Int = {
+    Management.eventserver(ea).awaitTermination
+    0
+  }
+
+  def adminserver(aa: AdminServerArgs): Int = {
+    Management.adminserver(aa).awaitTermination
+    0
+  }
+
+  def run(
+    mainClass: String,
+    driverArguments: Seq[String],
+    manifestJson: File,
+    buildArgs: BuildArgs,
+    sparkArgs: SparkArgs,
+    pioHome: String,
+    verbose: Boolean = false): Int =
+      doOnSuccess(Engine.run(
+        mainClass, driverArguments, manifestJson,
+        buildArgs, sparkArgs, pioHome, verbose)) { proc =>
+
+          val r = proc.exitValue()
+          if (r != 0) {
+            error(s"Return code of previous step is ${r}. Aborting.")
+            return 1
+          }
+          r
+        }
+
+
+  def status(pioHome: Option[String], sparkHome: Option[String]): Int = {
+    Management.status(pioHome, sparkHome)
+  }
+
+  def imprt(ia: ImportArgs, sa: SparkArgs, pioHome: String): Int = {
+    processAwaitAndClean(Import.fileToEvents(ia, sa, pioHome))
+  }
+
+  def export(ea: ExportArgs, sa: SparkArgs, pioHome: String): Int = {
+    processAwaitAndClean(Export.eventsToFile(ea, sa, pioHome))
+  }
+
+  object App {
+
+    def create(
+      name: String,
+      id: Option[Int] = None,
+      description: Option[String] = None,
+      accessKey: String = ""): Int =
+        doOnSuccess(AppCmd.create(name, id, description, accessKey)) { appDesc =>
+            info("Created a new app:")
+            info(s"      Name: ${appDesc.app.name}")
+            info(s"        ID: ${appDesc.app.id}")
+            info(s"Access Key: ${appDesc.keys.head.key}")
+            0
+        }
+
+    def list(): Int = {
+      val title = "Name"
+      val ak = "Access Key"
+      val apps = AppCmd.list
+      info(f"$title%20s |   ID | $ak%64s | Allowed Event(s)")
+      apps foreach { appDesc =>
+        appDesc.keys foreach { k =>
+          val events =
+            if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
+          info(f"${appDesc.app.name}%20s | ${appDesc.app.id}%4d | ${k.key}%64s | $events%s")
+        }
+      }
+      info(s"Finished listing ${apps.size} app(s).")
+      0
+    }
+
+    def show(appName: String): Int =
+      doOnSuccess(AppCmd.show(appName)) { case (appDesc, chans) =>
+        info(s"    App Name: ${appDesc.app.name}")
+        info(s"      App ID: ${appDesc.app.id}")
+        info(s" Description: ${appDesc.app.description.getOrElse("")}")
+
+        var firstKey = true
+        appDesc.keys foreach { k =>
+          val events =
+            if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
+          if (firstKey) {
+            info(f"  Access Key: ${k.key}%s | ${events}%s")
+            firstKey = false
+          } else {
+            info(f"              ${k.key}%s | ${events}%s")
+          }
+        }
+        var firstChan = true
+        val titleName = "Channel Name"
+        val titleID = "Channel ID"
+        chans.foreach { ch =>
+          if (firstChan) {
+            info(f"    Channels: ${titleName}%16s | ${titleID}%10s ")
+            firstChan = false
+          }
+          info(f"              ${ch.name}%16s | ${ch.id}%10s")
+        }
+        0
+      }
+
+    def delete(name: String, force: Boolean = false): Int =
+      doOnSuccess(AppCmd.show(name)) { case (appDesc, chans) =>
+        info(s"The following app (including all channels) will be deleted. Are you sure?")
+        info(s"    App Name: ${appDesc.app.name}")
+        info(s"      App ID: ${appDesc.app.id}")
+        info(s" Description: ${appDesc.app.description.getOrElse("")}")
+        var firstChan = true
+        val titleName = "Channel Name"
+        val titleID = "Channel ID"
+        chans.foreach { ch =>
+          if (firstChan) {
+            info(f"    Channels: ${titleName}%16s | ${titleID}%10s ")
+            firstChan = false
+          }
+          info(f"              ${ch.name}%16s | ${ch.id}%10s")
+        }
+
+        val choice = if(force) "YES" else readLine("Enter 'YES' to proceed: ")
+        choice match {
+          case "YES" =>
+            AppCmd.delete(name)
+          case _ =>
+            info("Aborted.")
+            0
+        }
+      }
+
+    def dataDelete(
+      name: String,
+      channel: Option[String] = None,
+      all: Boolean = false,
+      force: Boolean = false): Int =
+        doOnSuccess(AppCmd.show(name)) { case (appDesc, chans) =>
+
+          val channelId = channel.map { ch =>
+            val channelMap = chans.map(c => (c.name, c.id)).toMap
+            if (!channelMap.contains(ch)) {
+              error(s"Unable to delete data for channel.")
+              error(s"Channel ${ch} doesn't exist.")
+              return 1
+            }
+            channelMap(ch)
+          }
+          if (all) {
+            info(s"All data of the app (including default and all channels) will be deleted." +
+              " Are you sure?")
+          } else if (channelId.isDefined) {
+            info(s"Data of the following channel will be deleted. Are you sure?")
+            info(s"Channel Name: ${channel.get}")
+            info(s"  Channel ID: ${channelId.get}")
+          } else {
+            info(s"Data of the following app (default channel only) will be deleted. Are you sure?")
+          }
+          info(s"    App Name: ${appDesc.app.name}")
+          info(s"      App ID: ${appDesc.app.id}")
+          info(s" Description: ${appDesc.app.description}")
+
+          val choice = if(force) "YES" else readLine("Enter 'YES' to proceed: ")
+          choice match {
+            case "YES" =>
+              AppCmd.dataDelete(name, channel, all)
+            case _ =>
+              info("Aborted.")
+              0
+          }
+        }
+
+    def channelNew(appName: String, newChannel: String): Int =
+      AppCmd.channelNew(appName, newChannel)
+
+    def channelDelete(
+      appName: String,
+      deleteChannel: String,
+      force: Boolean = false): Int =
+        doOnSuccess(AppCmd.show(appName)) { case (appDesc, chans) =>
+          chans.find(chan => chan.name == deleteChannel) match {
+            case None =>
+              error(s"Unable to delete channel.")
+              error(s"Channel ${deleteChannel} doesn't exist.")
+              1
+            case Some(chan) =>
+              info(s"The following channel will be deleted. Are you sure?")
+              info(s"    Channel Name: ${deleteChannel}")
+              info(s"      Channel ID: ${chan.id}")
+              info(s"        App Name: ${appDesc.app.name}")
+              info(s"          App ID: ${appDesc.app.id}")
+              val choice = if(force) "YES" else readLine("Enter 'YES' to proceed: ")
+              choice match {
+                case "YES" =>
+                  AppCmd.channelDelete(appName, deleteChannel)
+                case _ =>
+                  info("Aborted.")
+                  0
+              }
+          }
+        }
+
+  }
+
+  object AccessKey {
+
+    def create(
+      appName: String,
+      key: String,
+      events: Seq[String]): Int =
+        AccessKeysCmd.create(appName, key, events)
+
+    def list(app: Option[String]): Int =
+      doOnSuccess(AccessKeysCmd.list(app)) { keys =>
+        val title = "Access Key(s)"
+        info(f"$title%64s | App ID | Allowed Event(s)")
+        keys.sortBy(k => k.appid) foreach { k =>
+          val events =
+            if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
+          info(f"${k.key}%64s | ${k.appid}%6d | $events%s")
+        }
+        info(s"Finished listing ${keys.size} access key(s).")
+        0
+      }
+
+    def delete(key: String): Int = AccessKeysCmd.delete(key)
+  }
+
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala
deleted file mode 100644
index 0136d25..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Template.scala
+++ /dev/null
@@ -1,432 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import java.io.BufferedInputStream
-import java.io.BufferedOutputStream
-import java.io.File
-import java.io.FileInputStream
-import java.io.FileOutputStream
-import java.net.ConnectException
-import java.net.URI
-import java.util.zip.ZipInputStream
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.controller.Utils
-import org.apache.predictionio.core.BuildInfo
-import org.apache.commons.io.FileUtils
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-import semverfi._
-
-import scala.io.Source
-import scala.sys.process._
-import scalaj.http._
-
-case class TemplateArgs(
-  directory: String = "",
-  repository: String = "",
-  version: Option[String] = None,
-  name: Option[String] = None,
-  packageName: Option[String] = None,
-  email: Option[String] = None)
-
-case class GitHubTag(
-  name: String,
-  zipball_url: String,
-  tarball_url: String,
-  commit: GitHubCommit)
-
-case class GitHubCommit(
-  sha: String,
-  url: String)
-
-case class GitHubCache(
-  headers: Map[String, String],
-  body: String)
-
-case class TemplateEntry(
-  repo: String)
-
-case class TemplateMetaData(
-  pioVersionMin: Option[String] = None)
-
-object Template extends Logging {
-  implicit val formats = Utils.json4sDefaultFormats
-
-  def templateMetaData(templateJson: File): TemplateMetaData = {
-    if (!templateJson.exists) {
-      warn(s"$templateJson does not exist. Template metadata will not be available. " +
-        "(This is safe to ignore if you are not working on a template.)")
-      TemplateMetaData()
-    } else {
-      val jsonString = Source.fromFile(templateJson)(scala.io.Codec.ISO8859).mkString
-      val json = try {
-        parse(jsonString)
-      } catch {
-        case e: org.json4s.ParserUtil.ParseException =>
-          warn(s"$templateJson cannot be parsed. Template metadata will not be available.")
-          return TemplateMetaData()
-      }
-      val pioVersionMin = json \ "pio" \ "version" \ "min"
-      pioVersionMin match {
-        case JString(s) => TemplateMetaData(pioVersionMin = Some(s))
-        case _ => TemplateMetaData()
-      }
-    }
-  }
-
-  /** Creates a wrapper that provides the functionality of scalaj.http.Http()
-    * with automatic proxy settings handling. The proxy settings will first
-    * come from "git" followed by system properties "http.proxyHost" and
-    * "http.proxyPort".
-    *
-    * @param url URL to be connected
-    * @return
-    */
-  def httpOptionalProxy(url: String): HttpRequest = {
-    val gitProxy = try {
-      Some(Process("git config --global http.proxy").lines.toList(0))
-    } catch {
-      case e: Throwable => None
-    }
-
-    val (host, port) = gitProxy map { p =>
-      val proxyUri = new URI(p)
-      (Option(proxyUri.getHost),
-        if (proxyUri.getPort == -1) None else Some(proxyUri.getPort))
-    } getOrElse {
-      (sys.props.get("http.proxyHost"),
-        sys.props.get("http.proxyPort").map { p =>
-          try {
-            Some(p.toInt)
-          } catch {
-            case e: NumberFormatException => None
-          }
-        } getOrElse None)
-    }
-
-    (host, port) match {
-      case (Some(h), Some(p)) => Http(url).proxy(h, p)
-      case _ => Http(url)
-    }
-  }
-
-  def getGitHubRepos(
-      repos: Seq[String],
-      apiType: String,
-      repoFilename: String): Map[String, GitHubCache] = {
-    val reposCache = try {
-      val cache =
-        Source.fromFile(repoFilename)(scala.io.Codec.ISO8859).mkString
-        read[Map[String, GitHubCache]](cache)
-    } catch {
-      case e: Throwable => Map[String, GitHubCache]()
-    }
-    val newReposCache = reposCache ++ (try {
-      repos.map { repo =>
-        val url = s"https://api.github.com/repos/$repo/$apiType"
-        val http = httpOptionalProxy(url)
-        val response = reposCache.get(repo).map { cache =>
-          cache.headers.get("ETag").map { etag =>
-            http.header("If-None-Match", etag).asString
-          } getOrElse {
-            http.asString
-          }
-        } getOrElse {
-          http.asString
-        }
-
-        val body = if (response.code == 304) {
-          reposCache(repo).body
-        } else {
-          response.body
-        }
-
-        repo -> GitHubCache(headers = response.headers, body = body)
-      }.toMap
-    } catch {
-      case e: ConnectException =>
-        githubConnectErrorMessage(e)
-        Map()
-    })
-    FileUtils.writeStringToFile(
-      new File(repoFilename),
-      write(newReposCache),
-      "ISO-8859-1")
-    newReposCache
-  }
-
-  def sub(repo: String, name: String, email: String, org: String): Unit = {
-    val data = Map(
-      "repo" -> repo,
-      "name" -> name,
-      "email" -> email,
-      "org" -> org)
-    try {
-      httpOptionalProxy("https://update.prediction.io/templates.subscribe").
-        postData("json=" + write(data)).asString
-    } catch {
-      case e: Throwable => error("Unable to subscribe.")
-    }
-  }
-
-  def meta(repo: String, name: String, org: String): Unit = {
-    try {
-      httpOptionalProxy(
-        s"https://meta.prediction.io/templates/$repo/$org/$name").asString
-    } catch {
-      case e: Throwable => debug("Template metadata unavailable.")
-    }
-  }
-
-  def list(ca: ConsoleArgs): Int = {
-    val templatesUrl = "https://templates.prediction.io/index.json"
-    try {
-      val templatesJson = Source.fromURL(templatesUrl).mkString("")
-      val templates = read[List[TemplateEntry]](templatesJson)
-      println("The following is a list of template IDs registered on " +
-        "PredictionIO Template Gallery:")
-      println()
-      templates.sortBy(_.repo.toLowerCase).foreach { template =>
-        println(template.repo)
-      }
-      println()
-      println("Notice that it is possible use any GitHub repository as your " +
-        "engine template ID (e.g. YourOrg/YourTemplate).")
-      0
-    } catch {
-      case e: Throwable =>
-        error(s"Unable to list templates from $templatesUrl " +
-          s"(${e.getMessage}). Aborting.")
-        1
-    }
-  }
-
-  def githubConnectErrorMessage(e: ConnectException): Unit = {
-    error(s"Unable to connect to GitHub (Reason: ${e.getMessage}). " +
-      "Please check your network configuration and proxy settings.")
-  }
-
-  def get(ca: ConsoleArgs): Int = {
-    val repos =
-      getGitHubRepos(Seq(ca.template.repository), "tags", ".templates-cache")
-
-    repos.get(ca.template.repository).map { repo =>
-      try {
-        read[List[GitHubTag]](repo.body)
-      } catch {
-        case e: MappingException =>
-          error(s"Either ${ca.template.repository} is not a valid GitHub " +
-            "repository, or it does not have any tag. Aborting.")
-          return 1
-      }
-    } getOrElse {
-      error(s"Failed to retrieve ${ca.template.repository}. Aborting.")
-      return 1
-    }
-
-    val name = ca.template.name getOrElse {
-      try {
-        Process("git config --global user.name").lines.toList(0)
-      } catch {
-        case e: Throwable =>
-          readLine("Please enter author's name: ")
-      }
-    }
-
-    val organization = ca.template.packageName getOrElse {
-      readLine(
-        "Please enter the template's Scala package name (e.g. com.mycompany): ")
-    }
-
-    val email = ca.template.email getOrElse {
-      try {
-        Process("git config --global user.email").lines.toList(0)
-      } catch {
-        case e: Throwable =>
-          readLine("Please enter author's e-mail address: ")
-      }
-    }
-
-    println(s"Author's name:         $name")
-    println(s"Author's e-mail:       $email")
-    println(s"Author's organization: $organization")
-
-    var subscribe = readLine("Would you like to be informed about new bug " +
-      "fixes and security updates of this template? (Y/n) ")
-    var valid = false
-
-    do {
-      subscribe match {
-        case "" | "Y" | "y" =>
-          sub(ca.template.repository, name, email, organization)
-          valid = true
-        case "n" | "N" =>
-          meta(ca.template.repository, name, organization)
-          valid = true
-        case _ =>
-          println("Please answer 'y' or 'n'")
-          subscribe = readLine("(Y/n)? ")
-      }
-    } while (!valid)
-
-    val repo = repos(ca.template.repository)
-
-    println(s"Retrieving ${ca.template.repository}")
-    val tags = read[List[GitHubTag]](repo.body)
-    println(s"There are ${tags.size} tags")
-
-    if (tags.size == 0) {
-      println(s"${ca.template.repository} does not have any tag. Aborting.")
-      return 1
-    }
-
-    val tag = ca.template.version.map { v =>
-      tags.find(_.name == v).getOrElse {
-        println(s"${ca.template.repository} does not have tag $v. Aborting.")
-        return 1
-      }
-    } getOrElse tags.head
-
-    println(s"Using tag ${tag.name}")
-    val url =
-      s"https://github.com/${ca.template.repository}/archive/${tag.name}.zip"
-    println(s"Going to download $url")
-    val trial = try {
-      httpOptionalProxy(url).asBytes
-    } catch {
-      case e: ConnectException =>
-        githubConnectErrorMessage(e)
-        return 1
-    }
-    val finalTrial = try {
-      trial.location.map { loc =>
-        println(s"Redirecting to $loc")
-        httpOptionalProxy(loc).asBytes
-      } getOrElse trial
-    } catch {
-      case e: ConnectException =>
-        githubConnectErrorMessage(e)
-        return 1
-    }
-    val zipFilename =
-      s"${ca.template.repository.replace('/', '-')}-${tag.name}.zip"
-    FileUtils.writeByteArrayToFile(
-      new File(zipFilename),
-      finalTrial.body)
-    val zis = new ZipInputStream(
-      new BufferedInputStream(new FileInputStream(zipFilename)))
-    val bufferSize = 4096
-    val filesToModify = collection.mutable.ListBuffer[String]()
-    var ze = zis.getNextEntry
-    while (ze != null) {
-      val filenameSegments = ze.getName.split(File.separatorChar)
-      val destFilename = (ca.template.directory +: filenameSegments.tail).
-        mkString(File.separator)
-      if (ze.isDirectory) {
-        new File(destFilename).mkdirs
-      } else {
-        val os = new BufferedOutputStream(
-          new FileOutputStream(destFilename),
-          bufferSize)
-        val data = Array.ofDim[Byte](bufferSize)
-        var count = zis.read(data, 0, bufferSize)
-        while (count != -1) {
-          os.write(data, 0, count)
-          count = zis.read(data, 0, bufferSize)
-        }
-        os.flush()
-        os.close()
-
-        val nameOnly = new File(destFilename).getName
-
-        if (organization != "" &&
-          (nameOnly.endsWith(".scala") ||
-            nameOnly == "build.sbt" ||
-            nameOnly == "engine.json")) {
-          filesToModify += destFilename
-        }
-      }
-      ze = zis.getNextEntry
-    }
-    zis.close()
-    new File(zipFilename).delete
-
-    val engineJsonFile =
-      new File(ca.template.directory, "engine.json")
-
-    val engineJson = try {
-      Some(parse(Source.fromFile(engineJsonFile).mkString))
-    } catch {
-      case e: java.io.IOException =>
-        error("Unable to read engine.json. Skipping automatic package " +
-          "name replacement.")
-        None
-      case e: MappingException =>
-        error("Unable to parse engine.json. Skipping automatic package " +
-          "name replacement.")
-        None
-    }
-
-    val engineFactory = engineJson.map { ej =>
-      (ej \ "engineFactory").extractOpt[String]
-    } getOrElse None
-
-    engineFactory.map { ef =>
-      val pkgName = ef.split('.').dropRight(1).mkString(".")
-      println(s"Replacing $pkgName with $organization...")
-
-      filesToModify.foreach { ftm =>
-        println(s"Processing $ftm...")
-        val fileContent = Source.fromFile(ftm).getLines()
-        val processedLines =
-          fileContent.map(_.replaceAllLiterally(pkgName, organization))
-        FileUtils.writeStringToFile(
-          new File(ftm),
-          processedLines.mkString("\n"))
-      }
-    } getOrElse {
-      error("engineFactory is not found in engine.json. Skipping automatic " +
-        "package name replacement.")
-    }
-
-    verifyTemplateMinVersion(new File(ca.template.directory, "template.json"))
-
-    println(s"Engine template ${ca.template.repository} is now ready at " +
-      ca.template.directory)
-
-    0
-  }
-
-  def verifyTemplateMinVersion(templateJsonFile: File): Unit = {
-    val metadata = templateMetaData(templateJsonFile)
-
-    metadata.pioVersionMin.foreach { pvm =>
-      if (Version(BuildInfo.version) < Version(pvm)) {
-        error(s"This engine template requires at least PredictionIO $pvm. " +
-          s"The template may not work with PredictionIO ${BuildInfo.version}.")
-        sys.exit(1)
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
index 402e853..2e9fb26 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
@@ -55,12 +55,13 @@ object Dashboard extends Logging with SSLConfiguration {
     }
 
     parser.parse(args, DashboardConfig()) map { dc =>
-      createDashboard(dc)
+      createDashboard(dc).awaitTermination
     }
   }
 
-  def createDashboard(dc: DashboardConfig): Unit = {
-    implicit val system = ActorSystem("pio-dashboard")
+  def createDashboard(dc: DashboardConfig): ActorSystem = {
+    val systemName = "pio-dashboard"
+    implicit val system = ActorSystem(systemName)
     val service =
       system.actorOf(Props(classOf[DashboardActor], dc), "dashboard")
     implicit val timeout = Timeout(5.seconds)
@@ -72,7 +73,7 @@ object Dashboard extends Logging with SSLConfiguration {
       interface = dc.ip,
       port = dc.port,
       settings = Some(settings.copy(sslEncryption = sslEnforced)))
-    system.awaitTermination
+    system
   }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
----------------------------------------------------------------------
diff --git a/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt b/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
index f97c8ce..909e15c 100644
--- a/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
+++ b/tools/src/main/twirl/org/apache/predictionio/tools/console/template.scala.txt
@@ -1,25 +1,3 @@
 Usage: pio template list
 
-Retrieves a list of available template IDs.
-
-Usage: pio template get <template ID> <new engine directory>
-                        [--version <version>]
-                        [--name <value>] [--package <value>] [--email <value>]
-
-Seeds a directory with an engine template.
-
-  <template ID>
-      Engine template ID.
-  <new engine directory>
-      Location of the new engine.
-  --version <value>
-      The template version to get. By default, the most recently tagged version
-      will be downloaded.
-  --name <value>
-      Name of the author of the new engine.
-  --package <value>
-      Scala package name of the new engine.
-  --email <value>
-      E-mail address of the author of the new engine. Specify this if you want
-      to receive updates (critical bug fixes, etc) about the engine template
-      that you are going to use.
+No longer supported! Please use git to download and manage your templates.



[3/3] incubator-predictionio git commit: [PIO-28] Console refactor

Posted by do...@apache.org.
[PIO-28] Console refactor

Extract logic from functions handling console commands in tools package

Closes #283


Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/e4a3c0c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/e4a3c0c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/e4a3c0c9

Branch: refs/heads/develop
Commit: e4a3c0c9fc1251d7355d921acb66168226446b3f
Parents: f7c30d3
Author: Marcin Ziemi\u0144ski <zi...@gmail.com>
Authored: Tue Nov 1 10:54:07 2016 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Nov 1 10:54:07 2016 -0700

----------------------------------------------------------------------
 .../predictionio/data/api/EventServer.scala     |   5 +-
 .../org/apache/predictionio/tools/Common.scala  | 114 +++
 .../predictionio/tools/RegisterEngine.scala     |  18 +-
 .../apache/predictionio/tools/RunServer.scala   | 176 ++---
 .../apache/predictionio/tools/RunWorkflow.scala | 195 +----
 .../org/apache/predictionio/tools/Runner.scala  |  66 +-
 .../predictionio/tools/admin/AdminAPI.scala     |   5 +-
 .../predictionio/tools/commands/AccessKey.scala |  70 ++
 .../predictionio/tools/commands/App.scala       | 365 +++++++++
 .../predictionio/tools/commands/Engine.scala    | 424 +++++++++++
 .../predictionio/tools/commands/Export.scala    |  54 ++
 .../predictionio/tools/commands/Import.scala    |  51 ++
 .../tools/commands/Management.scala             | 178 +++++
 .../predictionio/tools/commands/Template.scala  |  71 ++
 .../predictionio/tools/console/AccessKey.scala  |  86 ---
 .../apache/predictionio/tools/console/App.scala | 540 --------------
 .../predictionio/tools/console/Console.scala    | 734 +++----------------
 .../predictionio/tools/console/Export.scala     |  45 --
 .../predictionio/tools/console/Import.scala     |  42 --
 .../apache/predictionio/tools/console/Pio.scala | 352 +++++++++
 .../predictionio/tools/console/Template.scala   | 432 -----------
 .../tools/dashboard/Dashboard.scala             |   9 +-
 .../tools/console/template.scala.txt            |  24 +-
 .../tools/console/upgrade.scala.txt             |  14 +-
 24 files changed, 1932 insertions(+), 2138 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
index a0ba40f..648316e 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
@@ -611,7 +611,7 @@ case class EventServerConfig(
   stats: Boolean = false)
 
 object EventServer {
-  def createEventServer(config: EventServerConfig): Unit = {
+  def createEventServer(config: EventServerConfig): ActorSystem = {
     implicit val system = ActorSystem("EventServerSystem")
 
     val eventClient = Storage.getLEvents()
@@ -630,7 +630,7 @@ object EventServer {
     if (config.stats) system.actorOf(Props[StatsActor], "StatsActor")
     system.actorOf(Props[PluginsActor], "PluginsActor")
     serverActor ! StartServer(config.ip, config.port)
-    system.awaitTermination()
+    system
   }
 }
 
@@ -639,5 +639,6 @@ object Run {
     EventServer.createEventServer(EventServerConfig(
       ip = "0.0.0.0",
       port = 7070))
+    .awaitTermination
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
new file mode 100644
index 0000000..c379138
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools
+
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.tools.ReturnTypes._
+
+import grizzled.slf4j.Logging
+import java.io.File
+
+
+object ReturnTypes {
+  sealed case class Ok()
+
+  type MaybeError = Either[String, Ok]
+  type Expected[T] = Either[String, T]
+
+  val Success: MaybeError = Right(Ok())
+}
+
+trait EitherLogging extends Logging {
+  import ReturnTypes._
+
+  protected def logAndFail[T](msg: => String): Expected[T] = {
+    error(msg)
+    Left(msg)
+  }
+
+  protected def logOnFail[T](msg: => String, t: => Throwable): Expected[T] = {
+    error(msg, t)
+    Left(msg)
+  }
+
+  protected def logAndReturn[T](value: T, msg: => Any): Expected[T] = {
+    info(msg)
+    Right(value)
+  }
+
+  protected def logAndSucceed(msg: => Any): MaybeError = {
+    info(msg)
+    Success
+  }
+}
+
+object Common extends EitherLogging {
+
+  def getSparkHome(sparkHome: Option[String]): String = {
+    sparkHome getOrElse {
+      sys.env.getOrElse("SPARK_HOME", ".")
+    }
+  }
+
+  def versionNoPatch(fullVersion: String): String = {
+    val v = """^(\d+\.\d+)""".r
+    val versionNoPatch = for {
+      v(np) <- v findFirstIn fullVersion
+    } yield np
+    versionNoPatch.getOrElse(fullVersion)
+  }
+
+  def jarFilesForScala: Array[File] = {
+    def recursiveListFiles(f: File): Array[File] = {
+      Option(f.listFiles) map { these =>
+        these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
+      } getOrElse Array[File]()
+    }
+    def jarFilesForScalaFilter(jars: Array[File]): Array[File] =
+      jars.filterNot { f =>
+        f.getName.toLowerCase.endsWith("-javadoc.jar") ||
+        f.getName.toLowerCase.endsWith("-sources.jar")
+      }
+    def jarFilesAt(path: File): Array[File] = recursiveListFiles(path) filter {
+      _.getName.toLowerCase.endsWith(".jar")
+    }
+    val libFiles = jarFilesForScalaFilter(jarFilesAt(new File("lib")))
+    val scalaVersionNoPatch = Common.versionNoPatch(BuildInfo.scalaVersion)
+    val targetFiles = jarFilesForScalaFilter(jarFilesAt(new File("target" +
+      File.separator + s"scala-${scalaVersionNoPatch}")))
+    // Use libFiles is target is empty.
+    if (targetFiles.size > 0) targetFiles else libFiles
+  }
+
+  def coreAssembly(pioHome: String): Expected[File] = {
+    val core = s"pio-assembly-${BuildInfo.version}.jar"
+    val coreDir =
+      if (new File(pioHome + File.separator + "RELEASE").exists) {
+        new File(pioHome + File.separator + "lib")
+      } else {
+        new File(pioHome + File.separator + "assembly")
+      }
+    val coreFile = new File(coreDir, core)
+    if (coreFile.exists) {
+      Right(coreFile)
+    } else {
+      logAndFail(s"PredictionIO Core Assembly (${coreFile.getCanonicalPath}) does " +
+        "not exist. Aborting.")
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala b/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
index 189e5a8..334981c 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RegisterEngine.scala
@@ -24,6 +24,7 @@ import grizzled.slf4j.Logging
 import org.apache.predictionio.data.storage.EngineManifest
 import org.apache.predictionio.data.storage.EngineManifestSerializer
 import org.apache.predictionio.data.storage.Storage
+import org.apache.predictionio.tools.ReturnTypes._
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.FileSystem
 import org.apache.hadoop.fs.Path
@@ -32,35 +33,34 @@ import org.json4s.native.Serialization.read
 
 import scala.io.Source
 
-object RegisterEngine extends Logging {
+object RegisterEngine extends EitherLogging {
   val engineManifests = Storage.getMetaDataEngineManifests
   implicit val formats = DefaultFormats + new EngineManifestSerializer
 
   def registerEngine(
       jsonManifest: File,
       engineFiles: Seq[File],
-      copyLocal: Boolean = false): Unit = {
+      copyLocal: Boolean = false): MaybeError = {
     val jsonString = try {
       Source.fromFile(jsonManifest).mkString
     } catch {
       case e: java.io.FileNotFoundException =>
-        error(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
-        sys.exit(1)
+        return logAndFail(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
     }
     val engineManifest = read[EngineManifest](jsonString)
 
     info(s"Registering engine ${engineManifest.id} ${engineManifest.version}")
     engineManifests.update(
       engineManifest.copy(files = engineFiles.map(_.toURI.toString)), true)
+    Success
   }
 
-  def unregisterEngine(jsonManifest: File): Unit = {
+  def unregisterEngine(jsonManifest: File): MaybeError = {
     val jsonString = try {
       Source.fromFile(jsonManifest).mkString
     } catch {
       case e: java.io.FileNotFoundException =>
-        error(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
-        sys.exit(1)
+        return logAndFail(s"Engine manifest file not found: ${e.getMessage}. Aborting.")
     }
     val fileEngineManifest = read[EngineManifest](jsonString)
     val engineManifest = engineManifests.get(
@@ -78,9 +78,9 @@ object RegisterEngine extends Logging {
       }
 
       engineManifests.delete(em.id, em.version)
-      info(s"Unregistered engine ${em.id} ${em.version}")
+      logAndSucceed(s"Unregistered engine ${em.id} ${em.version}")
     } getOrElse {
-      error(s"${fileEngineManifest.id} ${fileEngineManifest.version} is not " +
+      logAndFail(s"${fileEngineManifest.id} ${fileEngineManifest.version} is not " +
         "registered.")
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
index d091654..1431432 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
@@ -23,159 +23,75 @@ import java.net.URI
 
 import grizzled.slf4j.Logging
 import org.apache.predictionio.data.storage.EngineManifest
+import org.apache.predictionio.tools.ReturnTypes._
 import org.apache.predictionio.tools.console.ConsoleArgs
 import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.workflow.JsonExtractorOption
+import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
 
 import scala.sys.process._
 
-object RunServer extends Logging {
-  def runServer(
-      ca: ConsoleArgs,
-      core: File,
-      em: EngineManifest,
-      engineInstanceId: String): Int = {
-    val pioEnvVars = sys.env.filter(kv => kv._1.startsWith("PIO_")).map(kv =>
-      s"${kv._1}=${kv._2}"
-    ).mkString(",")
-
-    val sparkHome = ca.common.sparkHome.getOrElse(
-      sys.env.getOrElse("SPARK_HOME", "."))
-
-    val extraFiles = WorkflowUtils.thirdPartyConfFiles
-
-    val driverClassPathIndex =
-      ca.common.sparkPassThrough.indexOf("--driver-class-path")
-    val driverClassPathPrefix =
-      if (driverClassPathIndex != -1) {
-        Seq(ca.common.sparkPassThrough(driverClassPathIndex + 1))
-      } else {
-        Seq()
-      }
-    val extraClasspaths =
-      driverClassPathPrefix ++ WorkflowUtils.thirdPartyClasspaths
+case class DeployArgs(
+  ip: String = "0.0.0.0",
+  port: Int = 8000,
+  logUrl: Option[String] = None,
+  logPrefix: Option[String] = None)
 
-    val deployModeIndex =
-      ca.common.sparkPassThrough.indexOf("--deploy-mode")
-    val deployMode = if (deployModeIndex != -1) {
-      ca.common.sparkPassThrough(deployModeIndex + 1)
-    } else {
-      "client"
-    }
+case class EventServerArgs(
+  enabled: Boolean = false,
+  ip: String = "0.0.0.0",
+  port: Int = 7070,
+  stats: Boolean = false)
 
-    val mainJar =
-      if (ca.build.uberJar) {
-        if (deployMode == "cluster") {
-          em.files.filter(_.startsWith("hdfs")).head
-        } else {
-          em.files.filterNot(_.startsWith("hdfs")).head
-        }
-      } else {
-        if (deployMode == "cluster") {
-          em.files.filter(_.contains("pio-assembly")).head
-        } else {
-          core.getCanonicalPath
-        }
-      }
-
-    val jarFiles = (em.files ++ Option(new File(ca.common.pioHome.get, "plugins")
-      .listFiles()).getOrElse(Array.empty[File]).map(_.getAbsolutePath)).mkString(",")
-
-    val sparkSubmit =
-      Seq(Seq(sparkHome, "bin", "spark-submit").mkString(File.separator)) ++
-      ca.common.sparkPassThrough ++
-      Seq(
-        "--class",
-        "org.apache.predictionio.workflow.CreateServer",
-        "--name",
-        s"PredictionIO Engine Instance: ${engineInstanceId}") ++
-      (if (!ca.build.uberJar) {
-        Seq("--jars", jarFiles)
-      } else Seq()) ++
-      (if (extraFiles.size > 0) {
-        Seq("--files", extraFiles.mkString(","))
-      } else {
-        Seq()
-      }) ++
-      (if (extraClasspaths.size > 0) {
-        Seq("--driver-class-path", extraClasspaths.mkString(":"))
-      } else {
-        Seq()
-      }) ++
-      (if (ca.common.sparkKryo) {
-        Seq(
-          "--conf",
-          "spark.serializer=org.apache.spark.serializer.KryoSerializer")
-      } else {
-        Seq()
-      }) ++
-      Seq(
-        mainJar,
-        "--engineInstanceId",
-        engineInstanceId,
-        "--ip",
-        ca.deploy.ip,
-        "--port",
-        ca.deploy.port.toString,
-        "--event-server-ip",
-        ca.eventServer.ip,
-        "--event-server-port",
-        ca.eventServer.port.toString) ++
-      (if (ca.accessKey.accessKey != "") {
-        Seq("--accesskey", ca.accessKey.accessKey)
-      } else {
-        Seq()
-      }) ++
-      (if (ca.eventServer.enabled) Seq("--feedback") else Seq()) ++
-      (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
-      (if (ca.common.verbose) Seq("--verbose") else Seq()) ++
-      ca.deploy.logUrl.map(x => Seq("--log-url", x)).getOrElse(Seq()) ++
-      ca.deploy.logPrefix.map(x => Seq("--log-prefix", x)).getOrElse(Seq()) ++
-      Seq("--json-extractor", ca.common.jsonExtractor.toString)
+case class ServerArgs(
+  deploy: DeployArgs = DeployArgs(),
+  eventServer: EventServerArgs = EventServerArgs(),
+  batch: String = "",
+  accessKey: String = "",
+  variantJson: File = new File("engine.json"),
+  jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
 
-    info(s"Submission command: ${sparkSubmit.mkString(" ")}")
 
-    val proc =
-      Process(sparkSubmit, None, "CLASSPATH" -> "", "SPARK_YARN_USER_ENV" -> pioEnvVars).run()
-    Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
-      def run(): Unit = {
-        proc.destroy()
-      }
-    }))
-    proc.exitValue()
-  }
+object RunServer extends Logging {
 
-  def newRunServer(
-    ca: ConsoleArgs,
+  def runServer(
+    engineInstanceId: String,
+    serverArgs: ServerArgs,
+    sparkArgs: SparkArgs,
     em: EngineManifest,
-    engineInstanceId: String): Int = {
+    pioHome: String,
+    verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
     val jarFiles = em.files.map(new URI(_)) ++
-      Option(new File(ca.common.pioHome.get, "plugins").listFiles())
+      Option(new File(pioHome, "plugins").listFiles())
         .getOrElse(Array.empty[File]).map(_.toURI)
     val args = Seq(
       "--engineInstanceId",
       engineInstanceId,
       "--engine-variant",
-      ca.common.variantJson.toURI.toString,
+      serverArgs.variantJson.toURI.toString,
       "--ip",
-      ca.deploy.ip,
+      serverArgs.deploy.ip,
       "--port",
-      ca.deploy.port.toString,
+      serverArgs.deploy.port.toString,
       "--event-server-ip",
-      ca.eventServer.ip,
+      serverArgs.eventServer.ip,
       "--event-server-port",
-      ca.eventServer.port.toString) ++
-      (if (ca.accessKey.accessKey != "") {
-        Seq("--accesskey", ca.accessKey.accessKey)
+      serverArgs.eventServer.port.toString) ++
+      (if (serverArgs.accessKey != "") {
+        Seq("--accesskey", serverArgs.accessKey)
       } else {
         Nil
       }) ++
-      (if (ca.eventServer.enabled) Seq("--feedback") else Nil) ++
-      (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Nil) ++
-      (if (ca.common.verbose) Seq("--verbose") else Nil) ++
-      ca.deploy.logUrl.map(x => Seq("--log-url", x)).getOrElse(Nil) ++
-      ca.deploy.logPrefix.map(x => Seq("--log-prefix", x)).getOrElse(Nil) ++
-      Seq("--json-extractor", ca.common.jsonExtractor.toString)
-
-    Runner.runOnSpark("org.apache.predictionio.workflow.CreateServer", args, ca, jarFiles)
+      (if (serverArgs.eventServer.enabled) Seq("--feedback") else Nil) ++
+      (if (serverArgs.batch != "") Seq("--batch", serverArgs.batch) else Nil) ++
+      (if (verbose) Seq("--verbose") else Nil) ++
+      serverArgs.deploy.logUrl.map(x => Seq("--log-url", x)).getOrElse(Nil) ++
+      serverArgs.deploy.logPrefix.map(x => Seq("--log-prefix", x)).getOrElse(Nil) ++
+      Seq("--json-extractor", serverArgs.jsonExtractor.toString)
+
+    Runner.runOnSpark(
+      "org.apache.predictionio.workflow.CreateServer",
+      args, sparkArgs, jarFiles, pioHome, verbose)
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
index 097eb83..8b8d769 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
@@ -24,156 +24,39 @@ import java.net.URI
 import grizzled.slf4j.Logging
 import org.apache.predictionio.data.storage.EngineManifest
 import org.apache.predictionio.tools.console.ConsoleArgs
+import org.apache.predictionio.tools.ReturnTypes._
 import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.workflow.JsonExtractorOption
+import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
+
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.FileSystem
 import org.apache.hadoop.fs.Path
 
 import scala.sys.process._
 
-object RunWorkflow extends Logging {
-  def runWorkflow(
-      ca: ConsoleArgs,
-      core: File,
-      em: EngineManifest,
-      variantJson: File): Int = {
-    // Collect and serialize PIO_* environmental variables
-    val pioEnvVars = sys.env.filter(kv => kv._1.startsWith("PIO_")).map(kv =>
-      s"${kv._1}=${kv._2}"
-    ).mkString(",")
-
-    val sparkHome = ca.common.sparkHome.getOrElse(
-      sys.env.getOrElse("SPARK_HOME", "."))
-
-    val hadoopConf = new Configuration
-    val hdfs = FileSystem.get(hadoopConf)
-
-    val driverClassPathIndex =
-      ca.common.sparkPassThrough.indexOf("--driver-class-path")
-    val driverClassPathPrefix =
-      if (driverClassPathIndex != -1) {
-        Seq(ca.common.sparkPassThrough(driverClassPathIndex + 1))
-      } else {
-        Seq()
-      }
-    val extraClasspaths =
-      driverClassPathPrefix ++ WorkflowUtils.thirdPartyClasspaths
-
-    val deployModeIndex =
-      ca.common.sparkPassThrough.indexOf("--deploy-mode")
-    val deployMode = if (deployModeIndex != -1) {
-      ca.common.sparkPassThrough(deployModeIndex + 1)
-    } else {
-      "client"
-    }
-
-    val extraFiles = WorkflowUtils.thirdPartyConfFiles
+case class WorkflowArgs(
+  batch: String = "",
+  variantJson: File = new File("engine.json"),
+  verbosity: Int = 0,
+  engineParamsKey: Option[String] = None,
+  engineFactory: Option[String] = None,
+  evaluation: Option[String] = None,
+  engineParamsGenerator: Option[String] = None,
+  stopAfterRead: Boolean = false,
+  stopAfterPrepare: Boolean = false,
+  skipSanityCheck: Boolean = false,
+  jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
 
-    val mainJar =
-      if (ca.build.uberJar) {
-        if (deployMode == "cluster") {
-          em.files.filter(_.startsWith("hdfs")).head
-        } else {
-          em.files.filterNot(_.startsWith("hdfs")).head
-        }
-      } else {
-        if (deployMode == "cluster") {
-          em.files.filter(_.contains("pio-assembly")).head
-        } else {
-          core.getCanonicalPath
-        }
-      }
-
-    val workMode =
-      ca.common.evaluation.map(_ => "Evaluation").getOrElse("Training")
-
-    val engineLocation = Seq(
-      sys.env("PIO_FS_ENGINESDIR"),
-      em.id,
-      em.version)
-
-    if (deployMode == "cluster") {
-      val dstPath = new Path(engineLocation.mkString(Path.SEPARATOR))
-      info("Cluster deploy mode detected. Trying to copy " +
-        s"${variantJson.getCanonicalPath} to " +
-        s"${hdfs.makeQualified(dstPath).toString}.")
-      hdfs.copyFromLocalFile(new Path(variantJson.toURI), dstPath)
-    }
-
-    val sparkSubmit =
-      Seq(Seq(sparkHome, "bin", "spark-submit").mkString(File.separator)) ++
-      ca.common.sparkPassThrough ++
-      Seq(
-        "--class",
-        "org.apache.predictionio.workflow.CreateWorkflow",
-        "--name",
-        s"PredictionIO $workMode: ${em.id} ${em.version} (${ca.common.batch})") ++
-      (if (!ca.build.uberJar) {
-        Seq("--jars", em.files.mkString(","))
-      } else Seq()) ++
-      (if (extraFiles.size > 0) {
-        Seq("--files", extraFiles.mkString(","))
-      } else {
-        Seq()
-      }) ++
-      (if (extraClasspaths.size > 0) {
-        Seq("--driver-class-path", extraClasspaths.mkString(":"))
-      } else {
-        Seq()
-      }) ++
-      (if (ca.common.sparkKryo) {
-        Seq(
-          "--conf",
-          "spark.serializer=org.apache.spark.serializer.KryoSerializer")
-      } else {
-        Seq()
-      }) ++
-      Seq(
-        mainJar,
-        "--env",
-        pioEnvVars,
-        "--engine-id",
-        em.id,
-        "--engine-version",
-        em.version,
-        "--engine-variant",
-        if (deployMode == "cluster") {
-          hdfs.makeQualified(new Path(
-            (engineLocation :+ variantJson.getName).mkString(Path.SEPARATOR))).
-            toString
-        } else {
-          variantJson.getCanonicalPath
-        },
-        "--verbosity",
-        ca.common.verbosity.toString) ++
-      ca.common.engineFactory.map(
-        x => Seq("--engine-factory", x)).getOrElse(Seq()) ++
-      ca.common.engineParamsKey.map(
-        x => Seq("--engine-params-key", x)).getOrElse(Seq()) ++
-      (if (deployMode == "cluster") Seq("--deploy-mode", "cluster") else Seq()) ++
-      (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
-      (if (ca.common.verbose) Seq("--verbose") else Seq()) ++
-      (if (ca.common.skipSanityCheck) Seq("--skip-sanity-check") else Seq()) ++
-      (if (ca.common.stopAfterRead) Seq("--stop-after-read") else Seq()) ++
-      (if (ca.common.stopAfterPrepare) {
-        Seq("--stop-after-prepare")
-      } else {
-        Seq()
-      }) ++
-      ca.common.evaluation.map(x => Seq("--evaluation-class", x)).
-        getOrElse(Seq()) ++
-      // If engineParamsGenerator is specified, it overrides the evaluation.
-      ca.common.engineParamsGenerator.orElse(ca.common.evaluation)
-        .map(x => Seq("--engine-params-generator-class", x))
-        .getOrElse(Seq()) ++
-      (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
-      Seq("--json-extractor", ca.common.jsonExtractor.toString)
+object RunWorkflow extends Logging {
 
-    info(s"Submission command: ${sparkSubmit.mkString(" ")}")
-    Process(sparkSubmit, None, "CLASSPATH" -> "", "SPARK_YARN_USER_ENV" -> pioEnvVars).!
-  }
+  def runWorkflow(
+    wa: WorkflowArgs,
+    sa: SparkArgs,
+    em: EngineManifest,
+    pioHome: String,
+    verbose: Boolean = false): Expected[(Process, () => Unit)] = {
 
-  def newRunWorkflow(ca: ConsoleArgs, em: EngineManifest): Int = {
     val jarFiles = em.files.map(new URI(_))
     val args = Seq(
       "--engine-id",
@@ -181,35 +64,37 @@ object RunWorkflow extends Logging {
       "--engine-version",
       em.version,
       "--engine-variant",
-      ca.common.variantJson.toURI.toString,
+      wa.variantJson.toURI.toString,
       "--verbosity",
-      ca.common.verbosity.toString) ++
-      ca.common.engineFactory.map(
+      wa.verbosity.toString) ++
+      wa.engineFactory.map(
         x => Seq("--engine-factory", x)).getOrElse(Seq()) ++
-      ca.common.engineParamsKey.map(
+      wa.engineParamsKey.map(
         x => Seq("--engine-params-key", x)).getOrElse(Seq()) ++
-      (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
-      (if (ca.common.verbose) Seq("--verbose") else Seq()) ++
-      (if (ca.common.skipSanityCheck) Seq("--skip-sanity-check") else Seq()) ++
-      (if (ca.common.stopAfterRead) Seq("--stop-after-read") else Seq()) ++
-      (if (ca.common.stopAfterPrepare) {
+      (if (wa.batch != "") Seq("--batch", wa.batch) else Seq()) ++
+      (if (verbose) Seq("--verbose") else Seq()) ++
+      (if (wa.skipSanityCheck) Seq("--skip-sanity-check") else Seq()) ++
+      (if (wa.stopAfterRead) Seq("--stop-after-read") else Seq()) ++
+      (if (wa.stopAfterPrepare) {
         Seq("--stop-after-prepare")
       } else {
         Seq()
       }) ++
-      ca.common.evaluation.map(x => Seq("--evaluation-class", x)).
+      wa.evaluation.map(x => Seq("--evaluation-class", x)).
         getOrElse(Seq()) ++
       // If engineParamsGenerator is specified, it overrides the evaluation.
-      ca.common.engineParamsGenerator.orElse(ca.common.evaluation)
+      wa.engineParamsGenerator.orElse(wa.evaluation)
         .map(x => Seq("--engine-params-generator-class", x))
         .getOrElse(Seq()) ++
-      (if (ca.common.batch != "") Seq("--batch", ca.common.batch) else Seq()) ++
-      Seq("--json-extractor", ca.common.jsonExtractor.toString)
+      (if (wa.batch != "") Seq("--batch", wa.batch) else Seq()) ++
+      Seq("--json-extractor", wa.jsonExtractor.toString)
 
     Runner.runOnSpark(
       "org.apache.predictionio.workflow.CreateWorkflow",
       args,
-      ca,
-      jarFiles)
+      sa,
+      jarFiles,
+      pioHome,
+      verbose)
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index b3ec51c..d9752df 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -21,16 +21,22 @@ package org.apache.predictionio.tools
 import java.io.File
 import java.net.URI
 
-import grizzled.slf4j.Logging
 import org.apache.predictionio.tools.console.ConsoleArgs
 import org.apache.predictionio.workflow.WorkflowUtils
+import org.apache.predictionio.tools.ReturnTypes._
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.FileSystem
 import org.apache.hadoop.fs.Path
 
 import scala.sys.process._
 
-object Runner extends Logging {
+case class SparkArgs(
+  sparkHome: Option[String] = None,
+  sparkPassThrough: Seq[String] = Seq(),
+  sparkKryo: Boolean = false,
+  scratchUri: Option[URI] = None)
+
+object Runner extends EitherLogging {
   def envStringToMap(env: String): Map[String, String] =
     env.split(',').flatMap(p =>
       p.split('=') match {
@@ -95,26 +101,27 @@ object Runner extends Logging {
   def runOnSpark(
       className: String,
       classArgs: Seq[String],
-      ca: ConsoleArgs,
-      extraJars: Seq[URI]): Int = {
+      sa: SparkArgs,
+      extraJars: Seq[URI],
+      pioHome: String,
+      verbose: Boolean = false): Expected[(Process, () => Unit)] = {
     // Return error for unsupported cases
     val deployMode =
-      argumentValue(ca.common.sparkPassThrough, "--deploy-mode").getOrElse("client")
+      argumentValue(sa.sparkPassThrough, "--deploy-mode").getOrElse("client")
     val master =
-      argumentValue(ca.common.sparkPassThrough, "--master").getOrElse("local")
+      argumentValue(sa.sparkPassThrough, "--master").getOrElse("local")
 
-    (ca.common.scratchUri, deployMode, master) match {
+    (sa.scratchUri, deployMode, master) match {
       case (Some(u), "client", m) if m != "yarn-cluster" =>
-        error("--scratch-uri cannot be set when deploy mode is client")
-        return 1
+        return logAndFail("--scratch-uri cannot be set when deploy mode is client")
       case (_, "cluster", m) if m.startsWith("spark://") =>
-        error("Using cluster deploy mode with Spark standalone cluster is not supported")
-        return 1
+        return logAndFail(
+          "Using cluster deploy mode with Spark standalone cluster is not supported")
       case _ => Unit
     }
 
     // Initialize HDFS API for scratch URI
-    val fs = ca.common.scratchUri map { uri =>
+    val fs = sa.scratchUri map { uri =>
       FileSystem.get(uri, new Configuration())
     }
 
@@ -124,18 +131,18 @@ object Runner extends Logging {
     ).mkString(",")
 
     // Location of Spark
-    val sparkHome = ca.common.sparkHome.getOrElse(
+    val sparkHome = sa.sparkHome.getOrElse(
       sys.env.getOrElse("SPARK_HOME", "."))
 
     // Local path to PredictionIO assembly JAR
-    val mainJar = handleScratchFile(
-      fs,
-      ca.common.scratchUri,
-      console.Console.coreAssembly(ca.common.pioHome.get))
+    val mainJar = Common.coreAssembly(pioHome) fold(
+        errStr => return Left(errStr),
+        assembly => handleScratchFile(fs, sa.scratchUri, assembly)
+      )
 
     // Extra JARs that are needed by the driver
     val driverClassPathPrefix =
-      argumentValue(ca.common.sparkPassThrough, "--driver-class-path") map { v =>
+      argumentValue(sa.sparkPassThrough, "--driver-class-path") map { v =>
         Seq(v)
       } getOrElse {
         Nil
@@ -146,11 +153,11 @@ object Runner extends Logging {
 
     // Extra files that are needed to be passed to --files
     val extraFiles = WorkflowUtils.thirdPartyConfFiles map { f =>
-      handleScratchFile(fs, ca.common.scratchUri, new File(f))
+      handleScratchFile(fs, sa.scratchUri, new File(f))
     }
 
     val deployedJars = extraJars map { j =>
-      handleScratchFile(fs, ca.common.scratchUri, new File(j))
+      handleScratchFile(fs, sa.scratchUri, new File(j))
     }
 
     val sparkSubmitCommand =
@@ -174,7 +181,7 @@ object Runner extends Logging {
       Nil
     }
 
-    val sparkSubmitKryo = if (ca.common.sparkKryo) {
+    val sparkSubmitKryo = if (sa.sparkKryo) {
       Seq(
         "--conf",
         "spark.serializer=org.apache.spark.serializer.KryoSerializer")
@@ -182,33 +189,26 @@ object Runner extends Logging {
       Nil
     }
 
-    val verbose = if (ca.common.verbose) Seq("--verbose") else Nil
+    val verboseArg = if (verbose) Seq("--verbose") else Nil
 
     val sparkSubmit = Seq(
       sparkSubmitCommand,
-      ca.common.sparkPassThrough,
+      sa.sparkPassThrough,
       Seq("--class", className),
       sparkSubmitJars,
       sparkSubmitFiles,
       sparkSubmitExtraClasspaths,
       sparkSubmitKryo,
       Seq(mainJar),
-      detectFilePaths(fs, ca.common.scratchUri, classArgs),
+      detectFilePaths(fs, sa.scratchUri, classArgs),
       Seq("--env", pioEnvVars),
-      verbose).flatten.filter(_ != "")
+      verboseArg).flatten.filter(_ != "")
     info(s"Submission command: ${sparkSubmit.mkString(" ")}")
     val proc = Process(
       sparkSubmit,
       None,
       "CLASSPATH" -> "",
       "SPARK_YARN_USER_ENV" -> pioEnvVars).run()
-    Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
-      def run(): Unit = {
-        cleanup(fs, ca.common.scratchUri)
-        proc.destroy()
-      }
-    }))
-    cleanup(fs, ca.common.scratchUri)
-    proc.exitValue()
+    Right((proc, () => cleanup(fs, sa.scratchUri)))
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala b/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
index f53d84c..bbe39a5 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
@@ -133,7 +133,7 @@ case class AdminServerConfig(
 )
 
 object AdminServer {
-  def createAdminServer(config: AdminServerConfig): Unit = {
+  def createAdminServer(config: AdminServerConfig): ActorSystem = {
     implicit val system = ActorSystem("AdminServerSystem")
 
     val commandClient = new CommandClient(
@@ -146,7 +146,7 @@ object AdminServer {
       Props(classOf[AdminServerActor], commandClient),
       "AdminServerActor")
     serverActor ! StartServer(config.ip, config.port)
-    system.awaitTermination
+    system
   }
 }
 
@@ -155,5 +155,6 @@ object AdminRun {
     AdminServer.createAdminServer(AdminServerConfig(
       ip = "localhost",
       port = 7071))
+    .awaitTermination
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
new file mode 100644
index 0000000..715cf8f
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.tools
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.ReturnTypes._
+
+import grizzled.slf4j.Logging
+import scala.util.Either
+
+object AccessKey extends EitherLogging {
+
+  def create(
+    appName: String,
+    key: String,
+    events: Seq[String]): Expected[storage.AccessKey] = {
+
+    val apps = storage.Storage.getMetaDataApps
+    apps.getByName(appName) map { app =>
+      val accessKeys = storage.Storage.getMetaDataAccessKeys
+      val newKey = storage.AccessKey(
+        key = key,
+        appid = app.id,
+        events = events)
+      accessKeys.insert(newKey) map { k =>
+        info(s"Created new access key: ${k}")
+        Right(newKey.copy(key = k))
+      } getOrElse {
+        logAndFail(s"Unable to create new access key.")
+      }
+    } getOrElse {
+      logAndFail(s"App ${appName} does not exist. Aborting.")
+    }
+  }
+
+  def list(app: Option[String]): Expected[Seq[storage.AccessKey]] =
+    app map { appName =>
+      App.show(appName).right map { appChansPair => appChansPair._1.keys }
+    } getOrElse {
+      Right(storage.Storage.getMetaDataAccessKeys.getAll)
+    }
+
+  def delete(key: String): MaybeError = {
+    try {
+      storage.Storage.getMetaDataAccessKeys.delete(key)
+      logAndSucceed(s"Deleted access key ${key}.")
+    } catch {
+      case e: Exception =>
+        error(s"Error deleting access key ${key}.", e)
+        Left(s"Error deleting access key ${key}.")
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
new file mode 100644
index 0000000..44fa667
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.data.storage.Channel
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.ReturnTypes._
+
+sealed case class AppDescription(
+  app: storage.App,
+  keys: Seq[storage.AccessKey])
+
+object App extends EitherLogging {
+
+  def create(
+    name: String,
+    id: Option[Int] = None,
+    description: Option[String] = None,
+    accessKey: String = "") : Expected[AppDescription] = {
+
+    val apps = storage.Storage.getMetaDataApps()
+    // get the client in the beginning so error exit right away if can't access client
+    val events = storage.Storage.getLEvents()
+    var errStr = ""
+
+    apps.getByName(name) map { app =>
+      errStr = s"App ${name} already exists. Aborting."
+      error(errStr)
+      errStr
+    } orElse {
+      id.flatMap { id =>
+        apps.get(id) map { app =>
+          errStr = s"App ID ${id} already exists and maps to the app '${app.name}'. " +
+            "Aborting."
+          error(errStr)
+          errStr
+        }
+      }
+    } map {err => Left(err)} getOrElse {
+      val newApp = storage.App(
+        id = id.getOrElse(0),
+        name = name,
+        description = description)
+      val appid = apps.insert(newApp)
+
+      appid map { id =>
+        val dbInit = events.init(id)
+        val r = if (dbInit) {
+          info(s"Initialized Event Store for this app ID: ${id}.")
+          val accessKeys = storage.Storage.getMetaDataAccessKeys
+          val newKey = storage.AccessKey(
+            key = accessKey,
+            appid = id,
+            events = Seq())
+          accessKeys.insert(newKey)
+          .map { k =>
+            Right(AppDescription(
+              app = newApp.copy(id = id),
+              keys = Seq(newKey.copy(key = k))))
+          } getOrElse {
+            logAndFail(s"Unable to create new access key.")
+          }
+        } else {
+          errStr = s"Unable to initialize Event Store for this app ID: ${id}."
+          try {
+            apps.delete(id)
+          } catch {
+            case e: Exception =>
+              errStr += s"""
+                |Failed to revert back the App meta-data change.
+                |The app ${name} CANNOT be used!
+                |Please run 'pio app delete ${name}' to delete this app!"""
+          }
+          logAndFail(errStr)
+        }
+        events.close()
+        r
+      } getOrElse {
+        logAndFail(s"Unable to create new app.")
+      }
+    }
+  }
+
+  def list: Seq[AppDescription] = {
+    val apps = storage.Storage.getMetaDataApps.getAll().sortBy(_.name)
+    val accessKeys = storage.Storage.getMetaDataAccessKeys
+
+    apps map { app =>
+      AppDescription(
+        app = app,
+        keys = accessKeys.getByAppid(app.id))
+    }
+  }
+
+  def show(appName: String): Expected[(AppDescription, Seq[Channel])] = {
+    val apps = storage.Storage.getMetaDataApps
+    val accessKeys = storage.Storage.getMetaDataAccessKeys
+    val channels = storage.Storage.getMetaDataChannels
+
+    apps.getByName(appName) map { app =>
+      Right(
+        (AppDescription(
+          app = app,
+          keys = accessKeys.getByAppid(app.id)),
+        channels.getByAppid(app.id))
+      )
+    } getOrElse {
+      logAndFail(s"App ${appName} does not exist. Aborting.")
+    }
+  }
+
+  def delete(name: String): MaybeError = {
+    val events = storage.Storage.getLEvents()
+    try {
+      show(name).right.flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+
+        val delChannelStatus: MaybeError =
+          channels.map { ch =>
+            if (events.remove(appDesc.app.id, Some(ch.id))) {
+              info(s"Removed Event Store of the channel ID: ${ch.id}")
+              try {
+                storage.Storage.getMetaDataChannels.delete(ch.id)
+                info(s"Deleted channel ${ch.name}")
+                None
+              } catch {
+                case e: Exception =>
+                  val errStr = s"Error deleting channel ${ch.name}."
+                  error(errStr, e)
+                  Some(errStr)
+              }
+              } else {
+                val errStr = s"Error removing Event Store of the channel ID: ${ch.id}."
+                error(errStr)
+                Some(errStr)
+              }
+          }
+          .flatten
+          .reduceOption(_ + "\n" + _)
+          .map(Left(_)) getOrElse Success
+
+          if (delChannelStatus.isLeft) {
+            return delChannelStatus
+          }
+
+          try {
+            events.remove(appDesc.app.id)
+            info(s"Removed Event Store for this app ID: ${appDesc.app.id}")
+          } catch {
+            case e: Exception =>
+              logAndFail(s"Error removing Event Store for this app. Aborting.")
+          }
+
+          appDesc.keys foreach { key =>
+            try {
+              storage.Storage.getMetaDataAccessKeys.delete(key.key)
+              info(s"Removed access key ${key.key}")
+            } catch {
+              case e: Exception =>
+                logAndFail(s"Error removing access key ${key.key}. Aborting.")
+            }
+          }
+
+          try {
+            storage.Storage.getMetaDataApps.delete(appDesc.app.id)
+            info(s"Deleted app ${appDesc.app.name}.")
+          } catch {
+            case e: Exception =>
+              logAndFail(s"Error deleting app ${appDesc.app.name}. Aborting.")
+          }
+          logAndSucceed("Done.")
+      }
+
+    } finally {
+      events.close()
+    }
+  }
+
+  def dataDelete(
+    name: String,
+    channel: Option[String] = None,
+    all: Boolean = false): MaybeError = {
+
+    var errStr = ""
+    val events = storage.Storage.getLEvents()
+    try {
+      show(name).right.flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+
+        val chanIdsToRemove: Seq[Option[Int]] =
+          if (all) {
+            channels.map(ch => Some(ch.id)) :+ None // remove default channel too
+          } else {
+            channel.map { chName =>
+              channels.find(ch => ch.name == chName) match {
+                case None =>
+                  return logAndFail(s"""Unable to delete data for channel.
+                              |Channel ${chName} doesn't exist.""")
+                case Some(ch) => Seq(Some(ch.id))
+              }
+              } getOrElse {
+                Seq(None) // for default channel
+              }
+          }
+
+        chanIdsToRemove.map { chId: Option[Int] =>
+
+          val r1 = if (events.remove(appDesc.app.id, chId)) {
+            if (chId.isDefined) {
+              info(s"Removed Event Store for the channel ID: ${chId.get}")
+            } else {
+              info(s"Removed Event Store for the app ID: ${appDesc.app.id}")
+            }
+            None
+          } else {
+            errStr =
+              if (chId.isDefined) s"Error removing Event Store for the channel ID: ${chId.get}."
+              else s"Error removing Event Store for the app ID: ${appDesc.app.id}."
+            error(errStr)
+            Some(errStr)
+          }
+          // re-create table
+          val dbInit = events.init(appDesc.app.id, chId)
+          val r2 = if (dbInit) {
+            if (chId.isDefined) {
+              info(s"Initialized Event Store for the channel ID: ${chId.get}")
+            } else {
+              info(s"Initialized Event Store for the app ID: ${appDesc.app.id}")
+            }
+            None
+          } else {
+            errStr =
+              if (chId.isDefined) {
+                s"Unable to initialize Event Store for the channel ID: ${chId.get}."
+              }
+              else {
+                s"Unable to initialize Event tore for the app ID: ${appDesc.app.id}."
+              }
+            error(errStr)
+            Some(errStr)
+          }
+          Seq(r1, r2)
+        }
+        .flatten.flatten
+        .reduceOption(_ + "\n" + _)
+        .toLeft(Ok())
+      }
+    } finally {
+      events.close()
+    }
+  }
+
+  def channelNew(appName: String, newChannel: String): Expected[Channel] = {
+    val events = storage.Storage.getLEvents()
+    val chanStorage = storage.Storage.getMetaDataChannels
+    var errStr = ""
+    try {
+      show(appName).right flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+        if (channels.find(ch => ch.name == newChannel).isDefined) {
+          logAndFail(s"""Channel ${newChannel} already exists.
+                      |Unable to create new channel.""")
+        } else if (!storage.Channel.isValidName(newChannel)) {
+          logAndFail(s"""Unable to create new channel.
+                      |The channel name ${newChannel} is invalid.
+                      |${storage.Channel.nameConstraint}""")
+        } else {
+
+          val channel = Channel(
+            id = 0,
+            appid = appDesc.app.id,
+            name = newChannel)
+
+          chanStorage.insert(channel) map { chanId =>
+
+            info(s"Updated Channel meta-data.")
+
+            // initialize storage
+            val dbInit = events.init(appDesc.app.id, Some(chanId))
+            if (dbInit) {
+              info(s"Initialized Event Store for the channel: ${newChannel}.")
+              info(s"Created new channel:")
+              info(s"    Channel Name: ${newChannel}")
+              info(s"      Channel ID: ${chanId}")
+              info(s"          App ID: ${appDesc.app.id}")
+              Right(channel.copy(id = chanId))
+            } else {
+              errStr = s"""Unable to create new channel.
+                          |Failed to initalize Event Store."""
+              error(errStr)
+              // reverted back the meta data
+              try {
+                chanStorage.delete(chanId)
+                Left(errStr)
+              } catch {
+                case e: Exception =>
+                  val nextErrStr = s"""
+                    |Failed to revert back the Channel meta-data change.
+                    |The channel ${newChannel} CANNOT be used!
+                    |Please run 'pio app channel-delete ${appName} ${newChannel}'""" +
+                    " to delete this channel!"
+                  logAndFail(errStr + nextErrStr)
+              }
+            }
+          } getOrElse {
+            logAndFail(s"""Unable to create new channel.
+                          |Failed to update Channel meta-data.""")
+          }
+        }
+      }
+    } finally {
+      events.close()
+    }
+  }
+
+  def channelDelete(appName: String, deleteChannel: String): MaybeError = {
+    val chanStorage = storage.Storage.getMetaDataChannels
+    val events = storage.Storage.getLEvents()
+    var errStr = ""
+    try {
+      show(appName).right.flatMap { case (appDesc: AppDescription, channels: Seq[Channel]) =>
+        val foundChannel = channels.find(ch => ch.name == deleteChannel)
+        if (foundChannel.isEmpty) {
+          logAndFail(s"""Unable to delete channel
+                        |Channel ${deleteChannel} doesn't exists.""")
+        } else {
+          val chId = foundChannel.get.id
+          val dbRemoved = events.remove(appDesc.app.id, Some(chId))
+          if (dbRemoved) {
+            info(s"Removed Event Store for this channel: ${deleteChannel}")
+            try {
+              chanStorage.delete(chId)
+              logAndSucceed(s"Deleted channel: ${deleteChannel}.")
+            } catch {
+              case e: Exception =>
+                logAndFail(s"""Unable to delete channel.
+                  |Failed to update Channel meta-data.
+                  |The channel ${deleteChannel} CANNOT be used!
+                  |Please run 'pio app channel-delete ${appDesc.app.name} ${deleteChannel}'""" +
+                  " to delete this channel again!")
+            }
+          } else {
+            logAndFail(s"""Unable to delete channel.
+                          |Error removing Event Store for this channel.""")
+          }
+        }
+      }
+    } finally {
+      events.close()
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
new file mode 100644
index 0000000..6fd8977
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.controller.Utils
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.data.storage.EngineManifest
+import org.apache.predictionio.data.storage.EngineManifestSerializer
+import org.apache.predictionio.tools.RegisterEngine
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.{RunWorkflow, RunServer}
+import org.apache.predictionio.tools.{DeployArgs, WorkflowArgs, SparkArgs, ServerArgs}
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.tools.Common._
+import org.apache.predictionio.workflow.WorkflowUtils
+
+import org.apache.commons.io.FileUtils
+import org.json4s.native.Serialization.read
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+import scala.io.Source
+import scala.util.Random
+import scala.collection.JavaConversions._
+import scala.sys.process._
+import scalaj.http.Http
+import java.io.File
+
+case class BuildArgs(
+  sbt: Option[File] = None,
+  sbtExtra: Option[String] = None,
+  sbtAssemblyPackageDependency: Boolean = true,
+  sbtClean: Boolean = false,
+  uberJar: Boolean = false,
+  forceGeneratePIOSbt: Boolean = false)
+
+case class EngineArgs(
+  manifestJson: File = new File("manifest.json"),
+  engineId: Option[String] = None,
+  engineVersion: Option[String] = None)
+
+object Engine extends EitherLogging {
+
+  private val manifestAutogenTag = "pio-autogen-manifest"
+
+  private def readManifestJson(json: File): Expected[EngineManifest] = {
+    implicit val formats = Utils.json4sDefaultFormats +
+      new EngineManifestSerializer
+    try {
+      Right(read[EngineManifest](Source.fromFile(json).mkString))
+    } catch {
+      case e: java.io.FileNotFoundException =>
+        logAndFail(s"${json.getCanonicalPath} does not exist. Aborting.")
+      case e: MappingException =>
+        logAndFail(s"${json.getCanonicalPath} has invalid content: " +
+          e.getMessage)
+    }
+  }
+
+  private def withRegisteredManifest[T](ea: EngineArgs)(
+      op: EngineManifest => Expected[T]): Expected[T] = {
+    val res: Expected[Expected[T]] = for {
+      ej <- readManifestJson(ea.manifestJson).right
+      id <- Right(ea.engineId getOrElse ej.id).right
+      version <- Right(ea.engineVersion getOrElse ej.version).right
+      manifest <- storage.Storage.getMetaDataEngineManifests.get(id, version)
+        .toRight {
+          val errStr =
+            s"""Engine ${id} ${version} cannot be found in the system.")
+                |Possible reasons:
+                |- the engine is not yet built by the 'build' command;
+                |- the meta data store is offline."""
+          error(errStr)
+          errStr
+        }.right
+    } yield {
+      op(manifest)
+    }
+    res.joinRight
+  }
+
+  private def generateManifestJson(json: File): MaybeError = {
+    val cwd = sys.props("user.dir")
+    implicit val formats = Utils.json4sDefaultFormats +
+      new EngineManifestSerializer
+    val rand = Random.alphanumeric.take(32).mkString
+    val ha = java.security.MessageDigest.getInstance("SHA-1").
+      digest(cwd.getBytes).map("%02x".format(_)).mkString
+    val em = EngineManifest(
+      id = rand,
+      version = ha,
+      name = new File(cwd).getName,
+      description = Some(manifestAutogenTag),
+      files = Seq(),
+      engineFactory = "")
+    try {
+      FileUtils.writeStringToFile(json, write(em), "ISO-8859-1")
+      Success
+    } catch {
+      case e: java.io.IOException =>
+        logAndFail(s"Cannot generate ${json} automatically (${e.getMessage}). " +
+          "Aborting.")
+    }
+  }
+
+  private def regenerateManifestJson(json: File): MaybeError = {
+    val cwd = sys.props("user.dir")
+    val ha = java.security.MessageDigest.getInstance("SHA-1").
+      digest(cwd.getBytes).map("%02x".format(_)).mkString
+    if (json.exists) {
+      readManifestJson(json).right.flatMap { em =>
+        if (em.description == Some(manifestAutogenTag) && ha != em.version) {
+          warn("This engine project directory contains an auto-generated " +
+            "manifest that has been copied/moved from another location. ")
+          warn("Regenerating the manifest to reflect the updated location. " +
+            "This will dissociate with all previous engine instances.")
+          generateManifestJson(json)
+        } else {
+          logAndSucceed(s"Using existing engine manifest JSON at " +
+            "${json.getCanonicalPath}")
+        }
+      }
+    } else {
+      generateManifestJson(json)
+    }
+  }
+
+  private def detectSbt(sbt: Option[File], pioHome: String): String = {
+    sbt map {
+      _.getCanonicalPath
+    } getOrElse {
+      val f = new File(Seq(pioHome, "sbt", "sbt").mkString(File.separator))
+      if (f.exists) f.getCanonicalPath else "sbt"
+    }
+  }
+
+  private def outputSbtError(line: String): Unit = {
+    """\[.*error.*\]""".r findFirstIn line foreach { _ => error(line) }
+  }
+
+  private def compile(
+    buildArgs: BuildArgs, pioHome: String, verbose: Boolean): MaybeError = {
+    // only add pioVersion to sbt if project/pio.sbt exists
+    if (new File("project", "pio-build.sbt").exists || buildArgs.forceGeneratePIOSbt) {
+      FileUtils.writeLines(
+        new File("pio.sbt"),
+        Seq(
+          "// Generated automatically by pio build.",
+          "// Changes in this file will be overridden.",
+          "",
+          "pioVersion := \"" + BuildInfo.version + "\""))
+    }
+    implicit val formats = Utils.json4sDefaultFormats
+
+    val sbt = detectSbt(buildArgs.sbt, pioHome)
+    info(s"Using command '${sbt}' at the current working directory to build.")
+    info("If the path above is incorrect, this process will fail.")
+    val asm =
+      if (buildArgs.sbtAssemblyPackageDependency) {
+        " assemblyPackageDependency"
+      } else {
+        ""
+      }
+    val clean = if (buildArgs.sbtClean) " clean" else ""
+    val buildCmd = s"${sbt} ${buildArgs.sbtExtra.getOrElse("")}${clean} " +
+      (if (buildArgs.uberJar) "assembly" else s"package${asm}")
+    val core = new File(s"pio-assembly-${BuildInfo.version}.jar")
+    if (buildArgs.uberJar) {
+      info(s"Uber JAR enabled. Putting ${core.getName} in lib.")
+      val dst = new File("lib")
+      dst.mkdir()
+      coreAssembly(pioHome) match {
+        case Right(coreFile) =>
+          FileUtils.copyFileToDirectory(
+            coreFile,
+            dst,
+            true)
+        case Left(errStr) => return Left(errStr)
+      }
+    } else {
+      if (new File("engine.json").exists()) {
+        info(s"Uber JAR disabled. Making sure lib/${core.getName} is absent.")
+        new File("lib", core.getName).delete()
+      } else {
+        info("Uber JAR disabled, but current working directory does not look " +
+          s"like an engine project directory. Please delete lib/${core.getName} manually.")
+      }
+    }
+    info(s"Going to run: ${buildCmd}")
+    try {
+      val r =
+        if (verbose) {
+          buildCmd.!(ProcessLogger(line => info(line), line => error(line)))
+        } else {
+          buildCmd.!(ProcessLogger(
+            line => outputSbtError(line),
+            line => outputSbtError(line)))
+        }
+      if (r != 0) {
+        logAndFail(s"Return code of build command: ${buildCmd} is ${r}. Aborting.")
+      } else {
+        logAndSucceed("Compilation finished successfully.")
+      }
+    } catch {
+      case e: java.io.IOException =>
+        logAndFail(s"Exception during compilation: ${e.getMessage}")
+    }
+  }
+
+  def build(
+    buildArgs: BuildArgs,
+    pioHome: String,
+    manifestJson: File,
+    verbose: Boolean): MaybeError = {
+
+    regenerateManifestJson(manifestJson) match {
+      case Left(err) => return Left(err)
+      case _ => Unit
+    }
+
+    Template.verifyTemplateMinVersion(new File("template.json")) match {
+      case Left(err) => return Left(err)
+      case Right(_) =>
+        compile(buildArgs, pioHome, verbose)
+        info("Looking for an engine...")
+        val jarFiles = jarFilesForScala
+        if (jarFiles.isEmpty) {
+          return logAndFail("No engine found. Your build might have failed. Aborting.")
+        }
+        jarFiles foreach { f => info(s"Found ${f.getName}")}
+        RegisterEngine.registerEngine(
+          manifestJson,
+          jarFiles,
+          false)
+    }
+  }
+
+  /** Training an engine.
+    *  The function starts a training process to bu run concurrenlty.
+    *
+    * @param ea An instance of [[EngineArgs]]
+    * @param wa An instance of [[WorkflowArgs]] for running a single training.
+    * @param sa An instance of [[SparkArgs]]
+    * @param pioHome [[String]] with a path to PIO installation
+    * @param verbose A [[Boolean]]
+    * @return An instance of [[Expected]] contaning either [[Left]]
+    *         with an error message or [[Right]] with a handle to a process
+    *         responsible for training and a function () => Unit,
+    *         that must be called when the process is complete
+    */
+  def train(
+    ea: EngineArgs,
+    wa: WorkflowArgs,
+    sa: SparkArgs,
+    pioHome: String,
+    verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
+    regenerateManifestJson(ea.manifestJson) match {
+      case Left(err) => return Left(err)
+      case _ => Unit
+    }
+
+    Template.verifyTemplateMinVersion(new File("template.json")).right.flatMap {
+      _ =>
+        withRegisteredManifest(ea) { em =>
+            RunWorkflow.runWorkflow(wa, sa, em, pioHome, verbose)
+          }
+    }
+  }
+
+  /** Deploying an engine.
+    *  The function starts a new process to be run concerrently.
+    *
+    * @param ea An instance of [[EngineArgs]]
+    * @param engineInstanceId An instance of [[engineInstanceId]]
+    * @param serverArgs An instance of [[ServerArgs]]
+    * @param sparkArgs An instance of [[SparkArgs]]
+    * @param pioHome [[String]] with a path to PIO installation
+    * @param verbose A [[Boolean]]
+    * @return An instance of [[Expected]] contaning either [[Left]]
+    *         with an error message or [[Right]] with a handle to process
+    *         of a running angine  and a function () => Unit,
+    *         that must be called when the process is complete
+    */
+  def deploy(
+    ea: EngineArgs,
+    engineInstanceId: Option[String],
+    serverArgs: ServerArgs,
+    sparkArgs: SparkArgs,
+    pioHome: String,
+    verbose: Boolean = false): Expected[(Process, () => Unit)] = {
+
+    val verifyResult = Template.verifyTemplateMinVersion(new File("template.json"))
+    if (verifyResult.isLeft) {
+      return Left(verifyResult.left.get)
+    }
+    withRegisteredManifest(ea) { em =>
+      val variantJson = parse(Source.fromFile(serverArgs.variantJson).mkString)
+      val variantId = variantJson \ "id" match {
+        case JString(s) => s
+        case _ =>
+          return logAndFail("Unable to read engine variant ID from " +
+            s"${serverArgs.variantJson.getCanonicalPath}. Aborting.")
+      }
+      val engineInstances = storage.Storage.getMetaDataEngineInstances
+      val engineInstance = engineInstanceId map { eid =>
+        engineInstances.get(eid)
+      } getOrElse {
+        engineInstances.getLatestCompleted(em.id, em.version, variantId)
+      }
+      engineInstance map { r =>
+        RunServer.runServer(r.id, serverArgs, sparkArgs, em, pioHome, verbose)
+      } getOrElse {
+        engineInstanceId map { eid =>
+          logAndFail(s"Invalid engine instance ID ${eid}. Aborting.")
+        } getOrElse {
+          logAndFail(s"No valid engine instance found for engine ${em.id} " +
+            s"${em.version}.\nTry running 'train' before 'deploy'. Aborting.")
+        }
+      }
+    }
+  }
+
+  def undeploy(da: DeployArgs): MaybeError = {
+
+    val serverUrl = s"http://${da.ip}:${da.port}"
+    info(
+      s"Undeploying any existing engine instance at ${serverUrl}")
+    try {
+      val code = Http(s"${serverUrl}/stop").asString.code
+      code match {
+        case 200 => Success
+        case 404 =>
+          logAndFail(s"Another process is using ${serverUrl}. Unable to undeploy.")
+        case _ =>
+          logAndFail(s"Another process is using ${serverUrl}, or an existing " +
+            s"engine server is not responding properly (HTTP ${code}). " +
+            "Unable to undeploy.")
+      }
+    } catch {
+      case e: java.net.ConnectException =>
+        logAndFail(s"Nothing at ${serverUrl}")
+      case _: Throwable =>
+        logAndFail("Another process might be occupying " +
+          s"${da.ip}:${da.port}. Unable to undeploy.")
+    }
+  }
+
+  /** Running a driver on spark.
+    *  The function starts a process and returns immediately
+    *
+    * @param mainClass A [[String]] with the class containing a main functionto run
+    * @param driverArguments Arguments to be passed to the main function
+    * @param manifestJson An instance of [[File]] for running a single training.
+    * @param buildArgs An instance of [[BuildArgs]]
+    * @param sparkArgs an instance of [[SparkArgs]]
+    * @param pioHome [[String]] with a path to PIO installation
+    * @param verbose A [[Boolean]]
+    * @return An instance of [[Expected]] contaning either [[Left]]
+    *         with an error message or [[Right]] with a handle to a process
+    *         of a running driver
+    */
+  def run(
+    mainClass: String,
+    driverArguments: Seq[String],
+    manifestJson: File,
+    buildArgs: BuildArgs,
+    sparkArgs: SparkArgs,
+    pioHome: String,
+    verbose: Boolean): Expected[Process] = {
+
+    generateManifestJson(manifestJson) match {
+      case Left(err) => return Left(err)
+      case _ => Unit
+    }
+
+    compile(buildArgs, pioHome, verbose)
+
+    val extraFiles = WorkflowUtils.thirdPartyConfFiles
+
+    val jarFiles = jarFilesForScala
+    jarFiles foreach { f => info(s"Found JAR: ${f.getName}") }
+    val allJarFiles = jarFiles.map(_.getCanonicalPath)
+    val cmd = s"${getSparkHome(sparkArgs.sparkHome)}/bin/spark-submit --jars " +
+      s"${allJarFiles.mkString(",")} " +
+      (if (extraFiles.size > 0) {
+        s"--files ${extraFiles.mkString(",")} "
+      } else {
+        ""
+      }) +
+      "--class " +
+      s"${mainClass} ${sparkArgs.sparkPassThrough.mkString(" ")} " +
+      coreAssembly(pioHome) + " " +
+      driverArguments.mkString(" ")
+    info(s"Submission command: ${cmd}")
+    Right(Process(
+      cmd,
+      None,
+      "SPARK_YARN_USER_ENV" -> sys.env.filter(kv => kv._1.startsWith("PIO_")).
+        map(kv => s"${kv._1}=${kv._2}").mkString(",")).run())
+  }
+
+  def unregister(jsonManifest: File): MaybeError = {
+    RegisterEngine.unregisterEngine(jsonManifest)
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala
new file mode 100644
index 0000000..ed6b487
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Export.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.tools.Runner
+import org.apache.predictionio.tools.SparkArgs
+import org.apache.predictionio.tools.ReturnTypes._
+
+import scala.sys.process._
+
+case class ExportArgs(
+  appId: Int = 0,
+  channel: Option[String] = None,
+  outputPath: String = "",
+  format: String = "json")
+
+object Export {
+  def eventsToFile(
+    ea: ExportArgs,
+    sa: SparkArgs,
+    pioHome: String): Expected[(Process, () => Unit)] = {
+
+    val channelArg = ea.channel
+      .map(ch => Seq("--channel", ch)).getOrElse(Nil)
+    Runner.runOnSpark(
+      "org.apache.predictionio.tools.export.EventsToFile",
+      Seq(
+        "--appid",
+        ea.appId.toString,
+        "--output",
+        ea.outputPath,
+        "--format",
+        ea.format) ++ channelArg,
+      sa,
+      Nil,
+      pioHome)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala
new file mode 100644
index 0000000..9fac559
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Import.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.tools.Runner
+import org.apache.predictionio.tools.SparkArgs
+import org.apache.predictionio.tools.ReturnTypes._
+
+import scala.sys.process._
+
+case class ImportArgs(
+  appId: Int = 0,
+  channel: Option[String] = None,
+  inputPath: String = "")
+
+object Import {
+  def fileToEvents(
+    ia: ImportArgs,
+    sa: SparkArgs,
+    pioHome: String): Expected[(Process, () => Unit)] = {
+
+    val channelArg = ia.channel
+      .map(ch => Seq("--channel", ch)).getOrElse(Nil)
+    Runner.runOnSpark(
+      "org.apache.predictionio.tools.imprt.FileToEvents",
+      Seq(
+        "--appid",
+        ia.appId.toString,
+        "--input",
+        ia.inputPath) ++ channelArg,
+      sa,
+      Nil,
+      pioHome)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
new file mode 100644
index 0000000..10aca41
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.data.storage
+import org.apache.predictionio.data.api.EventServer
+import org.apache.predictionio.data.api.EventServerConfig
+import org.apache.predictionio.tools.EventServerArgs
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.Common
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.tools.dashboard.Dashboard
+import org.apache.predictionio.tools.dashboard.DashboardConfig
+import org.apache.predictionio.tools.admin.AdminServer
+import org.apache.predictionio.tools.admin.AdminServerConfig
+
+import akka.actor.ActorSystem
+import java.io.File
+import scala.io.Source
+import semverfi._
+
+case class DashboardArgs(
+  ip: String = "127.0.0.1",
+  port: Int = 9000)
+
+case class AdminServerArgs(
+  ip: String = "127.0.0.1",
+  port: Int = 7071)
+
+case class PioStatus(
+  version: String = "",
+  pioHome: String = "",
+  sparkHome: String = "",
+  sparkVersion: String = "",
+  sparkMinVersion: String = "",
+  warnings: Seq[String] = Seq())
+
+object Management extends EitherLogging {
+
+  def version(): String = BuildInfo.version
+
+  /** Starts a dashboard server and returns immediately
+    *
+    * @param da An instance of [[DashboardArgs]]
+    * @return An instance of [[ActorSystem]] in which the server is being executed
+    */
+  def dashboard(da: DashboardArgs): ActorSystem = {
+    info(s"Creating dashboard at ${da.ip}:${da.port}")
+    Dashboard.createDashboard(DashboardConfig(
+      ip = da.ip,
+      port = da.port))
+  }
+
+  /** Starts an eventserver server and returns immediately
+    *
+    * @param ea An instance of [[EventServerArgs]]
+    * @return An instance of [[ActorSystem]] in which the server is being executed
+    */
+  def eventserver(ea: EventServerArgs): ActorSystem = {
+    info(s"Creating Event Server at ${ea.ip}:${ea.port}")
+    EventServer.createEventServer(EventServerConfig(
+      ip = ea.ip,
+      port = ea.port,
+      stats = ea.stats))
+  }
+
+  /** Starts an adminserver server and returns immediately
+    *
+    * @param aa An instance of [[AdminServerArgs]]
+    * @return An instance of [[ActorSystem]] in which the server is being executed
+    */
+  def adminserver(aa: AdminServerArgs): ActorSystem = {
+    info(s"Creating Admin Server at ${aa.ip}:${aa.port}")
+    AdminServer.createAdminServer(AdminServerConfig(
+      ip = aa.ip,
+      port = aa.port
+    ))
+  }
+
+  private def stripMarginAndNewlines(string: String): String =
+    string.stripMargin.replaceAll("\n", " ")
+
+  def status(pioHome: Option[String], sparkHome: Option[String]): Expected[PioStatus] = {
+    var pioStatus = PioStatus()
+    info("Inspecting PredictionIO...")
+    pioHome map { pioHome =>
+      info(s"PredictionIO ${BuildInfo.version} is installed at $pioHome")
+      pioStatus = pioStatus.copy(version = version(), pioHome = pioHome)
+    } getOrElse {
+      return logAndFail("Unable to locate PredictionIO installation. Aborting.")
+    }
+    info("Inspecting Apache Spark...")
+    val sparkHomePath = Common.getSparkHome(sparkHome)
+    if (new File(s"$sparkHomePath/bin/spark-submit").exists) {
+      info(s"Apache Spark is installed at $sparkHome")
+      val sparkMinVersion = "1.3.0"
+      pioStatus = pioStatus.copy(
+        sparkHome = sparkHomePath,
+        sparkMinVersion = sparkMinVersion)
+      val sparkReleaseFile = new File(s"$sparkHomePath/RELEASE")
+      if (sparkReleaseFile.exists) {
+        val sparkReleaseStrings =
+          Source.fromFile(sparkReleaseFile).mkString.split(' ')
+        if (sparkReleaseStrings.length < 2) {
+          val warning = (stripMarginAndNewlines(
+            s"""|Apache Spark version information cannot be found (RELEASE file
+                |is empty). This is a known issue for certain vendors (e.g.
+                |Cloudera). Please make sure you are using a version of at least
+                |$sparkMinVersion."""))
+          warn(warning)
+          pioStatus = pioStatus.copy(warnings = pioStatus.warnings :+ warning)
+        } else {
+          val sparkReleaseVersion = sparkReleaseStrings(1)
+          val parsedMinVersion = Version.apply(sparkMinVersion)
+          val parsedCurrentVersion = Version.apply(sparkReleaseVersion)
+          if (parsedCurrentVersion >= parsedMinVersion) {
+            info(stripMarginAndNewlines(
+              s"""|Apache Spark $sparkReleaseVersion detected (meets minimum
+                  |requirement of $sparkMinVersion)"""))
+            pioStatus = pioStatus.copy(sparkVersion = sparkReleaseVersion)
+          } else {
+            return logAndFail(stripMarginAndNewlines(
+              s"""|Apache Spark $sparkReleaseVersion detected (does not meet
+                  |minimum requirement. Aborting."""))
+          }
+        }
+      } else {
+        val warning = (stripMarginAndNewlines(
+          s"""|Apache Spark version information cannot be found. If you are
+              |using a developmental tree, please make sure you are using a
+              |version of at least $sparkMinVersion."""))
+        warn(warning)
+        pioStatus = pioStatus.copy(warnings = pioStatus.warnings :+ warning)
+      }
+    } else {
+      return logAndFail("Unable to locate a proper Apache Spark installation. Aborting.")
+    }
+    info("Inspecting storage backend connections...")
+    try {
+      storage.Storage.verifyAllDataObjects()
+    } catch {
+      case e: Throwable =>
+        val errStr = s"""Unable to connect to all storage backends successfully.
+            |The following shows the error message from the storage backend.
+            |${e.getMessage} (${e.getClass.getName})", e)
+            |Dumping configuration of initialized storage backend sources.
+            |"Please make sure they are correct.
+            |"""
+        val sources = storage.Storage.config.get("sources") map { src =>
+          src map { case (s, p) =>
+            s"Source Name: $s; Type: ${p.getOrElse("type", "(error)")}; " +
+              s"Configuration: ${p.getOrElse("config", "(error)")}"
+          } mkString("\n")
+        } getOrElse {
+          "No properly configured storage backend sources."
+        }
+        return logAndFail(errStr + sources)
+    }
+    info("Your system is all ready to go.")
+    Right(pioStatus)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
new file mode 100644
index 0000000..1476598
--- /dev/null
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools.commands
+
+import java.io.File
+
+import scala.io.Source
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.core.BuildInfo
+import org.apache.predictionio.tools.EitherLogging
+import org.apache.predictionio.tools.ReturnTypes._
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.{write, read}
+import semverfi._
+
+case class TemplateMetaData(
+  pioVersionMin: Option[String] = None)
+
+object Template extends EitherLogging {
+
+  def templateMetaData(templateJson: File): TemplateMetaData = {
+    if (!templateJson.exists) {
+      warn(s"$templateJson does not exist. Template metadata will not be available. " +
+        "(This is safe to ignore if you are not working on a template.)")
+      TemplateMetaData()
+    } else {
+      val jsonString = Source.fromFile(templateJson)(scala.io.Codec.ISO8859).mkString
+      val json = try {
+        parse(jsonString)
+      } catch {
+        case e: org.json4s.ParserUtil.ParseException =>
+          warn(s"$templateJson cannot be parsed. Template metadata will not be available.")
+          return TemplateMetaData()
+      }
+      val pioVersionMin = json \ "pio" \ "version" \ "min"
+      pioVersionMin match {
+        case JString(s) => TemplateMetaData(pioVersionMin = Some(s))
+        case _ => TemplateMetaData()
+      }
+    }
+  }
+
+  def verifyTemplateMinVersion(templateJsonFile: File): MaybeError = {
+    val metadata = templateMetaData(templateJsonFile)
+
+    for (pvm <- metadata.pioVersionMin) {
+      if (Version(BuildInfo.version) < Version(pvm)) {
+        return logAndFail(s"This engine template requires at least PredictionIO $pvm. " +
+          s"The template may not work with PredictionIO ${BuildInfo.version}.")
+      }
+    }
+    Success
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e4a3c0c9/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala
deleted file mode 100644
index fcb9608..0000000
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/AccessKey.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.tools.console
-
-import org.apache.predictionio.data.storage
-
-import grizzled.slf4j.Logging
-
-case class AccessKeyArgs(
-  accessKey: String = "",
-  events: Seq[String] = Seq())
-
-object AccessKey extends Logging {
-  def create(ca: ConsoleArgs): Int = {
-    val apps = storage.Storage.getMetaDataApps
-    apps.getByName(ca.app.name) map { app =>
-      val accessKeys = storage.Storage.getMetaDataAccessKeys
-      val accessKey = accessKeys.insert(storage.AccessKey(
-        key = ca.accessKey.accessKey,
-        appid = app.id,
-        events = ca.accessKey.events))
-      accessKey map { k =>
-        info(s"Created new access key: ${k}")
-        0
-      } getOrElse {
-        error(s"Unable to create new access key.")
-        1
-      }
-    } getOrElse {
-      error(s"App ${ca.app.name} does not exist. Aborting.")
-      1
-    }
-  }
-
-  def list(ca: ConsoleArgs): Int = {
-    val keys =
-      if (ca.app.name == "") {
-        storage.Storage.getMetaDataAccessKeys.getAll
-      } else {
-        val apps = storage.Storage.getMetaDataApps
-        apps.getByName(ca.app.name) map { app =>
-          storage.Storage.getMetaDataAccessKeys.getByAppid(app.id)
-        } getOrElse {
-          error(s"App ${ca.app.name} does not exist. Aborting.")
-          return 1
-        }
-      }
-    val title = "Access Key(s)"
-    info(f"$title%64s | App ID | Allowed Event(s)")
-    keys.sortBy(k => k.appid) foreach { k =>
-      val events =
-        if (k.events.size > 0) k.events.sorted.mkString(",") else "(all)"
-      info(f"${k.key}%64s | ${k.appid}%6d | $events%s")
-    }
-    info(s"Finished listing ${keys.size} access key(s).")
-    0
-  }
-
-  def delete(ca: ConsoleArgs): Int = {
-    try {
-      storage.Storage.getMetaDataAccessKeys.delete(ca.accessKey.accessKey)
-      info(s"Deleted access key ${ca.accessKey.accessKey}.")
-      0
-    } catch {
-      case e: Exception =>
-        error(s"Error deleting access key ${ca.accessKey.accessKey}.", e)
-        1
-    }
-  }
-}