You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2017/07/27 18:44:43 UTC
incubator-predictionio git commit: [PIO-56] No-setup unit tests for
core and data
Repository: incubator-predictionio
Updated Branches:
refs/heads/develop 9bbd1f51a -> cc4e2e0a2
[PIO-56] No-setup unit tests for core and data
Closes #355
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/cc4e2e0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/cc4e2e0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/cc4e2e0a
Branch: refs/heads/develop
Commit: cc4e2e0a2303962f2b2112ec7612f39579786439
Parents: 9bbd1f5
Author: lucasbm88 <lu...@gmail.com>
Authored: Thu Jul 27 11:40:19 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Jul 27 11:40:19 2017 -0700
----------------------------------------------------------------------
core/build.sbt | 5 +-
.../controller/EvaluationTest.scala | 6 +-
.../controller/MetricEvaluatorTest.scala | 6 +-
.../apache/predictionio/workflow/BaseTest.scala | 59 ++++++++++++++++--
.../workflow/EvaluationWorkflowTest.scala | 5 +-
data/build.sbt | 5 +-
.../predictionio/data/storage/Storage.scala | 41 +++++++++++--
.../data/api/EventServiceSpec.scala | 39 ++++++------
.../data/api/SegmentIOAuthSpec.scala | 36 ++++++-----
.../data/storage/StorageMockContext.scala | 64 ++++++++++++++++++++
10 files changed, 211 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/build.sbt
----------------------------------------------------------------------
diff --git a/core/build.sbt b/core/build.sbt
index b95a957..14b3449 100644
--- a/core/build.sbt
+++ b/core/build.sbt
@@ -30,7 +30,10 @@ libraryDependencies ++= Seq(
"org.scalaj" %% "scalaj-http" % "1.1.6",
"org.slf4j" % "slf4j-log4j12" % "1.7.18",
"org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.specs2" %% "specs2" % "2.3.13" % "test",
+ "org.scalamock" %% "scalamock-scalatest-support" % "3.5.0" % "test",
+ "com.h2database" % "h2" % "1.4.196" % "test"
+)
parallelExecution in Test := false
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala b/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
index a64826c..b60d358 100644
--- a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala
@@ -17,12 +17,10 @@
package org.apache.predictionio.controller
-import org.apache.predictionio.workflow.SharedSparkContext
-
+import org.apache.predictionio.workflow.{SharedSparkContext, SharedStorageContext}
import org.scalatest.FunSuite
import org.scalatest.Inside
import org.scalatest.Matchers._
-
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
@@ -42,7 +40,7 @@ object EvaluationSuite {
class EvaluationSuite
-extends FunSuite with Inside with SharedSparkContext {
+extends FunSuite with Inside with SharedSparkContext with SharedStorageContext {
import org.apache.predictionio.controller.EvaluationSuite._
test("Evaluation makes MetricEvaluator") {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala b/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
index 9aec6d5..165fc63 100644
--- a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala
@@ -18,8 +18,7 @@
package org.apache.predictionio.controller
-import org.apache.predictionio.workflow.SharedSparkContext
-import org.apache.predictionio.workflow.WorkflowParams
+import org.apache.predictionio.workflow.{SharedSparkContext, SharedStorageContext, WorkflowParams}
import org.scalatest.FunSuite
object MetricEvaluatorSuite {
@@ -30,7 +29,8 @@ object MetricEvaluatorSuite {
object Evaluation0 extends Evaluation {}
}
-class MetricEvaluatorDevSuite extends FunSuite with SharedSparkContext {
+class MetricEvaluatorDevSuite extends FunSuite with SharedSparkContext
+with SharedStorageContext {
import org.apache.predictionio.controller.MetricEvaluatorSuite._
test("a") {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala b/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
index 004dbb1..59d23f1 100644
--- a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala
@@ -15,20 +15,22 @@
* limitations under the License.
*/
-//package org.apache.spark
+// package org.apache.spark
package org.apache.predictionio.workflow
-import _root_.io.netty.util.internal.logging.{Slf4JLoggerFactory, InternalLoggerFactory}
+import _root_.io.netty.util.internal.logging.{InternalLoggerFactory, Slf4JLoggerFactory}
+import org.apache.predictionio.data.storage.{EnvironmentFactory, EnvironmentService}
import org.scalatest.BeforeAndAfterAll
import org.scalatest.BeforeAndAfterEach
import org.scalatest.Suite
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
+import org.scalamock.scalatest.MockFactory
/** Manages a local `sc` {@link SparkContext} variable, correctly stopping it
* after each test. */
-trait LocalSparkContext
+trait LocalSparkContext
extends BeforeAndAfterEach with BeforeAndAfterAll { self: Suite =>
@transient var sc: SparkContext = _
@@ -43,7 +45,7 @@ extends BeforeAndAfterEach with BeforeAndAfterAll { self: Suite =>
super.afterEach()
}
- def resetSparkContext() = {
+ def resetSparkContext() : Unit = {
LocalSparkContext.stop(sc)
sc = null
}
@@ -60,7 +62,7 @@ object LocalSparkContext {
}
/** Runs `f` by passing in `sc` and ensures that `sc` is stopped. */
- def withSpark[T](sc: SparkContext)(f: SparkContext => T) = {
+ def withSpark[T](sc: SparkContext)(f: SparkContext => T) : Unit = {
try {
f(sc)
} finally {
@@ -90,3 +92,50 @@ trait SharedSparkContext extends BeforeAndAfterAll { self: Suite =>
}
}
+trait SharedStorageContext extends BeforeAndAfterAll { self: Suite =>
+
+ override def beforeAll(): Unit ={
+ ConfigurationMockUtil.createJDBCMockedConfig
+ super.beforeAll()
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ }
+
+}
+
+object ConfigurationMockUtil extends MockFactory {
+
+ def createJDBCMockedConfig: Unit = {
+ val mockedEnvService = mock[EnvironmentService]
+ (mockedEnvService.envKeys _)
+ .expects
+ .returning(List("PIO_STORAGE_REPOSITORIES_METADATA_NAME",
+ "PIO_STORAGE_SOURCES_MYSQL_TYPE"))
+ .twice
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_NAME")
+ .returning("test_metadata")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_SOURCE")
+ .returning("MYSQL")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_SOURCES_MYSQL_TYPE")
+ .returning("jdbc")
+
+ (mockedEnvService.filter _)
+ .expects(*)
+ .returning(Map(
+ "URL" -> "jdbc:h2:~/test;MODE=MySQL;AUTO_SERVER=TRUE",
+ "USERNAME" -> "sa",
+ "PASSWORD" -> "")
+ )
+
+ EnvironmentFactory.environmentService = new Some(mockedEnvService)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala b/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
index 26fc936..70534e4 100644
--- a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala
@@ -18,11 +18,12 @@
package org.apache.predictionio.workflow
import org.apache.predictionio.controller._
-
+import org.scalamock.scalatest.MockFactory
import org.scalatest.FunSuite
import org.scalatest.Matchers._
-class EvaluationWorkflowSuite extends FunSuite with SharedSparkContext {
+class EvaluationWorkflowSuite extends FunSuite with SharedStorageContext
+ with SharedSparkContext with MockFactory {
test("Evaluation return best engine params, simple result type: Double") {
val engine = new Engine1()
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/build.sbt
----------------------------------------------------------------------
diff --git a/data/build.sbt b/data/build.sbt
index b6b2410..abfa74f 100644
--- a/data/build.sbt
+++ b/data/build.sbt
@@ -28,7 +28,10 @@ libraryDependencies ++= Seq(
"org.clapper" %% "grizzled-slf4j" % "1.0.2",
"org.json4s" %% "json4s-native" % json4sVersion.value,
"org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.specs2" %% "specs2" % "3.3.1" % "test"
+ exclude("org.scalaz.stream", s"scalaz-stream_${scalaBinaryVersion.value}"),
+ "org.scalamock" %% "scalamock-specs2-support" % "3.5.0" % "test",
+ "com.h2database" % "h2" % "1.4.196" % "test")
parallelExecution in Test := false
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
index 744a5e6..fd05767 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
@@ -112,6 +112,33 @@ class StorageException(message: String, cause: Throwable)
def this(message: String) = this(message, null)
}
+class EnvironmentService{
+
+ def envKeys(): Iterable[String] = {
+ sys.env.keys
+ }
+
+ def getByKey(key: String): String = {
+ sys.env(key)
+ }
+
+ def filter(filterExpression: ((String, String)) => Boolean): Map[String, String] = {
+ sys.env.filter(filterExpression)
+ }
+}
+
+object EnvironmentFactory{
+
+ var environmentService: Option[EnvironmentService] = None
+
+ def create(): EnvironmentService = {
+ if(environmentService.isEmpty){
+ environmentService = new Some[EnvironmentService](new EnvironmentService)
+ }
+ environmentService.get
+ }
+}
+
/** Backend-agnostic data storage layer with lazy initialization. Use this
* object when you need to interface with Event Store in your engine.
*
@@ -123,6 +150,8 @@ object Storage extends Logging {
client: BaseStorageClient,
config: StorageClientConfig)
+ var environmentService: EnvironmentService = EnvironmentFactory.create
+
private case class DataObjectMeta(sourceName: String, namespace: String)
private var errors = 0
@@ -131,7 +160,7 @@ object Storage extends Logging {
private val sourceTypesRegex = """PIO_STORAGE_SOURCES_([^_]+)_TYPE""".r
- private val sourceKeys: Seq[String] = sys.env.keys.toSeq.flatMap { k =>
+ private val sourceKeys: Seq[String] = environmentService.envKeys.toSeq.flatMap { k =>
sourceTypesRegex findFirstIn k match {
case Some(sourceTypesRegex(sourceType)) => Seq(sourceType)
case None => Nil
@@ -152,7 +181,7 @@ object Storage extends Logging {
private val repositoryNamesRegex =
"""PIO_STORAGE_REPOSITORIES_([^_]+)_NAME""".r
- private val repositoryKeys: Seq[String] = sys.env.keys.toSeq.flatMap { k =>
+ private val repositoryKeys: Seq[String] = environmentService.envKeys.toSeq.flatMap { k =>
repositoryNamesRegex findFirstIn k match {
case Some(repositoryNamesRegex(repositoryName)) => Seq(repositoryName)
case None => Nil
@@ -175,8 +204,8 @@ object Storage extends Logging {
repositoryKeys.map(r =>
try {
val keyedPath = repositoriesPrefixPath(r)
- val name = sys.env(prefixPath(keyedPath, "NAME"))
- val sourceName = sys.env(prefixPath(keyedPath, "SOURCE"))
+ val name = environmentService.getByKey(prefixPath(keyedPath, "NAME"))
+ val sourceName = environmentService.getByKey(prefixPath(keyedPath, "SOURCE"))
if (sourceKeys.contains(sourceName)) {
r -> DataObjectMeta(
sourceName = sourceName,
@@ -244,8 +273,8 @@ object Storage extends Logging {
Option[ClientMeta] = {
try {
val keyedPath = sourcesPrefixPath(k)
- val sourceType = sys.env(prefixPath(keyedPath, "TYPE"))
- val props = sys.env.filter(t => t._1.startsWith(keyedPath)).map(
+ val sourceType = environmentService.getByKey(prefixPath(keyedPath, "TYPE"))
+ val props = environmentService.filter(t => t._1.startsWith(keyedPath)).map(
t => t._1.replace(s"${keyedPath}_", "") -> t._2)
val clientConfig = StorageClientConfig(
properties = props,
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala b/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
index 988146e..7a45ca1 100644
--- a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
+++ b/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala
@@ -18,40 +18,41 @@
package org.apache.predictionio.data.api
-import org.apache.predictionio.data.storage.Storage
-
+import org.apache.predictionio.data.storage.{Storage, StorageMockContext}
import akka.testkit.TestProbe
-import akka.actor.ActorSystem
-import akka.actor.Props
-
+import akka.actor.{ActorRef, ActorSystem, Props}
import spray.http.HttpEntity
import spray.http.HttpResponse
import spray.http.ContentTypes
import spray.httpx.RequestBuilding.Get
-
import org.specs2.mutable.Specification
class EventServiceSpec extends Specification {
val system = ActorSystem("EventServiceSpecSystem")
- val eventClient = Storage.getLEvents()
- val accessKeysClient = Storage.getMetaDataAccessKeys()
- val channelsClient = Storage.getMetaDataChannels()
-
- val eventServiceActor = system.actorOf(
- Props(
- new EventServiceActor(
- eventClient,
- accessKeysClient,
- channelsClient,
- EventServerConfig()
+ def createEventServiceActor: ActorRef = {
+ val eventClient = Storage.getLEvents()
+ val accessKeysClient = Storage.getMetaDataAccessKeys()
+ val channelsClient = Storage.getMetaDataChannels()
+
+ system.actorOf(
+ Props(
+ new EventServiceActor(
+ eventClient,
+ accessKeysClient,
+ channelsClient,
+ EventServerConfig()
+ )
)
)
- )
+ }
+
"GET / request" should {
- "properly produce OK HttpResponses" in {
+ "properly produce OK HttpResponses" in new StorageMockContext {
+ Thread.sleep(2000)
+ val eventServiceActor = createEventServiceActor
val probe = TestProbe()(system)
probe.send(eventServiceActor, Get("/"))
probe.expectMsg(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala b/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
index a276c59..5927824 100644
--- a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
+++ b/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala
@@ -17,17 +17,18 @@
package org.apache.predictionio.data.api
-import akka.actor.{ActorSystem, Props}
+import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.TestProbe
import org.apache.predictionio.data.storage._
import org.joda.time.DateTime
+import org.scalamock.specs2.MockContext
import org.specs2.mutable.Specification
import spray.http.HttpHeaders.RawHeader
import spray.http.{ContentTypes, HttpEntity, HttpResponse}
import spray.httpx.RequestBuilding._
import sun.misc.BASE64Encoder
-import scala.concurrent.{Future, ExecutionContext}
+import scala.concurrent.{ExecutionContext, Future}
class SegmentIOAuthSpec extends Specification {
@@ -78,23 +79,26 @@ class SegmentIOAuthSpec extends Specification {
}
}
- val channelsClient = Storage.getMetaDataChannels()
- val eventServiceActor = system.actorOf(
- Props(
- new EventServiceActor(
- eventClient,
- accessKeysClient,
- channelsClient,
- EventServerConfig()
+ val base64Encoder = new BASE64Encoder
+
+ def createEventServiceActor(): ActorRef = {
+ val channelsClient = Storage.getMetaDataChannels()
+ system.actorOf(
+ Props(
+ new EventServiceActor(
+ eventClient,
+ accessKeysClient,
+ channelsClient,
+ EventServerConfig()
+ )
)
)
- )
-
- val base64Encoder = new BASE64Encoder
+ }
"Event Service" should {
- "reject with CredentialsRejected with invalid credentials" in {
+ "reject with CredentialsRejected with invalid credentials" in new StorageMockContext {
+ val eventServiceActor = createEventServiceActor
val accessKey = "abc123:"
val probe = TestProbe()(system)
probe.send(
@@ -119,6 +123,7 @@ class SegmentIOAuthSpec extends Specification {
}
"reject with CredentialsMissed without credentials" in {
+ val eventServiceActor = createEventServiceActor
val probe = TestProbe()(system)
probe.send(
eventServiceActor,
@@ -137,6 +142,7 @@ class SegmentIOAuthSpec extends Specification {
}
"process SegmentIO identity request properly" in {
+ val eventServiceActor = createEventServiceActor
val jsonReq =
"""
|{
@@ -190,3 +196,5 @@ class SegmentIOAuthSpec extends Specification {
step(system.shutdown())
}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala b/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala
new file mode 100644
index 0000000..8476c91
--- /dev/null
+++ b/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage
+
+import org.scalamock.specs2.MockContext
+
+trait StorageMockContext extends MockContext {
+
+ if(!EnvironmentFactory.environmentService.isDefined){
+ val mockedEnvService = mock[EnvironmentService]
+ (mockedEnvService.envKeys _)
+ .expects
+ .returning(List("PIO_STORAGE_REPOSITORIES_METADATA_NAME",
+ "PIO_STORAGE_SOURCES_MYSQL_TYPE",
+ "PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME",
+ "PIO_STORAGE_SOURCES_EVENTDATA_TYPE"))
+ .twice
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_NAME")
+ .returning("test_metadata")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_METADATA_SOURCE")
+ .returning("MYSQL")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME")
+ .returning("test_eventdata")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE")
+ .returning("MYSQL")
+
+ (mockedEnvService.getByKey _)
+ .expects("PIO_STORAGE_SOURCES_MYSQL_TYPE")
+ .returning("jdbc")
+
+ (mockedEnvService.filter _)
+ .expects(*)
+ .returning(Map(
+ "URL" -> "jdbc:h2:~/test;MODE=MySQL;AUTO_SERVER=TRUE",
+ "USERNAME" -> "sa",
+ "PASSWORD" -> "")
+ )
+
+ EnvironmentFactory.environmentService = new Some(mockedEnvService)
+ }
+}