You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by du...@apache.org on 2018/08/29 18:07:46 UTC
[incubator-openwhisk] branch master updated: Treat blackbox action
code as attachment (#3979)
This is an automated email from the ASF dual-hosted git repository.
dubeejw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new fcd0e4b Treat blackbox action code as attachment (#3979)
fcd0e4b is described below
commit fcd0e4b5bf4faf8a99f42868a898fa4e17616ec1
Author: Chetan Mehrotra <ch...@apache.org>
AuthorDate: Wed Aug 29 23:37:42 2018 +0530
Treat blackbox action code as attachment (#3979)
* Use attachments with BlackBoxExec
* Simplify code to avoid code duplication
* Test for blackbox serialization
* Test for cache for blackbox action
---
.../main/scala/whisk/core/entity/Attachments.scala | 7 +
.../src/main/scala/whisk/core/entity/Exec.scala | 58 +++---
.../main/scala/whisk/core/entity/WhiskAction.scala | 103 ++++++-----
.../core/controller/test/ActionsApiTests.scala | 204 ++++++++++-----------
.../test/AttachmentCompatibilityTests.scala | 15 ++
.../scala/whisk/core/database/test/DbUtils.scala | 5 +
.../test/behavior/ArtifactStoreBehaviorBase.scala | 2 +-
.../scala/whisk/core/entity/test/ExecHelpers.scala | 10 +-
.../scala/whisk/core/entity/test/ExecTests.scala | 129 ++++++++++++-
9 files changed, 357 insertions(+), 176 deletions(-)
diff --git a/common/scala/src/main/scala/whisk/core/entity/Attachments.scala b/common/scala/src/main/scala/whisk/core/entity/Attachments.scala
index 2d00a03..dc2c837 100644
--- a/common/scala/src/main/scala/whisk/core/entity/Attachments.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/Attachments.scala
@@ -54,6 +54,13 @@ object Attachments {
}
}
+ implicit class OptionSizeAttachment[T <% SizeConversion](a: Option[Attachment[T]]) extends SizeConversion {
+ def sizeIn(unit: SizeUnits.Unit): ByteSize = a match {
+ case Some(Inline(v)) => (v: SizeConversion).sizeIn(unit)
+ case _ => 0.bytes
+ }
+ }
+
object Attached {
implicit val serdes = {
implicit val contentTypeSerdes = new RootJsonFormat[ContentType] {
diff --git a/common/scala/src/main/scala/whisk/core/entity/Exec.scala b/common/scala/src/main/scala/whisk/core/entity/Exec.scala
index c98e571..8e5d8f6 100644
--- a/common/scala/src/main/scala/whisk/core/entity/Exec.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/Exec.scala
@@ -27,7 +27,6 @@ import spray.json.DefaultJsonProtocol._
import whisk.core.entity.Attachments._
import whisk.core.entity.ExecManifest._
import whisk.core.entity.size.SizeInt
-import whisk.core.entity.size.SizeOptionString
import whisk.core.entity.size.SizeString
/**
@@ -115,6 +114,11 @@ sealed abstract class ExecMetaData extends ExecMetaDataBase {
override def size = 0.B
}
+trait AttachedCode {
+ def inline(bytes: Array[Byte]): Exec
+ def attach(attached: Attached): Exec
+}
+
protected[core] case class CodeExecAsString(manifest: RuntimeManifest,
override val code: String,
override val entryPoint: Option[String])
@@ -142,7 +146,8 @@ protected[core] case class CodeExecAsAttachment(manifest: RuntimeManifest,
override val code: Attachment[String],
override val entryPoint: Option[String],
override val binary: Boolean = false)
- extends CodeExec[Attachment[String]] {
+ extends CodeExec[Attachment[String]]
+ with AttachedCode {
override val kind = manifest.kind
override val image = manifest.image
override val sentinelledLogs = manifest.sentinelledLogs.getOrElse(true)
@@ -150,12 +155,12 @@ protected[core] case class CodeExecAsAttachment(manifest: RuntimeManifest,
override val pull = false
override def codeAsJson = code.toJson
- def inline(bytes: Array[Byte]): CodeExecAsAttachment = {
+ override def inline(bytes: Array[Byte]): CodeExecAsAttachment = {
val encoded = new String(bytes, StandardCharsets.UTF_8)
copy(code = Inline(encoded))
}
- def attach(attached: Attached): CodeExecAsAttachment = {
+ override def attach(attached: Attached): CodeExecAsAttachment = {
copy(code = attached)
}
}
@@ -175,17 +180,27 @@ protected[core] case class CodeExecMetaDataAsAttachment(manifest: RuntimeManifes
* @param code an optional script or zip archive (as base64 encoded) string
*/
protected[core] case class BlackBoxExec(override val image: ImageName,
- override val code: Option[String],
+ override val code: Option[Attachment[String]],
override val entryPoint: Option[String],
- val native: Boolean)
- extends CodeExec[Option[String]] {
+ val native: Boolean,
+ override val binary: Boolean)
+ extends CodeExec[Option[Attachment[String]]]
+ with AttachedCode {
override val kind = Exec.BLACKBOX
override val deprecated = false
override def codeAsJson = code.toJson
- override lazy val binary = code map { Exec.isBinaryCode(_) } getOrElse false
override val sentinelledLogs = native
override val pull = !native
override def size = super.size + image.publicImageName.sizeInBytes
+
+ override def inline(bytes: Array[Byte]): BlackBoxExec = {
+ val encoded = new String(bytes, StandardCharsets.UTF_8)
+ copy(code = Some(Inline(encoded)))
+ }
+
+ override def attach(attached: Attached): BlackBoxExec = {
+ copy(code = Some(attached))
+ }
}
protected[core] case class BlackBoxExecMetaData(override val image: ImageName,
@@ -244,7 +259,7 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
case b: BlackBoxExec =>
val base =
Map("kind" -> JsString(b.kind), "image" -> JsString(b.image.publicImageName), "binary" -> JsBoolean(b.binary))
- val code = b.code.filter(_.trim.nonEmpty).map("code" -> JsString(_))
+ val code = b.code.map("code" -> attFmt[String].write(_))
val main = b.entryPoint.map("main" -> JsString(_))
JsObject(base ++ code ++ main)
case _ => JsObject.empty
@@ -283,15 +298,13 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
throw new DeserializationException(
s"'image' must be a string defined in 'exec' for '${Exec.BLACKBOX}' actions")
}
- val code: Option[String] = obj.fields.get("code") match {
- case Some(JsString(i)) => if (i.trim.nonEmpty) Some(i) else None
- case Some(_) =>
- throw new DeserializationException(
- s"if defined, 'code' must a string defined in 'exec' for '${Exec.BLACKBOX}' actions")
- case None => None
+ val (codeOpt: Option[Attachment[String]], binary) = obj.fields.get("code") match {
+ case None => (None, false)
+ case Some(JsString(i)) if i.trim.isEmpty => (None, false)
+ case Some(code) => (Some(attFmt[String].read(code)), isBinary(code, obj))
}
val native = execManifests.skipDockerPull(image)
- BlackBoxExec(image, code, optMainField, native)
+ BlackBoxExec(image, codeOpt, optMainField, native, binary)
case _ =>
// map "default" virtual runtime versions to the currently blessed actual runtime version
@@ -307,10 +320,6 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
throw new DeserializationException(
s"'code' must be a string or attachment object defined in 'exec' for '$kind' actions")
}
- val binary: Boolean = code match {
- case JsString(c) => isBinaryCode(c)
- case _ => obj.fields.get("binary").map(_.convertTo[Boolean]).getOrElse(false)
- }
val main = optMainField.orElse {
if (manifest.requireMain.exists(identity)) {
@@ -318,7 +327,7 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
} else None
}
- CodeExecAsAttachment(manifest, attFmt[String].read(code), main, binary)
+ CodeExecAsAttachment(manifest, attFmt[String].read(code), main, isBinary(code, obj))
}
.getOrElse {
val code: String = obj.fields.get("code") match {
@@ -340,6 +349,13 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
(t.length > 0) && (t.length % 4 == 0) && isBase64Pattern.matcher(t).matches()
} else false
}
+
+ private def isBinary(code: JsValue, obj: JsObject): Boolean = {
+ code match {
+ case JsString(c) => isBinaryCode(c)
+ case _ => obj.fields.get("binary").map(_.convertTo[Boolean]).getOrElse(false)
+ }
+ }
}
protected[core] object ExecMetaDataBase extends ArgNormalizer[ExecMetaDataBase] with DefaultJsonProtocol {
diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
index fd4d546..55e02a7 100644
--- a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
@@ -17,9 +17,8 @@
package whisk.core.entity
-import java.io.ByteArrayInputStream
-import java.io.ByteArrayOutputStream
-import java.nio.charset.StandardCharsets
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+import java.nio.charset.StandardCharsets.UTF_8
import java.util.Base64
import akka.http.scaladsl.model.ContentTypes
@@ -331,38 +330,31 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
implicit transid: TransactionId,
notifier: Option[CacheChangeNotification]): Future[DocInfo] = {
+ def putWithAttachment(code: String, binary: Boolean, exec: AttachedCode) = {
+ implicit val logger = db.logging
+ implicit val ec = db.executionContext
+
+ val oldAttachment = old.flatMap(getAttachment)
+ val (bytes, attachmentType) = if (binary) {
+ (Base64.getDecoder.decode(code), ContentTypes.`application/octet-stream`)
+ } else {
+ (code.getBytes(UTF_8), ContentTypes.`text/plain(UTF-8)`)
+ }
+ val stream = new ByteArrayInputStream(bytes)
+ super.putAndAttach(db, doc, attachmentUpdater, attachmentType, stream, oldAttachment, Some { a: WhiskAction =>
+ a.copy(exec = exec.inline(code.getBytes(UTF_8)))
+ })
+ }
+
Try {
require(db != null, "db undefined")
require(doc != null, "doc undefined")
} map { _ =>
doc.exec match {
case exec @ CodeExecAsAttachment(_, Inline(code), _, binary) =>
- implicit val logger = db.logging
- implicit val ec = db.executionContext
-
- val (bytes, attachmentType) = if (binary) {
- (Base64.getDecoder.decode(code), ContentTypes.`application/octet-stream`)
- } else {
- (code.getBytes(StandardCharsets.UTF_8), ContentTypes.`text/plain(UTF-8)`)
- }
- val stream = new ByteArrayInputStream(bytes)
- val oldAttachment = old
- .flatMap(_.exec match {
- case CodeExecAsAttachment(_, a: Attached, _, _) => Some(a)
- case _ => None
- })
-
- super.putAndAttach(
- db,
- doc,
- (d, a) => d.copy(exec = exec.attach(a)).revision[WhiskAction](d.rev),
- attachmentType,
- stream,
- oldAttachment,
- Some { a: WhiskAction =>
- a.copy(exec = exec.inline(code.getBytes("UTF-8")))
- })
-
+ putWithAttachment(code, binary, exec)
+ case exec @ BlackBoxExec(_, Some(Inline(code)), _, _, binary) =>
+ putWithAttachment(code, binary, exec)
case _ =>
super.put(db, doc, old)
}
@@ -384,18 +376,23 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
val fa = super.getWithAttachment(db, doc, rev, fromCache, Some(attachmentHandler _))
fa.flatMap { action =>
+ def getWithAttachment(attached: Attached, binary: Boolean, exec: AttachedCode) = {
+ val boas = new ByteArrayOutputStream()
+ val wrapped = if (binary) Base64.getEncoder().wrap(boas) else boas
+
+ getAttachment[A](db, action, attached, wrapped, Some { a: WhiskAction =>
+ wrapped.close()
+ val newAction = a.copy(exec = exec.inline(boas.toByteArray))
+ newAction.revision(a.rev)
+ newAction
+ })
+ }
+
action.exec match {
case exec @ CodeExecAsAttachment(_, attached: Attached, _, binary) =>
- val boas = new ByteArrayOutputStream()
- val wrapped = if (binary) Base64.getEncoder().wrap(boas) else boas
-
- getAttachment[A](db, action, attached, wrapped, Some { a: WhiskAction =>
- wrapped.close()
- val newAction = a.copy(exec = exec.inline(boas.toByteArray))
- newAction.revision(a.rev)
- newAction
- })
-
+ getWithAttachment(attached, binary, exec)
+ case exec @ BlackBoxExec(_, Some(attached: Attached), _, _, binary) =>
+ getWithAttachment(attached, binary, exec)
case _ =>
Future.successful(action)
}
@@ -403,17 +400,39 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
}
def attachmentHandler(action: WhiskAction, attached: Attached): WhiskAction = {
+ def checkName(name: String) = {
+ require(
+ name == attached.attachmentName,
+ s"Attachment name '${attached.attachmentName}' does not match the expected name '$name'")
+ }
val eu = action.exec match {
case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _, _, _), _, _) =>
- require(
- attachmentName == attached.attachmentName,
- s"Attachment name '${attached.attachmentName}' does not match the expected name '$attachmentName'")
+ checkName(attachmentName)
+ exec.attach(attached)
+ case exec @ BlackBoxExec(_, Some(Attached(attachmentName, _, _, _)), _, _, _) =>
+ checkName(attachmentName)
exec.attach(attached)
case exec => exec
}
action.copy(exec = eu).revision[WhiskAction](action.rev)
}
+ def attachmentUpdater(action: WhiskAction, updatedAttachment: Attached): WhiskAction = {
+ action.exec match {
+ case exec: AttachedCode =>
+ action.copy(exec = exec.attach(updatedAttachment)).revision[WhiskAction](action.rev)
+ case _ => action
+ }
+ }
+
+ def getAttachment(action: WhiskAction): Option[Attached] = {
+ action.exec match {
+ case CodeExecAsAttachment(_, a: Attached, _, _) => Some(a)
+ case BlackBoxExec(_, Some(a: Attached), _, _, _) => Some(a)
+ case _ => None
+ }
+ }
+
override def del[Wsuper >: WhiskAction](db: ArtifactStore[Wsuper], doc: DocInfo)(
implicit transid: TransactionId,
notifier: Option[CacheChangeNotification]): Future[Boolean] = {
diff --git a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
index 4b14019..fe4d217 100644
--- a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
+++ b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
@@ -36,11 +36,9 @@ import whisk.core.entitlement.Collection
import whisk.http.ErrorResponse
import whisk.http.Messages
import whisk.core.database.UserContext
-import java.io.ByteArrayInputStream
-import java.util.Base64
import akka.http.scaladsl.model.headers.RawHeader
-import akka.stream.scaladsl._
+import whisk.core.entity.Attachments.Inline
/**
* Tests Actions API.
@@ -591,7 +589,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi {
action.annotations ++ Parameters(WhiskAction.execFieldName, Exec.BLACKBOX)))
response.exec shouldBe an[BlackBoxExec]
val bb = response.exec.asInstanceOf[BlackBoxExec]
- bb.code shouldBe Some("cc")
+ bb.code shouldBe Some(Inline("cc"))
bb.binary shouldBe false
}
}
@@ -801,7 +799,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi {
annotations = Parameters("exec", "java"))
val nodeAction = WhiskAction(namespace, aname(), jsDefault(nonInlinedCode(entityStore)), Parameters("x", "b"))
val swiftAction = WhiskAction(namespace, aname(), swift3(nonInlinedCode(entityStore)), Parameters("x", "b"))
- val actions = Seq((javaAction, JAVA_DEFAULT), (nodeAction, NODEJS6), (swiftAction, SWIFT3))
+ val bbAction = WhiskAction(namespace, aname(), bb("bb", nonInlinedCode(entityStore), Some("bbMain")))
+ val actions = Seq((javaAction, JAVA_DEFAULT), (nodeAction, NODEJS6), (swiftAction, SWIFT3), (bbAction, BLACKBOX))
actions.foreach {
case (action, kind) =>
@@ -959,121 +958,112 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi {
it should "get an action with attachment that is not cached" in {
implicit val tid = transid()
- val code = nonInlinedCode(entityStore)
- val action =
- WhiskAction(namespace, aname(), javaDefault(code, Some("hello")), annotations = Parameters("exec", "java"))
- val content = WhiskActionPut(
- Some(action.exec),
- Some(action.parameters),
- Some(ActionLimitsOption(Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs))))
- val name = action.name
- val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)")
- val expectedGetLog = Seq(
- s"finding document: 'id: ${action.namespace}/${action.name}",
- s"finding attachment '[\\w-/:]+' of document 'id: ${action.namespace}/${action.name}").mkString("(?s).*")
+ val nodeAction = WhiskAction(namespace, aname(), jsDefault(nonInlinedCode(entityStore)), Parameters("x", "b"))
+ val swiftAction = WhiskAction(namespace, aname(), swift3(nonInlinedCode(entityStore)), Parameters("x", "b"))
+ val bbAction = WhiskAction(namespace, aname(), bb("bb", nonInlinedCode(entityStore), Some("bbMain")))
+ val actions = Seq((nodeAction, NODEJS6), (swiftAction, SWIFT3), (bbAction, BLACKBOX))
- action.exec match {
- case exec @ CodeExecAsAttachment(_, _, _, binary) =>
- val bytes = if (binary) Base64.getDecoder().decode(code) else code.getBytes("UTF-8")
- val stream = new ByteArrayInputStream(bytes)
- val manifest = exec.manifest.attached.get
- val src = StreamConverters.fromInputStream(() => stream)
- putAndAttach[WhiskAction, WhiskEntity](
- entityStore,
- action,
- (d, a) => d.copy(exec = exec.attach(a)).revision[WhiskAction](d.rev),
- manifest.attachmentType,
- src,
- None)
+ actions.foreach {
+ case (action, kind) =>
+ val content = WhiskActionPut(
+ Some(action.exec),
+ Some(action.parameters),
+ Some(ActionLimitsOption(Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs))))
+ val name = action.name
+ val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)")
+ val expectedGetLog = Seq(
+ s"finding document: 'id: ${action.namespace}/${action.name}",
+ s"finding attachment '[\\w-/:]+' of document 'id: ${action.namespace}/${action.name}").mkString("(?s).*")
- case _ =>
- }
+ Put(s"$collectionPath/$name", content) ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ }
- // second request should fetch from cache
- Get(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check {
- status should be(OK)
- val response = responseAs[WhiskAction]
- response should be(
- WhiskAction(
- action.namespace,
- action.name,
- action.exec,
- action.parameters,
- action.limits,
- action.version,
- action.publish,
- action.annotations ++ Parameters(WhiskAction.execFieldName, JAVA_DEFAULT)))
- }
+ removeFromCache(action, WhiskAction)
- stream.toString should include regex (expectedGetLog)
- stream.reset()
+ // second request should not fetch from cache
+ Get(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName, kind)))
+ }
+
+ stream.toString should include regex (expectedGetLog)
+ stream.reset()
+ }
}
it should "update an existing action with attachment that is not cached" in {
implicit val tid = transid()
- val code = nonInlinedCode(entityStore)
- val action =
- WhiskAction(namespace, aname(), javaDefault(code, Some("hello")), annotations = Parameters("exec", "java"))
- val content = WhiskActionPut(
- Some(action.exec),
- Some(action.parameters),
- Some(ActionLimitsOption(Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs))))
- val name = action.name
- val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)")
- val expectedPutLog =
- Seq(s"uploading attachment '[\\w-/:]+' of document 'id: ${action.namespace}/${action.name}", s"caching $cacheKey")
- .mkString("(?s).*")
+ val nodeAction = WhiskAction(namespace, aname(), jsDefault(nonInlinedCode(entityStore)), Parameters("x", "b"))
+ val swiftAction = WhiskAction(namespace, aname(), swift3(nonInlinedCode(entityStore)), Parameters("x", "b"))
+ val bbAction = WhiskAction(namespace, aname(), bb("bb", nonInlinedCode(entityStore), Some("bbMain")))
+ val actions = Seq((nodeAction, NODEJS6), (swiftAction, SWIFT3), (bbAction, BLACKBOX))
- action.exec match {
- case exec @ CodeExecAsAttachment(_, _, _, _) =>
- val stream = new ByteArrayInputStream(code.getBytes)
- val manifest = exec.manifest.attached.get
- val src = StreamConverters.fromInputStream(() => stream)
- putAndAttach[WhiskAction, WhiskEntity](
- entityStore,
- action,
- (d, a) => d.copy(exec = exec.attach(a)).revision[WhiskAction](d.rev),
- manifest.attachmentType,
- src,
- None)
+ actions.foreach {
+ case (action, kind) =>
+ val content = WhiskActionPut(
+ Some(action.exec),
+ Some(action.parameters),
+ Some(ActionLimitsOption(Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs))))
+ val name = action.name
+ val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)")
+ val expectedPutLog =
+ Seq(
+ s"uploading attachment '[\\w-/:]+' of document 'id: ${action.namespace}/${action.name}",
+ s"caching $cacheKey")
+ .mkString("(?s).*")
- case _ =>
- }
+ Put(s"$collectionPath/$name", content) ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ }
- Put(s"$collectionPath/$name?overwrite=true", content) ~> Route.seal(routes(creds)(transid())) ~> check {
- status should be(OK)
- val response = responseAs[WhiskAction]
- response should be(
- WhiskAction(
- action.namespace,
- action.name,
- action.exec,
- action.parameters,
- action.limits,
- action.version.upPatch,
- action.publish,
- action.annotations ++ Parameters(WhiskAction.execFieldName, JAVA_DEFAULT)))
- }
- stream.toString should include regex (expectedPutLog)
- stream.reset()
+ removeFromCache(action, WhiskAction)
- // delete should invalidate cache
- Delete(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check {
- status should be(OK)
- val response = responseAs[WhiskAction]
- response should be(
- WhiskAction(
- action.namespace,
- action.name,
- action.exec,
- action.parameters,
- action.limits,
- action.version.upPatch,
- action.publish,
- action.annotations ++ Parameters(WhiskAction.execFieldName, JAVA_DEFAULT)))
+ Put(s"$collectionPath/$name?overwrite=true", content) ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version.upPatch,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName, kind)))
+ }
+ stream.toString should include regex (expectedPutLog)
+ stream.reset()
+
+ // delete should invalidate cache
+ Delete(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version.upPatch,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName, kind)))
+ }
+ stream.toString should include(s"invalidating ${CacheKey(action)}")
+ stream.reset()
}
- stream.toString should include(s"invalidating ${CacheKey(action)}")
- stream.reset()
}
it should "ensure old and new action schemas are supported" in {
diff --git a/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala b/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala
index 82b6932..685f27d 100644
--- a/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala
+++ b/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala
@@ -138,6 +138,21 @@ class AttachmentCompatibilityTests
codeExec(action2).codeAsJson shouldBe JsString("while (true)")
}
+ it should "read existing simple code string for blackbox action" in {
+ implicit val tid: TransactionId = transid()
+ val exec = """{
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "code": "while (true)",
+ | "binary": false
+ |}""".stripMargin.parseJson.asJsObject
+ val (id, action) = makeActionJson(namespace, aname(), exec)
+ val info = putDoc(id, action)
+
+ val action2 = WhiskAction.get(entityStore, info.id).futureValue
+ codeExec(action2).codeAsJson shouldBe JsString("while (true)")
+ }
+
private def codeExec(a: WhiskAction) = a.exec.asInstanceOf[CodeExec[_]]
private def makeActionJson(namespace: EntityPath, name: EntityName, exec: JsObject): (String, JsObject) = {
diff --git a/tests/src/test/scala/whisk/core/database/test/DbUtils.scala b/tests/src/test/scala/whisk/core/database/test/DbUtils.scala
index f4a1aac..28a360d 100644
--- a/tests/src/test/scala/whisk/core/database/test/DbUtils.scala
+++ b/tests/src/test/scala/whisk/core/database/test/DbUtils.scala
@@ -350,6 +350,11 @@ trait DbUtils extends Assertions {
def isMemoryStore(store: ArtifactStore[_]): Boolean = store.isInstanceOf[MemoryArtifactStore[_]]
def isCouchStore(store: ArtifactStore[_]): Boolean = store.isInstanceOf[CouchDbRestStore[_]]
+ protected def removeFromCache[A <: DocumentRevisionProvider](entity: WhiskEntity, factory: DocumentFactory[A])(
+ implicit ec: ExecutionContext): Unit = {
+ factory.removeId(CacheKey(entity))
+ }
+
protected def randomBytes(size: Int): Array[Byte] = {
val arr = new Array[Byte](size)
Random.nextBytes(arr)
diff --git a/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreBehaviorBase.scala b/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreBehaviorBase.scala
index d4bcd5c..8b6e86e 100644
--- a/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreBehaviorBase.scala
+++ b/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreBehaviorBase.scala
@@ -125,7 +125,7 @@ trait ArtifactStoreBehaviorBase
WhiskNamespace(Namespace(EntityName(name), uuid), BasicAuthenticationAuthKey(uuid, Secret()))
}
- private val exec = BlackBoxExec(ExecManifest.ImageName("image"), None, None, native = false)
+ private val exec = BlackBoxExec(ExecManifest.ImageName("image"), None, None, native = false, binary = false)
protected def newAction(ns: EntityPath): WhiskAction = {
WhiskAction(ns, aname(), exec)
diff --git a/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala b/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
index fa8fd2e..7bf608c 100644
--- a/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
+++ b/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
@@ -19,7 +19,6 @@ package whisk.core.entity.test
import org.scalatest.Matchers
import org.scalatest.Suite
-
import common.StreamLogging
import common.WskActorSystem
import whisk.core.WhiskConfig
@@ -27,7 +26,6 @@ import whisk.core.entity._
import whisk.core.entity.ArgNormalizer.trim
import whisk.core.entity.ExecManifest._
import whisk.core.entity.size._
-
import spray.json._
import spray.json.DefaultJsonProtocol._
@@ -41,6 +39,7 @@ trait ExecHelpers extends Matchers with WskActorSystem with StreamLogging {
protected val NODEJS6 = "nodejs:6"
protected val SWIFT = "swift"
protected val SWIFT3 = "swift:3.1.1"
+ protected val BLACKBOX = "blackbox"
protected val SWIFT3_IMAGE = "action-swift-v3.1.1"
protected val JAVA_DEFAULT = "java"
@@ -136,10 +135,13 @@ trait ExecHelpers extends Matchers with WskActorSystem with StreamLogging {
protected def sequenceMetaData(components: Vector[FullyQualifiedEntityName]) = SequenceExecMetaData(components)
- protected def bb(image: String) = BlackBoxExec(ExecManifest.ImageName(trim(image)), None, None, false)
+ protected def bb(image: String) = BlackBoxExec(ExecManifest.ImageName(trim(image)), None, None, false, false)
protected def bb(image: String, code: String, main: Option[String] = None) = {
- BlackBoxExec(ExecManifest.ImageName(trim(image)), Some(trim(code)).filter(_.nonEmpty), main, false)
+ val (codeOpt, binary) =
+ if (code.trim.nonEmpty) (Some(attFmt[String].read(code.toJson)), Exec.isBinaryCode(code))
+ else (None, false)
+ BlackBoxExec(ExecManifest.ImageName(trim(image)), codeOpt, main, false, binary)
}
protected def blackBoxMetaData(image: String, main: Option[String] = None, binary: Boolean) = {
diff --git a/tests/src/test/scala/whisk/core/entity/test/ExecTests.scala b/tests/src/test/scala/whisk/core/entity/test/ExecTests.scala
index 48a4b0d..0a6397b 100644
--- a/tests/src/test/scala/whisk/core/entity/test/ExecTests.scala
+++ b/tests/src/test/scala/whisk/core/entity/test/ExecTests.scala
@@ -17,6 +17,7 @@
package whisk.core.entity.test
+import akka.http.scaladsl.model.ContentTypes
import common.StreamLogging
import spray.json._
import org.junit.runner.RunWith
@@ -24,7 +25,8 @@ import org.scalatest.junit.JUnitRunner
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
import whisk.core.WhiskConfig
import whisk.core.entity.Attachments.{Attached, Inline}
-import whisk.core.entity.{CodeExecAsAttachment, CodeExecAsString, Exec, ExecManifest, WhiskAction}
+import whisk.core.entity.ExecManifest.ImageName
+import whisk.core.entity.{BlackBoxExec, CodeExecAsAttachment, CodeExecAsString, Exec, ExecManifest, WhiskAction}
import scala.collection.mutable
@@ -168,6 +170,131 @@ class ExecTests extends FlatSpec with Matchers with StreamLogging with BeforeAnd
ExecManifest.initialize(config)
}
+ behavior of "blackbox exec deserialization"
+
+ it should "read existing code string as attachment" in {
+ val json = """{
+ | "name": "action_tests_name2",
+ | "_id": "anon-Yzycx8QnIYDp3Tby0Fnj23KcMtH/action_tests_name2",
+ | "publish": false,
+ | "annotations": [],
+ | "version": "0.0.1",
+ | "updated": 1533623651650,
+ | "entityType": "action",
+ | "exec": {
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "code": "foo",
+ | "binary": false
+ | },
+ | "parameters": [
+ | {
+ | "key": "x",
+ | "value": "b"
+ | }
+ | ],
+ | "limits": {
+ | "timeout": 60000,
+ | "memory": 256,
+ | "logs": 10
+ | },
+ | "namespace": "anon-Yzycx8QnIYDp3Tby0Fnj23KcMtH"
+ |}""".stripMargin.parseJson.asJsObject
+ val action = WhiskAction.serdes.read(json)
+ action.exec should matchPattern { case BlackBoxExec(_, Some(Inline("foo")), None, false, false) => }
+ }
+
+ it should "properly determine binary property" in {
+ val j1 = """{
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "code": "SGVsbG8gT3BlbldoaXNr",
+ | "binary": false
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j1) should matchPattern {
+ case BlackBoxExec(_, Some(Inline("SGVsbG8gT3BlbldoaXNr")), None, false, true) =>
+ }
+
+ val j2 = """{
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "code": "while (true)",
+ | "binary": false
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j2) should matchPattern {
+ case BlackBoxExec(_, Some(Inline("while (true)")), None, false, false) =>
+ }
+
+ //Empty code should resolve as None
+ val j3 = """{
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "code": " "
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j3) should matchPattern {
+ case BlackBoxExec(_, None, None, false, false) =>
+ }
+
+ val j4 = """{
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "code": {
+ | "attachmentName": "foo:bar",
+ | "attachmentType": "application/octet-stream",
+ | "length": 32768,
+ | "digest": "sha256-foo"
+ | },
+ | "binary": true,
+ | "main": "hello"
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j4) should matchPattern {
+ case BlackBoxExec(_, Some(Attached("foo:bar", _, Some(32768), Some("sha256-foo"))), Some("hello"), false, true) =>
+ }
+ }
+
+ behavior of "blackbox exec serialization"
+
+ it should "serialize with inline attachment" in {
+ val bb = BlackBoxExec(
+ ImageName.fromString("docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1").get,
+ Some(Inline("foo")),
+ None,
+ false,
+ false)
+ val js = Exec.serdes.write(bb)
+
+ val js2 = """{
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "binary": false,
+ | "code": "foo"
+ |}""".stripMargin.parseJson.asJsObject
+ js shouldBe js2
+ }
+
+ it should "serialize with attached attachment" in {
+ val bb = BlackBoxExec(
+ ImageName.fromString("docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1").get,
+ Some(Attached("foo", ContentTypes.`application/octet-stream`, Some(42), Some("sha1-42"))),
+ None,
+ false,
+ true)
+ val js = Exec.serdes.write(bb)
+
+ val js2 = """{
+ | "kind": "blackbox",
+ | "image": "docker-custom.com/openwhisk-runtime/magic/nodejs:0.0.1",
+ | "binary": true,
+ | "code": {
+ | "attachmentName": "foo",
+ | "attachmentType": "application/octet-stream",
+ | "length": 42,
+ | "digest": "sha1-42"
+ | }
+ |}""".stripMargin.parseJson.asJsObject
+ js shouldBe js2
+ }
+
private class TestConfig(val props: Map[String, String], requiredProperties: Map[String, String])
extends WhiskConfig(requiredProperties) {
override protected def getProperties() = mutable.Map(props.toSeq: _*)