You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by du...@apache.org on 2018/08/20 18:26:43 UTC
[incubator-openwhisk] branch master updated: Treat action code as
attachments (#3945)
This is an automated email from the ASF dual-hosted git repository.
dubeejw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 74ffb4d Treat action code as attachments (#3945)
74ffb4d is described below
commit 74ffb4d759610d394c21dba0008f78e4a3b4d59f
Author: Chetan Mehrotra <ch...@apache.org>
AuthorDate: Mon Aug 20 23:56:39 2018 +0530
Treat action code as attachments (#3945)
* Treat action code as attachments for all newly created or update action runtimes
* Store base64 encoded streams in raw form
* Add test for exec deserialization compatibility
* Support code stored in jar property
* Use octet stream type for binary and text/plain otherwise
Ignore contentType info from manifest
* Reword exception message to state that code can be string on object
* Drop rewrite of jar files as base64 encoded file
* Remove unnecessary special handling of java code
* Fix expected attachmentType
* Add some more compat tests
* Simplify json deserialization based on Markus patch
* Use std charset
* Add support for tweaking code size via CODE_SIZE
If env CODE_SIZE is set then code would be padded with space * CODE_SIZE
* Use attachment for php runtime also
---
ansible/files/runtimes.json | 74 +++++--
.../src/main/scala/whisk/core/entity/Exec.scala | 24 +--
.../main/scala/whisk/core/entity/WhiskAction.scala | 29 +--
.../scala/ColdBlockingInvokeSimulation.scala | 11 +-
.../core/controller/test/ActionsApiTests.scala | 215 ++++++++++++++-------
.../test/AttachmentCompatibilityTests.scala | 94 +++++++--
.../core/database/test/CacheConcurrencyTests.scala | 6 +-
.../ArtifactStoreAttachmentBehaviors.scala | 12 +-
.../scala/whisk/core/entity/test/ExecHelpers.scala | 45 ++++-
.../scala/whisk/core/entity/test/ExecTests.scala | 175 +++++++++++++++++
tools/db/moveCodeToAttachment.py | 23 +--
11 files changed, 550 insertions(+), 158 deletions(-)
diff --git a/ansible/files/runtimes.json b/ansible/files/runtimes.json
index 4efd814..13583d2 100644
--- a/ansible/files/runtimes.json
+++ b/ansible/files/runtimes.json
@@ -8,7 +8,11 @@
"name": "nodejsaction",
"tag": "latest"
},
- "deprecated": true
+ "deprecated": true,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "nodejs:6",
@@ -19,6 +23,10 @@
"tag": "latest"
},
"deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ },
"stemCells": [{
"count": 2,
"memory": "256 MB"
@@ -32,7 +40,11 @@
"name": "action-nodejs-v8",
"tag": "latest"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
}
],
"python": [
@@ -43,7 +55,11 @@
"name": "python2action",
"tag": "latest"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "python:2",
@@ -53,7 +69,11 @@
"name": "python2action",
"tag": "latest"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "python:3",
@@ -62,7 +82,11 @@
"name": "python3action",
"tag": "latest"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
}
],
"swift": [
@@ -73,7 +97,11 @@
"name": "swiftaction",
"tag": "latest"
},
- "deprecated": true
+ "deprecated": true,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "swift:3",
@@ -82,7 +110,11 @@
"name": "swift3action",
"tag": "latest"
},
- "deprecated": true
+ "deprecated": true,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "swift:3.1.1",
@@ -91,7 +123,11 @@
"name": "action-swift-v3.1.1",
"tag": "latest"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "swift:4.1",
@@ -101,7 +137,11 @@
"name": "action-swift-v4.1",
"tag": "latest"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
}
],
"java": [
@@ -115,8 +155,8 @@
},
"deprecated": false,
"attached": {
- "attachmentName": "jarfile",
- "attachmentType": "application/java-archive"
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
},
"requireMain": true
}
@@ -130,6 +170,10 @@
"prefix": "openwhisk",
"name": "action-php-v7.1",
"tag": "latest"
+ },
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
}
},
{
@@ -140,6 +184,10 @@
"prefix": "openwhisk",
"name": "action-php-v7.2",
"tag": "latest"
+ },
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
}
}
],
@@ -148,6 +196,10 @@
"kind": "ruby:2.5",
"default": true,
"deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ },
"image": {
"prefix": "openwhisk",
"name": "action-ruby-v2.5",
diff --git a/common/scala/src/main/scala/whisk/core/entity/Exec.scala b/common/scala/src/main/scala/whisk/core/entity/Exec.scala
index ca77a9d..c98e571 100644
--- a/common/scala/src/main/scala/whisk/core/entity/Exec.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/Exec.scala
@@ -140,14 +140,14 @@ protected[core] case class CodeExecMetaDataAsString(manifest: RuntimeManifest,
protected[core] case class CodeExecAsAttachment(manifest: RuntimeManifest,
override val code: Attachment[String],
- override val entryPoint: Option[String])
+ override val entryPoint: Option[String],
+ override val binary: Boolean = false)
extends CodeExec[Attachment[String]] {
override val kind = manifest.kind
override val image = manifest.image
override val sentinelledLogs = manifest.sentinelledLogs.getOrElse(true)
override val deprecated = manifest.deprecated.getOrElse(false)
override val pull = false
- override lazy val binary = true
override def codeAsJson = code.toJson
def inline(bytes: Array[Byte]): CodeExecAsAttachment = {
@@ -301,22 +301,24 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
}
manifest.attached
- .map { a =>
- val jar: Attachment[String] = {
- // java actions once stored the attachment in "jar" instead of "code"
- obj.fields.get("code").orElse(obj.fields.get("jar"))
- } map {
- attFmt[String].read(_)
- } getOrElse {
+ .map { _ =>
+ // java actions once stored the attachment in "jar" instead of "code"
+ val code = obj.fields.get("code").orElse(obj.fields.get("jar")).getOrElse {
throw new DeserializationException(
- s"'code' must be a valid base64 string in 'exec' for '$kind' actions")
+ s"'code' must be a string or attachment object defined in 'exec' for '$kind' actions")
}
+ val binary: Boolean = code match {
+ case JsString(c) => isBinaryCode(c)
+ case _ => obj.fields.get("binary").map(_.convertTo[Boolean]).getOrElse(false)
+ }
+
val main = optMainField.orElse {
if (manifest.requireMain.exists(identity)) {
throw new DeserializationException(s"'main' must be a string defined in 'exec' for '$kind' actions")
} else None
}
- CodeExecAsAttachment(manifest, jar, main)
+
+ CodeExecAsAttachment(manifest, attFmt[String].read(code), main, binary)
}
.getOrElse {
val code: String = obj.fields.get("code") match {
diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
index 3617ae0..fd4d546 100644
--- a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
@@ -19,8 +19,11 @@ package whisk.core.entity
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
+import java.nio.charset.StandardCharsets
import java.util.Base64
+import akka.http.scaladsl.model.ContentTypes
+
import scala.concurrent.ExecutionContext
import scala.concurrent.Future
import scala.util.{Failure, Success, Try}
@@ -333,23 +336,27 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
require(doc != null, "doc undefined")
} map { _ =>
doc.exec match {
- case exec @ CodeExecAsAttachment(_, Inline(code), _) =>
+ case exec @ CodeExecAsAttachment(_, Inline(code), _, binary) =>
implicit val logger = db.logging
implicit val ec = db.executionContext
- val stream = new ByteArrayInputStream(Base64.getDecoder().decode(code))
- val manifest = exec.manifest.attached.get
+ val (bytes, attachmentType) = if (binary) {
+ (Base64.getDecoder.decode(code), ContentTypes.`application/octet-stream`)
+ } else {
+ (code.getBytes(StandardCharsets.UTF_8), ContentTypes.`text/plain(UTF-8)`)
+ }
+ val stream = new ByteArrayInputStream(bytes)
val oldAttachment = old
.flatMap(_.exec match {
- case CodeExecAsAttachment(_, a: Attached, _) => Some(a)
- case _ => None
+ case CodeExecAsAttachment(_, a: Attached, _, _) => Some(a)
+ case _ => None
})
super.putAndAttach(
db,
doc,
(d, a) => d.copy(exec = exec.attach(a)).revision[WhiskAction](d.rev),
- manifest.attachmentType,
+ attachmentType,
stream,
oldAttachment,
Some { a: WhiskAction =>
@@ -378,12 +385,12 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
fa.flatMap { action =>
action.exec match {
- case exec @ CodeExecAsAttachment(_, attached: Attached, _) =>
+ case exec @ CodeExecAsAttachment(_, attached: Attached, _, binary) =>
val boas = new ByteArrayOutputStream()
- val b64s = Base64.getEncoder().wrap(boas)
+ val wrapped = if (binary) Base64.getEncoder().wrap(boas) else boas
- getAttachment[A](db, action, attached, b64s, Some { a: WhiskAction =>
- b64s.close()
+ getAttachment[A](db, action, attached, wrapped, Some { a: WhiskAction =>
+ wrapped.close()
val newAction = a.copy(exec = exec.inline(boas.toByteArray))
newAction.revision(a.rev)
newAction
@@ -397,7 +404,7 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
def attachmentHandler(action: WhiskAction, attached: Attached): WhiskAction = {
val eu = action.exec match {
- case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _, _, _), _) =>
+ case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _, _, _), _, _) =>
require(
attachmentName == attached.attachmentName,
s"Attachment name '${attached.attachmentName}' does not match the expected name '$attachmentName'")
diff --git a/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala b/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
index d7ec74f..da68e7a 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
@@ -32,6 +32,7 @@ class ColdBlockingInvokeSimulation extends Simulation {
val host = sys.env("OPENWHISK_HOST")
val users: Int = sys.env("USERS").toInt
+ val codeSize: Int = sys.env.getOrElse("CODE_SIZE", "0").toInt
val seconds: FiniteDuration = sys.env.getOrElse("SECONDS", "10").toInt.seconds
val actionsPerUser: Int = sys.env.getOrElse("ACTIONS_PER_USER", "5").toInt
@@ -64,8 +65,7 @@ class ColdBlockingInvokeSimulation extends Simulation {
openWhisk("Create action")
.authenticate(uuid, key)
.action(actionName)
- .create(FileUtils
- .readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8)))
+ .create(actionCode))
}.rendezVous(users)
// Execute all actions for the given amount of time.
.during(seconds) {
@@ -81,6 +81,13 @@ class ColdBlockingInvokeSimulation extends Simulation {
}
}
+ private def actionCode = {
+ val code = FileUtils
+ .readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8)
+ //Pad the code with empty space to increase the stored code size
+ if (codeSize > 0) code + " " * codeSize else code
+ }
+
setUp(test.inject(atOnceUsers(users)))
.protocols(openWhiskProtocol)
// One failure will make the build yellow
diff --git a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
index 7bbf820..fb6f4f6 100644
--- a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
+++ b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
@@ -792,82 +792,91 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi {
}
it should "put and then get an action with attachment from cache" in {
- val action =
+ val javaAction =
WhiskAction(
namespace,
aname(),
javaDefault(nonInlinedCode(entityStore), Some("hello")),
annotations = Parameters("exec", "java"))
- val content = WhiskActionPut(
- Some(action.exec),
- Some(action.parameters),
- Some(ActionLimitsOption(Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs))))
- val name = action.name
- val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)")
- val expectedPutLog =
- Seq(s"uploading attachment '[\\w-]+' of document 'id: ${action.namespace}/${action.name}", s"caching $cacheKey")
- .mkString("(?s).*")
- val notExpectedGetLog = Seq(
- s"finding document: 'id: ${action.namespace}/${action.name}",
- s"finding attachment '[\\w-/:]+' of document 'id: ${action.namespace}/${action.name}").mkString("(?s).*")
+ val nodeAction = WhiskAction(namespace, aname(), jsDefault(nonInlinedCode(entityStore)), Parameters("x", "b"))
+ val swiftAction = WhiskAction(namespace, aname(), swift3(nonInlinedCode(entityStore)), Parameters("x", "b"))
+ val actions = Seq((javaAction, JAVA_DEFAULT), (nodeAction, NODEJS6), (swiftAction, SWIFT3))
- // first request invalidates any previous entries and caches new result
- Put(s"$collectionPath/$name", content) ~> Route.seal(routes(creds)(transid())) ~> check {
- status should be(OK)
- val response = responseAs[WhiskAction]
- response should be(
- WhiskAction(
- action.namespace,
- action.name,
- action.exec,
- action.parameters,
- action.limits,
- action.version,
- action.publish,
- action.annotations ++ Parameters(WhiskAction.execFieldName, JAVA_DEFAULT)))
- }
+ actions.foreach {
+ case (action, kind) =>
+ val content = WhiskActionPut(
+ Some(action.exec),
+ Some(action.parameters),
+ Some(ActionLimitsOption(Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs))))
+ val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)")
- stream.toString should not include (s"invalidating ${CacheKey(action)} on delete")
- stream.toString should include regex (expectedPutLog)
- stream.reset()
+ val expectedPutLog =
+ Seq(
+ s"uploading attachment '[\\w-]+' of document 'id: ${action.namespace}/${action.name}",
+ s"caching $cacheKey")
+ .mkString("(?s).*")
+ val notExpectedGetLog = Seq(
+ s"finding document: 'id: ${action.namespace}/${action.name}",
+ s"finding attachment '[\\w-/:]+' of document 'id: ${action.namespace}/${action.name}").mkString("(?s).*")
- // second request should fetch from cache
- Get(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check {
- status should be(OK)
- val response = responseAs[WhiskAction]
- response should be(
- WhiskAction(
- action.namespace,
- action.name,
- action.exec,
- action.parameters,
- action.limits,
- action.version,
- action.publish,
- action.annotations ++ Parameters(WhiskAction.execFieldName, JAVA_DEFAULT)))
- }
+ // first request invalidates any previous entries and caches new result
+ Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName, kind)))
+ }
- stream.toString should include(s"serving from cache: ${CacheKey(action)}")
- stream.toString should not include regex(notExpectedGetLog)
- stream.reset()
+ stream.toString should not include (s"invalidating ${CacheKey(action)} on delete")
+ stream.toString should include regex (expectedPutLog)
+ stream.reset()
- // delete should invalidate cache
- Delete(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check {
- status should be(OK)
- val response = responseAs[WhiskAction]
- response should be(
- WhiskAction(
- action.namespace,
- action.name,
- action.exec,
- action.parameters,
- action.limits,
- action.version,
- action.publish,
- action.annotations ++ Parameters(WhiskAction.execFieldName, JAVA_DEFAULT)))
+ // second request should fetch from cache
+ Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName, kind)))
+ }
+ stream.toString should include(s"serving from cache: ${CacheKey(action)}")
+ stream.toString should not include regex(notExpectedGetLog)
+ stream.reset()
+
+ // delete should invalidate cache
+ Delete(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName, kind)))
+ }
+
+ stream.toString should include(s"invalidating ${CacheKey(action)}")
+ stream.reset()
}
- stream.toString should include(s"invalidating ${CacheKey(action)}")
- stream.reset()
}
it should "put and then get an action with inlined attachment" in {
@@ -963,8 +972,9 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi {
s"finding attachment '[\\w-/:]+' of document 'id: ${action.namespace}/${action.name}").mkString("(?s).*")
action.exec match {
- case exec @ CodeExecAsAttachment(_, _, _) =>
- val stream = new ByteArrayInputStream(Base64.getDecoder().decode(code))
+ case exec @ CodeExecAsAttachment(_, _, _, binary) =>
+ val bytes = if (binary) Base64.getDecoder().decode(code) else code.getBytes("UTF-8")
+ val stream = new ByteArrayInputStream(bytes)
val manifest = exec.manifest.attached.get
val src = StreamConverters.fromInputStream(() => stream)
putAndAttach[WhiskAction, WhiskEntity](
@@ -1014,8 +1024,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi {
.mkString("(?s).*")
action.exec match {
- case exec @ CodeExecAsAttachment(_, _, _) =>
- val stream = new ByteArrayInputStream(Base64.getDecoder().decode(code))
+ case exec @ CodeExecAsAttachment(_, _, _, _) =>
+ val stream = new ByteArrayInputStream(code.getBytes)
val manifest = exec.manifest.attached.get
val src = StreamConverters.fromInputStream(() => stream)
putAndAttach[WhiskAction, WhiskEntity](
@@ -1065,6 +1075,73 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi {
stream.reset()
}
+ it should "ensure old and new action schemas are supported" in {
+ implicit val tid = transid()
+ val code = nonInlinedCode(entityStore)
+ val actionOldSchema = WhiskAction(namespace, aname(), js6Old(code))
+ val actionNewSchema = WhiskAction(namespace, aname(), jsDefault(code))
+ val content = WhiskActionPut(
+ Some(actionOldSchema.exec),
+ Some(actionOldSchema.parameters),
+ Some(
+ ActionLimitsOption(
+ Some(actionOldSchema.limits.timeout),
+ Some(actionOldSchema.limits.memory),
+ Some(actionOldSchema.limits.logs))))
+ val expectedPutLog =
+ Seq(s"uploading attachment '[\\w-/:]+' of document 'id: ${actionOldSchema.namespace}/${actionOldSchema.name}")
+ .mkString("(?s).*")
+
+ put(entityStore, actionOldSchema)
+
+ stream.toString should not include regex(expectedPutLog)
+ stream.reset()
+
+ Post(s"$collectionPath/${actionOldSchema.name}") ~> Route.seal(routes(creds)) ~> check {
+ status should be(Accepted)
+ val response = responseAs[JsObject]
+ response.fields("activationId") should not be None
+ }
+
+ Put(s"$collectionPath/${actionOldSchema.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check {
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ actionOldSchema.namespace,
+ actionOldSchema.name,
+ actionNewSchema.exec,
+ actionOldSchema.parameters,
+ actionOldSchema.limits,
+ actionOldSchema.version.upPatch,
+ actionOldSchema.publish,
+ actionOldSchema.annotations ++ Parameters(WhiskAction.execFieldName, NODEJS6)))
+ }
+
+ stream.toString should include regex (expectedPutLog)
+ stream.reset()
+
+ Post(s"$collectionPath/${actionOldSchema.name}") ~> Route.seal(routes(creds)) ~> check {
+ status should be(Accepted)
+ val response = responseAs[JsObject]
+ response.fields("activationId") should not be None
+ }
+
+ Delete(s"$collectionPath/${actionOldSchema.name}") ~> Route.seal(routes(creds)) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ actionOldSchema.namespace,
+ actionOldSchema.name,
+ actionNewSchema.exec,
+ actionOldSchema.parameters,
+ actionOldSchema.limits,
+ actionOldSchema.version.upPatch,
+ actionOldSchema.publish,
+ actionOldSchema.annotations ++ Parameters(WhiskAction.execFieldName, NODEJS6)))
+ }
+ }
+
it should "reject put with conflict for pre-existing action" in {
implicit val tid = transid()
val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b"))
diff --git a/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala b/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala
index dc8969f..82b6932 100644
--- a/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala
+++ b/tests/src/test/scala/whisk/core/database/test/AttachmentCompatibilityTests.scala
@@ -30,24 +30,15 @@ import org.scalatest.concurrent.ScalaFutures
import org.scalatest.junit.JUnitRunner
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, FlatSpec, Matchers}
import pureconfig.loadConfigOrThrow
-import spray.json.DefaultJsonProtocol
+import spray.json._
import whisk.common.TransactionId
import whisk.core.ConfigKeys
+import whisk.core.controller.test.WhiskAuthHelpers
import whisk.core.database.memory.MemoryAttachmentStoreProvider
import whisk.core.database.{CouchDbConfig, CouchDbRestClient, CouchDbStoreProvider, NoDocumentException}
import whisk.core.entity.Attachments.Inline
import whisk.core.entity.test.ExecHelpers
-import whisk.core.entity.{
- CodeExecAsAttachment,
- DocInfo,
- EntityName,
- EntityPath,
- WhiskAction,
- WhiskDocumentReader,
- WhiskEntity,
- WhiskEntityJsonFormat,
- WhiskEntityStore
-}
+import whisk.core.entity._
import scala.concurrent.Future
import scala.reflect.classTag
@@ -68,6 +59,10 @@ class AttachmentCompatibilityTests
//Bring in sync the timeout used by ScalaFutures and DBUtils
implicit override val patienceConfig: PatienceConfig = PatienceConfig(timeout = dbOpTimeout)
implicit val materializer = ActorMaterializer()
+
+ val creds = WhiskAuthHelpers.newIdentity()
+ val namespace = EntityPath(creds.subject.asString)
+ def aname() = MakeName.next("action_tests")
val config = loadConfigOrThrow[CouchDbConfig](ConfigKeys.couchdb)
val entityStore = WhiskEntityStore.datastore()
val client =
@@ -117,10 +112,75 @@ class AttachmentCompatibilityTests
doc2.exec shouldBe exec
}
+ it should "read existing base64 encoded code string" in {
+ implicit val tid: TransactionId = transid()
+ val exec = """{
+ | "kind": "nodejs:6",
+ | "code": "SGVsbG8gT3BlbldoaXNr"
+ |}""".stripMargin.parseJson.asJsObject
+ val (id, action) = makeActionJson(namespace, aname(), exec)
+ val info = putDoc(id, action)
+
+ val action2 = WhiskAction.get(entityStore, info.id).futureValue
+ codeExec(action2).codeAsJson shouldBe JsString("SGVsbG8gT3BlbldoaXNr")
+ }
+
+ it should "read existing simple code string" in {
+ implicit val tid: TransactionId = transid()
+ val exec = """{
+ | "kind": "nodejs:6",
+ | "code": "while (true)"
+ |}""".stripMargin.parseJson.asJsObject
+ val (id, action) = makeActionJson(namespace, aname(), exec)
+ val info = putDoc(id, action)
+
+ val action2 = WhiskAction.get(entityStore, info.id).futureValue
+ codeExec(action2).codeAsJson shouldBe JsString("while (true)")
+ }
+
+ private def codeExec(a: WhiskAction) = a.exec.asInstanceOf[CodeExec[_]]
+
+ private def makeActionJson(namespace: EntityPath, name: EntityName, exec: JsObject): (String, JsObject) = {
+ val id = namespace.addPath(name).asString
+ val base = s"""{
+ | "name": "${name.asString}",
+ | "_id": "$id",
+ | "publish": false,
+ | "annotations": [],
+ | "version": "0.0.1",
+ | "updated": 1533623651650,
+ | "entityType": "action",
+ | "parameters": [
+ | {
+ | "key": "x",
+ | "value": "b"
+ | }
+ | ],
+ | "limits": {
+ | "timeout": 60000,
+ | "memory": 256,
+ | "logs": 10
+ | },
+ | "namespace": "${namespace.asString}"
+ |}""".stripMargin.parseJson.asJsObject
+ (id, JsObject(base.fields + ("exec" -> exec)))
+ }
+
+ private def putDoc(id: String, js: JsObject): DocInfo = {
+ val r = client.putDoc(id, js).futureValue
+ r match {
+ case Right(response) =>
+ val info = response.convertTo[DocInfo]
+ docsToDelete += ((entityStore, info))
+ info
+ case _ => fail()
+ }
+ }
+
private def createAction(doc: WhiskAction) = {
implicit val tid: TransactionId = transid()
doc.exec match {
- case exec @ CodeExecAsAttachment(_, Inline(code), _) =>
+ case exec @ CodeExecAsAttachment(_, Inline(code), _, _) =>
val attached = exec.manifest.attached.get
val newDoc = doc.copy(exec = exec.copy(code = attached))
@@ -137,6 +197,14 @@ class AttachmentCompatibilityTests
}
}
+ object MakeName {
+ @volatile var counter = 1
+ def next(prefix: String = "test")(): EntityName = {
+ counter = counter + 1
+ EntityName(s"${prefix}_name$counter")
+ }
+ }
+
private def attach(doc: DocInfo,
name: String,
contentType: ContentType,
diff --git a/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala b/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala
index ec5a3e6..a669296 100644
--- a/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala
+++ b/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala
@@ -121,7 +121,11 @@ class CacheConcurrencyTests extends FlatSpec with WskTestHelpers with WskActorSy
para.tasksupport = new ForkJoinTaskSupport(new ForkJoinPool(nThreads))
para.map { i =>
if (i != 16) {
- wsk.action.get(name)
+ val rr = wsk.action.get(name, expectedExitCode = DONTCARE_EXIT)
+ withClue(s"expecting get to either succeed or fail with not found: $rr") {
+ // some will succeed and some should fail with not found
+ rr.exitCode should (be(SUCCESS_EXIT) or be(NOT_FOUND))
+ }
} else {
wsk.action.create(name, None, parameters = Map("color" -> JsString("blue")), update = true)
}
diff --git a/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreAttachmentBehaviors.scala b/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreAttachmentBehaviors.scala
index 1438e79..ec7fbe6 100644
--- a/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreAttachmentBehaviors.scala
+++ b/tests/src/test/scala/whisk/core/database/test/behavior/ArtifactStoreAttachmentBehaviors.scala
@@ -22,14 +22,15 @@ import java.util.Base64
import akka.http.scaladsl.model.{ContentTypes, Uri}
import akka.stream.IOResult
-import scala.concurrent.duration.DurationInt
import akka.stream.scaladsl.{Sink, StreamConverters}
import akka.util.{ByteString, ByteStringBuilder}
import whisk.common.TransactionId
import whisk.core.database.{AttachmentSupport, CacheChangeNotification, NoDocumentException}
import whisk.core.entity.Attachments.{Attached, Attachment, Inline}
import whisk.core.entity.test.ExecHelpers
-import whisk.core.entity.{CodeExec, DocInfo, EntityName, ExecManifest, WhiskAction}
+import whisk.core.entity.{CodeExec, DocInfo, EntityName, WhiskAction}
+
+import scala.concurrent.duration.DurationInt
trait ArtifactStoreAttachmentBehaviors extends ArtifactStoreBehaviorBase with ExecHelpers {
behavior of s"${storeType}ArtifactStore attachments"
@@ -133,12 +134,7 @@ trait ArtifactStoreAttachmentBehaviors extends ArtifactStoreBehaviorBase with Ex
docsToDelete += ((entityStore, i1))
- attached(action2).attachmentType shouldBe ExecManifest.runtimesManifest
- .resolveDefaultRuntime(JAVA_DEFAULT)
- .get
- .attached
- .get
- .attachmentType
+ attached(action2).attachmentType shouldBe ContentTypes.`application/octet-stream`
attached(action2).length shouldBe Some(size)
attached(action2).digest should not be empty
diff --git a/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala b/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
index 33dd7e1..fa8fd2e 100644
--- a/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
+++ b/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
@@ -50,11 +50,18 @@ trait ExecHelpers extends Matchers with WskActorSystem with StreamLogging {
ExecManifest.runtimesManifest.runtimes.flatMap(_.versions).find(_.kind == name).get.image
}
- protected def js(code: String, main: Option[String] = None) = {
+ protected def jsOld(code: String, main: Option[String] = None) = {
CodeExecAsString(RuntimeManifest(NODEJS, imagename(NODEJS), deprecated = Some(true)), trim(code), main.map(_.trim))
}
- protected def js6(code: String, main: Option[String] = None) = {
+ protected def js(code: String, main: Option[String] = None) = {
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest = ExecManifest.runtimesManifest.resolveDefaultRuntime(NODEJS).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim), Exec.isBinaryCode(code))
+ }
+
+ protected def js6Old(code: String, main: Option[String] = None) = {
CodeExecAsString(
RuntimeManifest(
NODEJS6,
@@ -65,12 +72,18 @@ trait ExecHelpers extends Matchers with WskActorSystem with StreamLogging {
trim(code),
main.map(_.trim))
}
+ protected def js6(code: String, main: Option[String] = None) = {
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest = ExecManifest.runtimesManifest.resolveDefaultRuntime(NODEJS6).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim), Exec.isBinaryCode(code))
+ }
protected def jsDefault(code: String, main: Option[String] = None) = {
js6(code, main)
}
- protected def js6MetaData(main: Option[String] = None, binary: Boolean) = {
+ protected def js6MetaDataOld(main: Option[String] = None, binary: Boolean) = {
CodeExecMetaDataAsString(
RuntimeManifest(
NODEJS6,
@@ -82,11 +95,17 @@ trait ExecHelpers extends Matchers with WskActorSystem with StreamLogging {
main.map(_.trim))
}
+ protected def js6MetaData(main: Option[String] = None, binary: Boolean) = {
+ val manifest = ExecManifest.runtimesManifest.resolveDefaultRuntime(NODEJS6).get
+
+ CodeExecMetaDataAsAttachment(manifest, binary, main.map(_.trim))
+ }
+
protected def javaDefault(code: String, main: Option[String] = None) = {
val attachment = attFmt[String].read(code.trim.toJson)
val manifest = ExecManifest.runtimesManifest.resolveDefaultRuntime(JAVA_DEFAULT).get
- CodeExecAsAttachment(manifest, attachment, main.map(_.trim))
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim), Exec.isBinaryCode(code))
}
protected def javaMetaData(main: Option[String] = None, binary: Boolean) = {
@@ -95,16 +114,22 @@ trait ExecHelpers extends Matchers with WskActorSystem with StreamLogging {
CodeExecMetaDataAsAttachment(manifest, binary, main.map(_.trim))
}
- protected def swift(code: String, main: Option[String] = None) = {
+ protected def swiftOld(code: String, main: Option[String] = None) = {
CodeExecAsString(RuntimeManifest(SWIFT, imagename(SWIFT), deprecated = Some(true)), trim(code), main.map(_.trim))
}
+ protected def swift(code: String, main: Option[String] = None) = {
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest = ExecManifest.runtimesManifest.resolveDefaultRuntime(SWIFT).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim), Exec.isBinaryCode(code))
+ }
+
protected def swift3(code: String, main: Option[String] = None) = {
- val default = ExecManifest.runtimesManifest.resolveDefaultRuntime(SWIFT3).flatMap(_.default)
- CodeExecAsString(
- RuntimeManifest(SWIFT3, imagename(SWIFT3), default = default, deprecated = Some(false)),
- trim(code),
- main.map(_.trim))
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest = ExecManifest.runtimesManifest.resolveDefaultRuntime(SWIFT3).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim), Exec.isBinaryCode(code))
}
protected def sequence(components: Vector[FullyQualifiedEntityName]) = SequenceExec(components)
diff --git a/tests/src/test/scala/whisk/core/entity/test/ExecTests.scala b/tests/src/test/scala/whisk/core/entity/test/ExecTests.scala
new file mode 100644
index 0000000..48a4b0d
--- /dev/null
+++ b/tests/src/test/scala/whisk/core/entity/test/ExecTests.scala
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package whisk.core.entity.test
+
+import common.StreamLogging
+import spray.json._
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import whisk.core.WhiskConfig
+import whisk.core.entity.Attachments.{Attached, Inline}
+import whisk.core.entity.{CodeExecAsAttachment, CodeExecAsString, Exec, ExecManifest, WhiskAction}
+
+import scala.collection.mutable
+
+@RunWith(classOf[JUnitRunner])
+class ExecTests extends FlatSpec with Matchers with StreamLogging with BeforeAndAfterAll {
+ behavior of "exec deserialization"
+
+ val config = new WhiskConfig(ExecManifest.requiredProperties)
+ ExecManifest.initialize(config)
+
+ override protected def afterAll(): Unit = {
+ ExecManifest.initialize(config)
+ super.afterAll()
+ }
+
+ it should "read existing code string as attachment" in {
+ val json = """{
+ | "name": "action_tests_name2",
+ | "_id": "anon-Yzycx8QnIYDp3Tby0Fnj23KcMtH/action_tests_name2",
+ | "publish": false,
+ | "annotations": [],
+ | "version": "0.0.1",
+ | "updated": 1533623651650,
+ | "entityType": "action",
+ | "exec": {
+ | "kind": "nodejs:6",
+ | "code": "foo",
+ | "binary": false
+ | },
+ | "parameters": [
+ | {
+ | "key": "x",
+ | "value": "b"
+ | }
+ | ],
+ | "limits": {
+ | "timeout": 60000,
+ | "memory": 256,
+ | "logs": 10
+ | },
+ | "namespace": "anon-Yzycx8QnIYDp3Tby0Fnj23KcMtH"
+ |}""".stripMargin.parseJson.asJsObject
+ val action = WhiskAction.serdes.read(json)
+ action.exec should matchPattern { case CodeExecAsAttachment(_, Inline("foo"), None, false) => }
+ }
+
+ it should "properly determine binary property" in {
+ val j1 = """{
+ | "kind": "nodejs:6",
+ | "code": "SGVsbG8gT3BlbldoaXNr",
+ | "binary": false
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j1) should matchPattern {
+ case CodeExecAsAttachment(_, Inline("SGVsbG8gT3BlbldoaXNr"), None, true) =>
+ }
+
+ val j2 = """{
+ | "kind": "nodejs:6",
+ | "code": "while (true)",
+ | "binary": false
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j2) should matchPattern {
+ case CodeExecAsAttachment(_, Inline("while (true)"), None, false) =>
+ }
+
+ //Defaults to binary
+ val j3 = """{
+ | "kind": "nodejs:6",
+ | "code": "while (true)"
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j3) should matchPattern {
+ case CodeExecAsAttachment(_, Inline("while (true)"), None, false) =>
+ }
+ }
+
+ it should "read code stored as attachment" in {
+ val json = """{
+ | "kind": "java",
+ | "code": {
+ | "attachmentName": "foo:bar",
+ | "attachmentType": "application/java-archive",
+ | "length": 32768,
+ | "digest": "sha256-foo"
+ | },
+ | "binary": true,
+ | "main": "hello"
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(json) should matchPattern {
+ case CodeExecAsAttachment(_, Attached("foo:bar", _, Some(32768), Some("sha256-foo")), Some("hello"), true) =>
+ }
+ }
+
+ it should "read code stored as jar property" in {
+ val j1 = """{
+ | "kind": "nodejs:6",
+ | "jar": "SGVsbG8gT3BlbldoaXNr",
+ | "binary": false
+ |}""".stripMargin.parseJson.asJsObject
+ Exec.serdes.read(j1) should matchPattern {
+ case CodeExecAsAttachment(_, Inline("SGVsbG8gT3BlbldoaXNr"), None, true) =>
+ }
+ }
+
+ it should "read existing code string as string with old manifest" in {
+ val oldManifestJson =
+ """{
+ | "runtimes": {
+ | "nodejs": [
+ | {
+ | "kind": "nodejs:6",
+ | "default": true,
+ | "image": {
+ | "prefix": "openwhisk",
+ | "name": "nodejs6action",
+ | "tag": "latest"
+ | },
+ | "deprecated": false,
+ | "stemCells": [{
+ | "count": 2,
+ | "memory": "256 MB"
+ | }]
+ | }
+ | ]
+ | }
+ |}""".stripMargin.parseJson.compactPrint
+
+ val oldConfig =
+ new TestConfig(Map(WhiskConfig.runtimesManifest -> oldManifestJson), ExecManifest.requiredProperties)
+ ExecManifest.initialize(oldConfig)
+ val j1 = """{
+ | "kind": "nodejs:6",
+ | "code": "SGVsbG8gT3BlbldoaXNr",
+ | "binary": false
+ |}""".stripMargin.parseJson.asJsObject
+
+ Exec.serdes.read(j1) should matchPattern {
+ case CodeExecAsString(_, "SGVsbG8gT3BlbldoaXNr", None) =>
+ }
+
+ //Reset config back
+ ExecManifest.initialize(config)
+ }
+
+ private class TestConfig(val props: Map[String, String], requiredProperties: Map[String, String])
+ extends WhiskConfig(requiredProperties) {
+ override protected def getProperties() = mutable.Map(props.toSeq: _*)
+ }
+}
diff --git a/tools/db/moveCodeToAttachment.py b/tools/db/moveCodeToAttachment.py
index be90eea..254362c 100755
--- a/tools/db/moveCodeToAttachment.py
+++ b/tools/db/moveCodeToAttachment.py
@@ -21,29 +21,8 @@
import argparse
import couchdb.client
import time
-import base64
from couchdb import ResourceNotFound
-def updateJavaAction(db, doc, id):
- updated = False
- attachment = db.get_attachment(doc, 'jarfile')
-
- if attachment != None:
- encodedAttachment = base64.b64encode(attachment.getvalue())
- db.put_attachment(doc, encodedAttachment, 'codefile', 'text/plain')
- doc = db.get(id)
- doc['exec']['code'] = {
- 'attachmentName': 'codefile',
- 'attachmentType': 'text/plain'
- }
- if 'jar' in doc['exec']:
- del doc['exec']['jar']
- db.save(doc)
- db.delete_attachment(doc, 'jarfile')
- updated = True
-
- return updated
-
def updateNonJavaAction(db, doc, id):
updated = False
code = doc['exec']['code']
@@ -96,7 +75,7 @@ def main(args):
if doc['exec']['kind'] != 'java':
updated = updateNonJavaAction(db, doc, id)
else:
- updated = updateJavaAction(db, doc, id)
+ updated = False
if updated:
print('Updated action: "{0}"'.format(id))