You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ra...@apache.org on 2019/06/26 17:53:20 UTC
[incubator-openwhisk] branch master updated: Openwhisk in a
standalone runnable jar (#4516)
This is an automated email from the ASF dual-hosted git repository.
rabbah pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 2f0155f Openwhisk in a standalone runnable jar (#4516)
2f0155f is described below
commit 2f0155fb750ce8b5eef6d5b0f4e2e2db40e5a037
Author: Chetan Mehrotra <ch...@apache.org>
AuthorDate: Wed Jun 26 23:23:09 2019 +0530
Openwhisk in a standalone runnable jar (#4516)
A standalone controller, using the lean load balancer and an in memory store.
* Use WhiskerControl port 3233 as default!
* Allow disabling read of whisk.properties, required for test run to avoid launched server influenced by whisk.properties generated for the tests. Also ensure that a system property is only set if no existing value found. This ensures that in special case user can still supersede the value being set via explicitly providing a system property override.
* Ensure that MemoryStore is only created when needed
* Fetch logs via `docker logs` command on Mac
* Add a OpenWhisk colored banner
* Package the default runtimes.json
* Include log timestamps and filter till sentinel
* Use host.docker.internal as api host for Mac and Windows
* Enable colored logging
* Add support for --verbose flag to enable debug logging easily
* Color transactionId and source also. Also support disabling color logging if needed
* Expose extension point for format log message
* Api host name is defined via system property
* Add a set of pre flight checks to confirm if OpenWhisk can run properly on this host
* Also check if docker is running
* New StandaloneDockerContainerFactory which adapts as per OS
* Only do pull for images having `openwhisk` prefix. For local image generally named `whisk/` no pull would be done
* Add ./gradlew :core:standalone:bootRun command to run jar directly. Use Spring boot default bootRun target to run the jar. Also copy the jar to bin directory.
* Make test run against standalone server
* Support disabling pull of standard images all together. Required for test runs
* Disable pause/resume support for non linux setups
* Print the wsk and docker cli version
* Update README with details on how to connect to db
* Increase time allowed for server to start to 30 secs and log logs upon timeout failure
* Enable verifySystemShutdown in Mesos tests
---
.../scala/org/apache/openwhisk/common/Config.scala | 20 +-
.../org/apache/openwhisk/common/Logging.scala | 4 +-
.../org/apache/openwhisk/core/WhiskConfig.scala | 12 +-
.../openwhisk/core/containerpool/Container.scala | 2 +
.../openwhisk/core/database/StoreUtils.scala | 17 +-
.../database/cosmosdb/CosmosDBArtifactStore.scala | 2 +-
.../core/database/cosmosdb/CosmosDBUtil.scala | 18 +-
.../core/database/memory/MemoryArtifactStore.scala | 78 +++---
.../apache/openwhisk/http/BasicHttpService.scala | 4 +-
core/controller/src/main/resources/reference.conf | 1 +
.../openwhisk/core/controller/Controller.scala | 13 +-
core/invoker/src/main/resources/application.conf | 5 +
.../containerpool/docker/DockerCliLogStore.scala | 81 ++++++
.../core/containerpool/docker/DockerClient.scala | 4 +-
.../docker/DockerContainerFactory.scala | 2 +-
.../docker/DockerForMacContainerFactory.scala | 11 +-
.../docker/StandaloneDockerContainerFactory.scala | 114 +++++++++
core/standalone/README.md | 193 +++++++++++++++
core/standalone/build.gradle | 56 +++++
.../src/main/resources/logback-standalone.xml | 36 +++
core/standalone/src/main/resources/standalone.conf | 71 ++++++
.../openwhisk/standalone/LogbackConfigurator.scala | 89 +++++++
.../openwhisk/standalone/PreFlightChecks.scala | 140 +++++++++++
.../openwhisk/standalone/StandaloneOpenWhisk.scala | 274 +++++++++++++++++++++
settings.gradle | 1 +
tests/build.gradle | 26 +-
tests/src/test/scala/common/FreePortFinder.scala | 33 +++
tests/src/test/scala/common/WhiskProperties.java | 2 +-
.../mesos/test/MesosContainerFactoryTest.scala | 2 +-
.../memory/MemoryArtifactStoreBehaviorBase.scala | 5 +
.../memory/MemoryAttachmentStoreTests.scala | 5 +
.../s3/S3AttachmentStoreBehaviorBase.scala | 7 +-
.../test/behavior/ArtifactStoreCRUDBehaviors.scala | 14 ++
.../standalone/StandaloneServerFixture.scala | 147 +++++++++++
.../StandaloneServerTests.scala} | 22 +-
.../scala/system/basic/WskRestBasicTests.scala | 2 +-
tools/build/redo | 8 +-
tools/travis/distDocker.sh | 1 +
38 files changed, 1418 insertions(+), 104 deletions(-)
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala b/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala
index b9fd333..91d5e65 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala
@@ -116,14 +116,14 @@ class Config(requiredProperties: Map[String, String], optionalProperties: Set[St
*/
protected def getProperties(): scala.collection.mutable.Map[String, String] = {
val required = scala.collection.mutable.Map[String, String]() ++= requiredProperties
- Config.readPropertiesFromEnvironment(required, env)
+ Config.readPropertiesFromSystemAndEnv(required, env)
// for optional value, assign them a default from the required properties list
// to prevent loss of a default value on a required property that may not otherwise be defined
val optional = scala.collection.mutable.Map[String, String]() ++= optionalProperties.map { k =>
k -> required.getOrElse(k, null)
}
- Config.readPropertiesFromEnvironment(optional, env)
+ Config.readPropertiesFromSystemAndEnv(optional, env)
required ++ optional
}
@@ -133,13 +133,15 @@ class Config(requiredProperties: Map[String, String], optionalProperties: Set[St
* Singleton object which provides global methods to manage configuration.
*/
object Config {
+ val prefix = "whisk-config."
/**
* Reads a Map of key-value pairs from the environment -- store them in the
* mutable properties object.
*/
- def readPropertiesFromEnvironment(properties: scala.collection.mutable.Map[String, String], env: Map[String, String])(
- implicit logging: Logging) = {
+ def readPropertiesFromSystemAndEnv(properties: scala.collection.mutable.Map[String, String],
+ env: Map[String, String])(implicit logging: Logging) = {
+ readPropertiesFromSystem(properties)
for (p <- properties.keys) {
val envp = p.replace('.', '_').toUpperCase
val envv = env.get(envp)
@@ -150,6 +152,16 @@ object Config {
}
}
+ def readPropertiesFromSystem(properties: scala.collection.mutable.Map[String, String])(implicit logging: Logging) = {
+ for (p <- properties.keys) {
+ val sysv = Option(System.getProperty(prefix + p))
+ if (sysv.isDefined) {
+ logging.info(this, s"system set value for $p")
+ properties += p -> sysv.get.trim
+ }
+ }
+ }
+
/**
* Checks that the properties object defines all the required properties.
*
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala
index e27d043..01c9754 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala
@@ -95,10 +95,12 @@ class AkkaLogging(loggingAdapter: LoggingAdapter) extends Logging {
val logmsg: String = message // generates the message
if (logmsg.nonEmpty) { // log it only if its not empty
val name = if (from.isInstanceOf[String]) from else Logging.getCleanSimpleClassName(from.getClass)
- loggingAdapter.log(loglevel, s"[$id] [$name] $logmsg")
+ loggingAdapter.log(loglevel, format(id, name.toString, logmsg))
}
}
}
+
+ protected def format(id: TransactionId, name: String, logmsg: String) = s"[$id] [$name] $logmsg"
}
/**
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala
index 4a0fb4f..6283033 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala
@@ -49,10 +49,14 @@ class WhiskConfig(requiredProperties: Map[String, String],
*/
override protected def getProperties() = {
val properties = super.getProperties()
- WhiskConfig.readPropertiesFromFile(properties, Option(propertiesFile) getOrElse (WhiskConfig.whiskPropertiesFile))
+ if (!disableReadFromFile()) {
+ WhiskConfig.readPropertiesFromFile(properties, Option(propertiesFile) getOrElse (WhiskConfig.whiskPropertiesFile))
+ }
properties
}
+ private def disableReadFromFile() = java.lang.Boolean.getBoolean(WhiskConfig.disableWhiskPropsFileRead)
+
val servicePort = this(WhiskConfig.servicePort)
val dockerEndpoint = this(WhiskConfig.dockerEndpoint)
val dockerPort = this(WhiskConfig.dockerPort)
@@ -85,6 +89,7 @@ class WhiskConfig(requiredProperties: Map[String, String],
}
object WhiskConfig {
+ val disableWhiskPropsFileRead = Config.prefix + "disable.whisks.props.file.read"
/**
* Reads a key from system environment as if it was part of WhiskConfig.
@@ -170,7 +175,7 @@ object WhiskConfig {
val kafkaHostList = "kafka.hosts"
val zookeeperHostList = "zookeeper.hosts"
- private val edgeHostApiPort = "edge.host.apiport"
+ val edgeHostApiPort = "edge.host.apiport"
val invokerHostsList = "invoker.hosts"
val dbHostsList = "db.hostsList"
@@ -217,6 +222,7 @@ object ConfigKeys {
val docker = "whisk.docker"
val dockerClient = s"$docker.client"
val dockerContainerFactory = s"$docker.container-factory"
+ val standaloneDockerContainerFactory = s"$docker.standalone.container-factory"
val runc = "whisk.runc"
val runcTimeouts = s"$runc.timeouts"
@@ -255,4 +261,6 @@ object ConfigKeys {
val metrics = "whisk.metrics"
val featureFlags = "whisk.feature-flags"
+
+ val whiskConfig = "whisk.config"
}
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala
index 1da3e2f..45662c2 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala
@@ -82,6 +82,8 @@ trait Container {
protected var containerHttpMaxConcurrent: Int = 1
protected var containerHttpTimeout: FiniteDuration = 60.seconds
+ def containerId: ContainerId = id
+
/** Stops the container from consuming CPU cycles. NOT thread-safe - caller must synchronize. */
def suspend()(implicit transid: TransactionId): Future[Unit] = {
//close connection first, then close connection pool
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala
index 2c168d4..1300a4c 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala
@@ -24,7 +24,7 @@ import akka.stream.SinkShape
import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Keep, Sink}
import akka.util.ByteString
import spray.json.DefaultJsonProtocol._
-import spray.json.{JsObject, RootJsonFormat}
+import spray.json.{JsObject, JsValue, RootJsonFormat}
import org.apache.openwhisk.common.{Logging, StartMarker, TransactionId}
import org.apache.openwhisk.core.entity.{DocInfo, DocRevision, DocumentReader, WhiskDocument}
@@ -97,6 +97,21 @@ private[database] object StoreUtils {
s"$encodedAlgoName-$digest"
}
+ /**
+ * Transforms a json object by adding and removing fields
+ *
+ * @param json base json object to transform
+ * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored
+ * @param fieldsToRemove list of field names to remove
+ * @return transformed json
+ */
+ def transform(json: JsObject,
+ fieldsToAdd: Seq[(String, Option[JsValue])],
+ fieldsToRemove: Seq[String] = Seq.empty): JsObject = {
+ val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove
+ JsObject(fields)
+ }
+
private def combineResult[T](digest: Future[String], length: Future[Long], upload: Future[T])(
implicit ec: ExecutionContext) = {
for {
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala
index c635092..1ddc8b8 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala
@@ -28,7 +28,7 @@ import com.microsoft.azure.cosmosdb.internal.Constants.Properties
import com.microsoft.azure.cosmosdb.rx.AsyncDocumentClient
import kamon.metric.MeasurementUnit
import org.apache.openwhisk.common.{LogMarkerToken, Logging, LoggingMarkers, MetricEmitter, Scheduler, TransactionId}
-import org.apache.openwhisk.core.database.StoreUtils.{checkDocHasRevision, deserialize, reportFailure}
+import org.apache.openwhisk.core.database.StoreUtils._
import org.apache.openwhisk.core.database._
import org.apache.openwhisk.core.database.cosmosdb.CosmosDBArtifactStoreProvider.DocumentClientRef
import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants._
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala
index 2953cd8..fbe1b49 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala
@@ -19,7 +19,8 @@ package org.apache.openwhisk.core.database.cosmosdb
import com.microsoft.azure.cosmosdb.internal.Constants.Properties.{AGGREGATE, E_TAG, ID, SELF_LINK}
import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants._
-import spray.json.{JsObject, JsString, JsValue}
+import org.apache.openwhisk.core.database.StoreUtils.transform
+import spray.json.{JsObject, JsString}
import scala.collection.immutable.Iterable
@@ -124,21 +125,6 @@ private[cosmosdb] trait CosmosDBUtil {
transform(stripInternalFields(js), fieldsToAdd, Seq.empty)
}
- /**
- * Transforms a json object by adding and removing fields
- *
- * @param json base json object to transform
- * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored
- * @param fieldsToRemove list of field names to remove
- * @return transformed json
- */
- def transform(json: JsObject,
- fieldsToAdd: Seq[(String, Option[JsValue])],
- fieldsToRemove: Seq[String] = Seq.empty): JsObject = {
- val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove
- JsObject(fields)
- }
-
private def stripInternalFields(js: JsObject) = {
//Strip out all field name starting with '_' which are considered as db specific internal fields
JsObject(js.fields.filter { case (k, _) => !k.startsWith("_") && k != cid })
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala
index 049bcbe..4e9c096 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala
@@ -17,13 +17,13 @@
package org.apache.openwhisk.core.database.memory
+import java.nio.charset.StandardCharsets.UTF_8
+
import akka.actor.ActorSystem
import akka.http.scaladsl.model.{ContentType, Uri}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.{Sink, Source}
import akka.util.ByteString
-import pureconfig.loadConfigOrThrow
-import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, RootJsonFormat}
import org.apache.openwhisk.common.{Logging, LoggingMarkers, TransactionId}
import org.apache.openwhisk.core.ConfigKeys
import org.apache.openwhisk.core.database.StoreUtils._
@@ -32,6 +32,8 @@ import org.apache.openwhisk.core.entity.Attachments.Attached
import org.apache.openwhisk.core.entity._
import org.apache.openwhisk.core.entity.size._
import org.apache.openwhisk.http.Messages
+import pureconfig.loadConfigOrThrow
+import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, RootJsonFormat}
import scala.collection.concurrent.TrieMap
import scala.concurrent.{ExecutionContext, Future}
@@ -39,6 +41,7 @@ import scala.reflect.ClassTag
import scala.util.{Failure, Success, Try}
object MemoryArtifactStoreProvider extends ArtifactStoreProvider {
+ private val stores = new TrieMap[String, MemoryArtifactStore[_]]()
override def makeStore[D <: DocumentSerializer: ClassTag](useBatching: Boolean)(
implicit jsonFormat: RootJsonFormat[D],
docReader: DocumentReader,
@@ -58,9 +61,12 @@ object MemoryArtifactStoreProvider extends ArtifactStoreProvider {
val classTag = implicitly[ClassTag[D]]
val (dbName, handler, viewMapper) = handlerAndMapper(classTag)
val inliningConfig = loadConfigOrThrow[InliningConfig](ConfigKeys.db)
- new MemoryArtifactStore(dbName, handler, viewMapper, inliningConfig, attachmentStore)
+ val storeFactory = () => new MemoryArtifactStore(dbName, handler, viewMapper, inliningConfig, attachmentStore)
+ stores.getOrElseUpdate(dbName, storeFactory.apply()).asInstanceOf[ArtifactStore[D]]
}
+ def purgeAll(): Unit = stores.clear()
+
private def handlerAndMapper[D](entityType: ClassTag[D])(
implicit actorSystem: ActorSystem,
logging: Logging,
@@ -96,6 +102,8 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
with DocumentProvider
with AttachmentSupport[DocumentAbstraction] {
+ logging.info(this, s"Created MemoryStore for [$dbName]")
+
override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher
private val artifacts = new TrieMap[String, Artifact]
@@ -110,22 +118,25 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
val id = asJson.fields(_id).convertTo[String].trim
require(!id.isEmpty, "document id must be defined")
- val rev: Int = getRevision(asJson)
- val docinfoStr = s"id: $id, rev: $rev"
+ val (oldRev, newRev) = computeRevision(asJson)
+ val docinfoStr = s"id: $id, rev: ${oldRev.getOrElse("null")}"
val start = transid.started(this, LoggingMarkers.DATABASE_SAVE, s"[PUT] '$dbName' saving document: '$docinfoStr'")
- val existing = Artifact(id, rev, asJson)
- val updated = existing.incrementRev()
+ val updated = Artifact(id, newRev, asJson)
val t = Try[DocInfo] {
- if (rev == 0) {
- artifacts.putIfAbsent(id, updated) match {
- case Some(_) => throw DocumentConflictException("conflict on 'put'")
- case None => updated.docInfo
- }
- } else if (artifacts.replace(id, existing, updated)) {
- updated.docInfo
- } else {
- throw DocumentConflictException("conflict on 'put'")
+ oldRev match {
+ case Some(rev) =>
+ val existing = Artifact(id, rev, asJson)
+ if (artifacts.replace(id, existing, updated)) {
+ updated.docInfo
+ } else {
+ throw DocumentConflictException("conflict on 'put'")
+ }
+ case None =>
+ artifacts.putIfAbsent(id, updated) match {
+ case Some(_) => throw DocumentConflictException("conflict on 'put'")
+ case None => updated.docInfo
+ }
}
}
@@ -285,7 +296,6 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
}
override def shutdown(): Unit = {
- artifacts.clear()
attachmentStore.shutdown()
}
@@ -308,38 +318,34 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
reportFailure(f, start, failure => s"[GET] '$dbName' internal error, doc: '$id', failure: '${failure.getMessage}'")
}
- private def getRevision(asJson: JsObject) = {
- asJson.fields.get(_rev) match {
- case Some(JsString(r)) => r.toInt
- case _ => 0
+ private def computeRevision(js: JsObject): (Option[String], String) = {
+ js.fields.get(_rev) match {
+ case Some(JsString(r)) => (Some(r), digest(js))
+ case _ => (None, digest(js))
}
}
+ private def digest(js: JsObject) = {
+ val jsWithoutRev = transform(js, Seq.empty, Seq(_rev))
+ val md = emptyDigest()
+ encodeDigest(md.digest(jsWithoutRev.compactPrint.getBytes(UTF_8)))
+ }
+
//Use curried case class to allow equals support only for id and rev
//This allows us to implement atomic replace and remove which check
//for id,rev equality only
- private case class Artifact(id: String, rev: Int)(val doc: JsObject, val computed: JsObject) {
- def incrementRev(): Artifact = {
- val (newRev, updatedDoc) = incrementAndGet()
- copy(rev = newRev)(updatedDoc, computed) //With Couch attachments are lost post update
- }
-
+ private case class Artifact(id: String, rev: String)(val doc: JsObject, val computed: JsObject) {
def docInfo = DocInfo(DocId(id), DocRevision(rev.toString))
-
- private def incrementAndGet() = {
- val newRev = rev + 1
- val updatedDoc = JsObject(doc.fields + (_rev -> JsString(newRev.toString)))
- (newRev, updatedDoc)
- }
}
private object Artifact {
- def apply(id: String, rev: Int, doc: JsObject): Artifact = {
- Artifact(id, rev)(doc, documentHandler.computedFields(doc))
+ def apply(id: String, rev: String, doc: JsObject): Artifact = {
+ val docWithRev = transform(doc, Seq((_rev, Some(JsString(rev)))))
+ Artifact(id, rev)(docWithRev, documentHandler.computedFields(doc))
}
def apply(info: DocInfo): Artifact = {
- Artifact(info.id.id, info.rev.rev.toInt)(JsObject.empty, JsObject.empty)
+ Artifact(info.id.id, info.rev.rev)(JsObject.empty, JsObject.empty)
}
}
}
diff --git a/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala b/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala
index e734d7a..9e9d965 100644
--- a/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala
+++ b/common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala
@@ -167,11 +167,11 @@ object BasicHttpService {
/**
* Starts an HTTP(S) route handler on given port and registers a shutdown hook.
*/
- def startHttpService(route: Route, port: Int, config: Option[HttpsConfig] = None)(
+ def startHttpService(route: Route, port: Int, config: Option[HttpsConfig] = None, interface: String = "0.0.0.0")(
implicit actorSystem: ActorSystem,
materializer: ActorMaterializer): Unit = {
val connectionContext = config.map(Https.connectionContext(_)).getOrElse(HttpConnectionContext)
- val httpBinding = Http().bindAndHandle(route, "0.0.0.0", port, connectionContext = connectionContext)
+ val httpBinding = Http().bindAndHandle(route, interface, port, connectionContext = connectionContext)
addShutdownHook(httpBinding)
}
diff --git a/core/controller/src/main/resources/reference.conf b/core/controller/src/main/resources/reference.conf
index a804020..55322de 100644
--- a/core/controller/src/main/resources/reference.conf
+++ b/core/controller/src/main/resources/reference.conf
@@ -29,5 +29,6 @@ whisk {
}
controller {
protocol: http
+ interface: "0.0.0.0"
}
}
diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
index 44a7099..88e9713 100644
--- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
+++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
@@ -170,6 +170,7 @@ class Controller(val instance: ControllerInstanceId,
object Controller {
protected val protocol = loadConfigOrThrow[String]("whisk.controller.protocol")
+ protected val interface = loadConfigOrThrow[String]("whisk.controller.interface")
// requiredProperties is a Map whose keys define properties that must be bound to
// a value, and whose values are default values. A null value in the Map means there is
@@ -207,10 +208,14 @@ object Controller {
"runtimes" -> runtimes.toJson)
def main(args: Array[String]): Unit = {
- ConfigMXBean.register()
- Kamon.loadReportersFromConfig()
implicit val actorSystem = ActorSystem("controller-actor-system")
implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this))
+ start(args)
+ }
+
+ def start(args: Array[String])(implicit actorSystem: ActorSystem, logger: Logging): Unit = {
+ ConfigMXBean.register()
+ Kamon.loadReportersFromConfig()
// Prepare Kamon shutdown
CoordinatedShutdown(actorSystem).addTask(CoordinatedShutdown.PhaseActorSystemTerminate, "shutdownKamon") { () =>
@@ -263,7 +268,9 @@ object Controller {
val httpsConfig =
if (Controller.protocol == "https") Some(loadConfigOrThrow[HttpsConfig]("whisk.controller.https")) else None
- BasicHttpService.startHttpService(controller.route, port, httpsConfig)(actorSystem, controller.materializer)
+ BasicHttpService.startHttpService(controller.route, port, httpsConfig, interface)(
+ actorSystem,
+ controller.materializer)
case Failure(t) =>
abort(s"Invalid runtimes manifest: $t")
diff --git a/core/invoker/src/main/resources/application.conf b/core/invoker/src/main/resources/application.conf
index 938af2a..4f36209 100644
--- a/core/invoker/src/main/resources/application.conf
+++ b/core/invoker/src/main/resources/application.conf
@@ -51,6 +51,11 @@ whisk {
use-runc: true
}
+ docker.standalone.container-factory {
+ #If enabled then pull would also be attempted for standard OpenWhisk images under`openwhisk` prefix
+ pull-standard-images: false
+ }
+
container-pool {
user-memory: 1024 m
concurrent-peek-factor: 0.5 #factor used to limit message peeking: 0 < factor <= 1.0 - larger number improves concurrent processing, but increases risk of message loss during invoker crash
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala
new file mode 100644
index 0000000..89f6a8f
--- /dev/null
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerCliLogStore.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.core.containerpool.docker
+
+import java.time.Instant
+
+import akka.actor.ActorSystem
+import org.apache.openwhisk.common.{AkkaLogging, Logging, TransactionId}
+import org.apache.openwhisk.core.containerpool.Container.ACTIVATION_LOG_SENTINEL
+import org.apache.openwhisk.core.containerpool.logging.{DockerToActivationLogStore, LogStore, LogStoreProvider}
+import org.apache.openwhisk.core.containerpool.{Container, ContainerId}
+import org.apache.openwhisk.core.entity.{ActivationLogs, ExecutableWhiskAction, Identity, WhiskActivation}
+
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+
+/**
+ * Docker based log store implementation which fetches logs via cli command.
+ * This mode is inefficient and is only provided for running in developer modes
+ */
+object DockerCliLogStoreProvider extends LogStoreProvider {
+ override def instance(actorSystem: ActorSystem): LogStore = {
+ //Logger is currently not passed implicitly to LogStoreProvider. So create one explicitly
+ implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this))
+ new DockerCliLogStore(actorSystem)
+ }
+}
+
+class DockerCliLogStore(system: ActorSystem)(implicit log: Logging) extends DockerToActivationLogStore(system) {
+ private val client = new ExtendedDockerClient()(system.dispatcher)(log, system)
+ override def collectLogs(transid: TransactionId,
+ user: Identity,
+ activation: WhiskActivation,
+ container: Container,
+ action: ExecutableWhiskAction): Future[ActivationLogs] = {
+ client
+ .collectLogs(container.containerId, activation.start, activation.end)(transid)
+ .map(logs => ActivationLogs(logs.linesIterator.takeWhile(!_.contains(ACTIVATION_LOG_SENTINEL)).toVector))
+ }
+}
+
+class ExtendedDockerClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)(implicit log: Logging,
+ as: ActorSystem)
+ extends DockerClientWithFileAccess(dockerHost)(executionContext)
+ with DockerApiWithFileAccess
+ with WindowsDockerClient {
+
+ implicit private val ec: ExecutionContext = executionContext
+ private val waitForLogs: FiniteDuration = 2.seconds
+ private val logTimeSpanMargin = 1.second
+
+ def collectLogs(id: ContainerId, since: Instant, untill: Instant)(implicit transid: TransactionId): Future[String] = {
+ //Add a slight buffer to account for delay writes of logs
+ val end = untill.plusSeconds(logTimeSpanMargin.toSeconds)
+ runCmd(
+ Seq(
+ "logs",
+ id.asString,
+ "--since",
+ since.getEpochSecond.toString,
+ "--until",
+ end.getEpochSecond.toString,
+ "--timestamps"),
+ waitForLogs)
+ }
+}
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
index d204258..fbf9d6e 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
@@ -87,7 +87,7 @@ class DockerClient(dockerHost: Option[String] = None,
// Determines how to run docker. Failure to find a Docker binary implies
// a failure to initialize this instance of DockerClient.
protected val dockerCmd: Seq[String] = {
- val alternatives = List("/usr/bin/docker", "/usr/local/bin/docker")
+ val alternatives = List("/usr/bin/docker", "/usr/local/bin/docker") ++ executableAlternatives
val dockerBin = Try {
alternatives.find(a => Files.isExecutable(Paths.get(a))).get
@@ -99,6 +99,8 @@ class DockerClient(dockerHost: Option[String] = None,
Seq(dockerBin) ++ host
}
+ protected def executableAlternatives: List[String] = List.empty
+
// Invoke docker CLI to determine client version.
// If the docker client version cannot be determined, an exception will be thrown and instance initialization will fail.
// Rationale: if we cannot invoke `docker version` successfully, it is unlikely subsequent `docker` invocations will succeed.
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
index 9035d96..94e5119 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
@@ -42,7 +42,7 @@ class DockerContainerFactory(instance: InvokerInstanceId,
parameters: Map[String, Set[String]],
containerArgsConfig: ContainerArgsConfig =
loadConfigOrThrow[ContainerArgsConfig](ConfigKeys.containerArgs),
- runtimesRegistryConfig: RuntimesRegistryConfig =
+ protected val runtimesRegistryConfig: RuntimesRegistryConfig =
loadConfigOrThrow[RuntimesRegistryConfig](ConfigKeys.runtimesRegistry),
dockerContainerFactoryConfig: DockerContainerFactoryConfig =
loadConfigOrThrow[DockerContainerFactoryConfig](ConfigKeys.dockerContainerFactory))(
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala
index fb7bc2e..97d265c 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala
@@ -64,12 +64,17 @@ class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: Ex
override def inspectIPAddress(id: ContainerId, network: String)(
implicit transid: TransactionId): Future[ContainerAddress] = {
super
- .runCmd(
- Seq("inspect", "--format", """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}""", id.asString),
- 10.seconds)
+ .runCmd(Seq("inspect", "--format", inspectCommand, id.asString), 10.seconds)
.flatMap {
case "<no value>" => Future.failed(new NoSuchElementException)
case stdout => Future.successful(ContainerAddress("localhost", stdout.toInt))
}
}
+
+ def inspectCommand: String = """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}"""
+
+ //Pause unpause is causing issue on non Linux setups. So disable by default
+ override def pause(id: ContainerId)(implicit transid: TransactionId): Future[Unit] = Future.successful(())
+
+ override def unpause(id: ContainerId)(implicit transid: TransactionId): Future[Unit] = Future.successful(())
}
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala
new file mode 100644
index 0000000..60df1b8
--- /dev/null
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/StandaloneDockerContainerFactory.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.core.containerpool.docker
+
+import akka.actor.ActorSystem
+import org.apache.commons.lang3.SystemUtils
+import org.apache.openwhisk.common.{Logging, TransactionId}
+import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig}
+import org.apache.openwhisk.core.containerpool.{Container, ContainerFactory, ContainerFactoryProvider}
+import org.apache.openwhisk.core.entity.{ByteSize, ExecManifest, InvokerInstanceId}
+import pureconfig.{loadConfig, loadConfigOrThrow}
+
+import scala.collection.concurrent.TrieMap
+import scala.concurrent.{ExecutionContext, Future}
+
+object StandaloneDockerContainerFactoryProvider extends ContainerFactoryProvider {
+ override def instance(actorSystem: ActorSystem,
+ logging: Logging,
+ config: WhiskConfig,
+ instanceId: InvokerInstanceId,
+ parameters: Map[String, Set[String]]): ContainerFactory = {
+ val client =
+ if (SystemUtils.IS_OS_MAC) new DockerForMacClient()(actorSystem.dispatcher)(logging, actorSystem)
+ else if (SystemUtils.IS_OS_WINDOWS) new DockerForWindowsClient()(actorSystem.dispatcher)(logging, actorSystem)
+ else new DockerClientWithFileAccess()(actorSystem.dispatcher)(logging, actorSystem)
+
+ new StandaloneDockerContainerFactory(instanceId, parameters)(
+ actorSystem,
+ actorSystem.dispatcher,
+ logging,
+ client,
+ new RuncClient()(actorSystem.dispatcher)(logging, actorSystem))
+ }
+}
+
+case class StandaloneDockerConfig(pullStandardImages: Boolean)
+
+class StandaloneDockerContainerFactory(instance: InvokerInstanceId, parameters: Map[String, Set[String]])(
+ implicit actorSystem: ActorSystem,
+ ec: ExecutionContext,
+ logging: Logging,
+ docker: DockerApiWithFileAccess,
+ runc: RuncApi)
+ extends DockerContainerFactory(instance, parameters) {
+ private val pulledImages = new TrieMap[String, Boolean]()
+ private val factoryConfig = loadConfigOrThrow[StandaloneDockerConfig](ConfigKeys.standaloneDockerContainerFactory)
+
+ override def createContainer(tid: TransactionId,
+ name: String,
+ actionImage: ExecManifest.ImageName,
+ userProvidedImage: Boolean,
+ memory: ByteSize,
+ cpuShares: Int)(implicit config: WhiskConfig, logging: Logging): Future[Container] = {
+
+ //For standalone server usage we would also want to pull the OpenWhisk provided image so as to ensure if
+ //local setup does not have the image then it pulls it down
+ //For standard usage its expected that standard images have already been pulled in.
+ val imageName = actionImage.localImageName(runtimesRegistryConfig.url)
+ val pulled =
+ if (!userProvidedImage
+ && factoryConfig.pullStandardImages
+ && !pulledImages.contains(imageName)
+ && actionImage.prefix.contains("openwhisk")) {
+ docker.pull(imageName)(tid).map { _ =>
+ logging.info(this, s"Pulled OpenWhisk provided image $imageName")
+ pulledImages.put(imageName, true)
+ true
+ }
+ } else Future.successful(true)
+
+ pulled.flatMap(_ => super.createContainer(tid, name, actionImage, userProvidedImage, memory, cpuShares))
+ }
+
+ override def init(): Unit = {
+ logging.info(
+ this,
+ s"Standalone docker container factory config pullStandardImages: ${factoryConfig.pullStandardImages}")
+ super.init()
+ }
+}
+
+trait WindowsDockerClient {
+ self: DockerClient =>
+
+ override protected def executableAlternatives: List[String] = {
+ val executable = loadConfig[String]("whisk.docker.executable").map(Some(_)).getOrElse(None)
+ List(Some("C:\\Program Files\\Docker\\Docker\\resources\\bin\\docker.exe"), executable).flatten
+ }
+}
+
+class DockerForWindowsClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)(
+ implicit log: Logging,
+ as: ActorSystem)
+ extends DockerForMacClient(dockerHost)(executionContext)
+ with WindowsDockerClient {
+ //Due to some Docker + Windows + Go parsing quirks need to add double quotes around whole command
+ //See https://github.com/moby/moby/issues/27592#issuecomment-255227097
+ override def inspectCommand: String = "\"{{(index (index .NetworkSettings.Ports \\\"8080/tcp\\\") 0).HostPort}}\""
+}
diff --git a/core/standalone/README.md b/core/standalone/README.md
new file mode 100644
index 0000000..831f276
--- /dev/null
+++ b/core/standalone/README.md
@@ -0,0 +1,193 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+# OpenWhisk Standalone Server
+
+OpenWhisk standalone server is meant to run a simple OpenWhisk server for local development and test purposes. It can be
+executed as a normal java application from command line.
+
+```bash
+java -jar openwhisk-standalone.jar
+```
+
+This should start the OpenWhisk server on port 3233 by default. Once the server is started then [configure the cli][1]
+and then try out the [samples][2].
+
+This server by default uses a memory based store and does not depend on any other external service like Kafka and CouchDB.
+It only needs Docker and Java to for running.
+
+Few key points related to it
+
+* Uses in memory store. Once the server is stopped all changes would be lost
+* Bootstraps the `guest` and `whisk.system` with default keys
+* Supports running on MacOS, Linux and Windows (experimental) setup
+* Can be customized to use any other storage like CouchDB
+
+
+### Build
+
+To build this standalone server run
+
+```bash
+$ ./gradlew :core:standalone:build
+```
+
+This would create the runnable jar in `bin/` directory. If you directly want to run the
+server then you can use following command
+
+```bash
+$ ./gradlew :core:standalone:bootRun
+```
+
+To pass argument to the run command use
+
+```bash
+$ ./gradlew :core:standalone:bootRun --args='-m runtimes.json'
+```
+
+### Usage
+
+OpenWhisk standalone server support various launch options
+
+```
+$ java -jar openwhisk-standalone.jar -h
+
+
+ /\ \ / _ \ _ __ ___ _ __ | | | | |__ (_)___| | __
+ /\ /__\ \ | | | | '_ \ / _ \ '_ \| | | | '_ \| / __| |/ /
+ / \____ \ / | |_| | |_) | __/ | | | |/\| | | | | \__ \ <
+ \ \ / \/ \___/| .__/ \___|_| |_|__/\__|_| |_|_|___/_|\_\
+ \___\/ tm |_|
+
+ -c, --config-file <arg> application.conf which overrides the default
+ standalone.conf
+ --disable-color-logging Disables colored logging
+ -m, --manifest <arg> Manifest json defining the supported runtimes
+ -p, --port <arg> Server port
+ -v, --verbose
+ -h, --help Show help message
+ --version Show version of this program
+
+OpenWhisk standalone server
+```
+
+Sections below would illustrate some of the supported options
+
+To change the default config you can provide a custom `application.conf` file via `-c` option. The application conf file
+must always include the default `standalone.conf`
+
+```hocon
+include classpath("standalone.conf")
+
+whisk {
+ //Custom config
+}
+```
+
+Then pass this config file
+
+```bash
+java -jar openwhisk-standalone.jar -c custom.conf
+```
+
+#### Adding custom namespaces
+
+If you need to register custom namespaces (aka users) then you can pass them via config file like below
+
+```hocon
+include classpath("standalone.conf")
+
+whisk {
+ users {
+ whisk-test = "cafebabe-cafe-babe-cafe-babecafebabe:007zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP"
+ }
+}
+```
+
+Then pass this config file via `-c` option. You can check the users created from log
+
+```
+[2019-06-21T19:52:02.923Z] [INFO] [#tid_userBootstrap] [StandaloneOpenWhisk] Created user [guest]
+[2019-06-21T19:52:03.008Z] [INFO] [#tid_userBootstrap] [StandaloneOpenWhisk] Created user [whisk.system]
+[2019-06-21T19:52:03.094Z] [INFO] [#tid_userBootstrap] [StandaloneOpenWhisk] Created user [whisk.test]
+```
+
+#### Using custom runtimes
+
+To use custom runtime pass the runtime manifest via `-m` option
+
+```json
+{
+ "runtimes": {
+ "ruby": [
+ {
+ "kind": "ruby:2.5",
+ "default": true,
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ },
+ "image": {
+ "prefix": "openwhisk",
+ "name": "action-ruby-v2.5",
+ "tag": "latest"
+ }
+ }
+ ]
+ }
+}
+```
+
+The pass this file at launch time
+
+```bash
+java -jar openwhisk-standalone.jar -m custom-runtime.json
+```
+
+You can then see the runtime config reflect in `http://localhost:3233`
+
+#### Using CouchDB
+
+If you need to connect to CouchDB or any other supported artifact store then you can pass the required config
+
+```hocon
+include classpath("standalone.conf")
+
+whisk {
+ couchdb {
+ protocol = "http"
+ host = "172.17.0.1"
+ port = "5984"
+ username = "whisk_admin"
+ password = "some_passw0rd"
+ provider = "CouchDB"
+ databases {
+ WhiskAuth = "whisk_local_subjects"
+ WhiskEntity = "whisk_local_whisks"
+ WhiskActivation = "whisk_local_activations"
+ }
+ }
+}
+```
+
+Then pass this config file via `-c` option.
+
+[1]: https://github.com/apache/incubator-openwhisk/blob/master/docs/cli.md
+[2]: https://github.com/apache/incubator-openwhisk/blob/master/docs/samples.md
diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle
new file mode 100644
index 0000000..06b5526
--- /dev/null
+++ b/core/standalone/build.gradle
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+ id 'org.springframework.boot' version '2.1.6.RELEASE'
+ id "com.gorylenko.gradle-git-properties" version "2.0.0"
+ id 'scala'
+}
+
+apply plugin: 'org.scoverage'
+apply plugin: 'maven'
+
+project.archivesBaseName = "openwhisk-standalone"
+
+repositories {
+ mavenCentral()
+}
+
+processResources {
+ from(new File(project.rootProject.projectDir, "ansible/files/runtimes.json")) {
+ into(".")
+ }
+}
+
+task copyBootJarToBin(type:Copy){
+ from ("${buildDir}/libs")
+ into file("${project.rootProject.projectDir}/bin")
+ rename("${project.archivesBaseName}-${version}.jar", "${project.archivesBaseName}.jar")
+}
+
+bootJar {
+ mainClassName = 'org.apache.openwhisk.standalone.StandaloneOpenWhisk'
+ finalizedBy copyBootJarToBin
+}
+
+dependencies {
+ compile project(':core:controller')
+ compile project(':tools:admin')
+ compile 'org.rogach:scallop_2.12:3.3.1'
+ scoverage gradle.scoverage.deps
+}
+
diff --git a/core/standalone/src/main/resources/logback-standalone.xml b/core/standalone/src/main/resources/logback-standalone.xml
new file mode 100644
index 0000000..4121b7e
--- /dev/null
+++ b/core/standalone/src/main/resources/logback-standalone.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <jmxConfigurator></jmxConfigurator>
+ <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}] %highlight([%p]) %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- Apache HttpClient -->
+ <logger name="org.apache.http" level="ERROR" />
+
+ <!-- Kafka -->
+ <logger name="org.apache.kafka" level="ERROR" />
+
+ <root level="${logback.log.level:-INFO}">
+ <appender-ref ref="console" />
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/core/standalone/src/main/resources/standalone.conf b/core/standalone/src/main/resources/standalone.conf
new file mode 100644
index 0000000..e3a89ad
--- /dev/null
+++ b/core/standalone/src/main/resources/standalone.conf
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include classpath("application.conf")
+
+kamon {
+ reporters = []
+}
+
+whisk {
+ metrics {
+ kamon-enabled = true
+ kamon-tags-enabled = true
+ prometheus-enabled = true
+ }
+
+ spi {
+ ArtifactStoreProvider = "org.apache.openwhisk.core.database.memory.MemoryArtifactStoreProvider"
+ MessagingProvider = "org.apache.openwhisk.connector.lean.LeanMessagingProvider"
+ LoadBalancerProvider = "org.apache.openwhisk.core.loadBalancer.LeanBalancer"
+ }
+
+ info {
+ build-no = "standalone"
+ date = "???"
+ }
+
+ config {
+ controller-instances = 1
+ limits-actions-sequence-maxLength = 50
+ limits-triggers-fires-perMinute = 60
+ limits-actions-invokes-perMinute = 60
+ limits-actions-invokes-concurrent = 30
+ }
+
+ controller {
+ protocol: http
+
+ # Bound only to localhost by default for better security
+ interface: localhost
+ }
+
+ # Default set of users which are bootstrapped upon start
+ users {
+ whisk-system = "789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP"
+ guest = "23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP"
+ }
+
+ docker {
+ # Path to docker executuable. Generally its /var/lib/docker
+ # executable =
+ standalone.container-factory {
+ #If enabled then pull would also be attempted for standard OpenWhisk images under`openwhisk` prefix
+ pull-standard-images: true
+ }
+ }
+}
diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala
new file mode 100644
index 0000000..e84a4a3
--- /dev/null
+++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/LogbackConfigurator.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.standalone
+
+import java.io.ByteArrayInputStream
+import java.nio.charset.StandardCharsets.UTF_8
+
+import akka.event.LoggingAdapter
+import ch.qos.logback.classic.joran.JoranConfigurator
+import ch.qos.logback.classic.{Level, LoggerContext}
+import ch.qos.logback.core.joran.spi.JoranException
+import ch.qos.logback.core.util.StatusPrinter
+import org.apache.commons.io.IOUtils
+import org.apache.openwhisk.common.{AkkaLogging, TransactionId}
+import org.slf4j.LoggerFactory
+
+import scala.io.AnsiColor
+import scala.util.Try
+
+/**
+ * Resets the Logback config if logging is configure via non standard file
+ */
+object LogbackConfigurator {
+
+ def initLogging(conf: Conf): Unit = {
+ val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]
+ ctx.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).setLevel(toLevel(conf.verbose()))
+ }
+
+ private def toLevel(v: Int) = {
+ v match {
+ case 0 => Level.INFO
+ case 1 => Level.DEBUG
+ case _ => Level.ALL
+ }
+ }
+
+ def configureLogbackFromResource(resourceName: String): Unit = {
+ Try(configureLogback(IOUtils.resourceToString("/" + resourceName, UTF_8))).failed.foreach(t =>
+ println(s"Could not load resource $resourceName- ${t.getMessage}"))
+ }
+
+ private def configureLogback(fileContent: String): Unit = {
+ val context = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]
+
+ try {
+ val configurator = new JoranConfigurator
+ configurator.setContext(context)
+ // Call context.reset() to clear any previous configuration, e.g. default
+ // configuration. For multi-step configuration, omit calling context.reset().
+ context.reset()
+ val is = new ByteArrayInputStream(fileContent.getBytes(UTF_8))
+ configurator.doConfigure(is)
+ } catch {
+ case _: JoranException =>
+ // StatusPrinter will handle this
+ }
+ StatusPrinter.printInCaseOfErrorsOrWarnings(context)
+ }
+}
+
+/**
+ * Similar to AkkaLogging but with color support
+ */
+class ColoredAkkaLogging(loggingAdapter: LoggingAdapter) extends AkkaLogging(loggingAdapter) with AnsiColor {
+ import ColorOutput.clr
+
+ override protected def format(id: TransactionId, name: String, logmsg: String) =
+ s"[${clr(id.toString, BOLD, true)}] [${clr(name.toString, CYAN, true)}] $logmsg"
+}
+
+object ColorOutput extends AnsiColor {
+ def clr(s: String, code: String, clrEnabled: Boolean) = if (clrEnabled) s"$code$s$RESET" else s
+}
diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala
new file mode 100644
index 0000000..f3a82d3
--- /dev/null
+++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/PreFlightChecks.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.standalone
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.commons.lang3.StringUtils
+import org.apache.openwhisk.standalone.StandaloneOpenWhisk.usersConfigKey
+import pureconfig.loadConfigOrThrow
+
+import scala.io.AnsiColor
+import scala.sys.process._
+import scala.util.Try
+
+case class PreFlightChecks(conf: Conf) extends AnsiColor {
+ import ColorOutput.clr
+ private val noopLogger = ProcessLogger(_ => ())
+ private val clrEnabled = conf.colorEnabled
+ private val separator = "=" * 80
+ private val pass = st("OK")
+ private val failed = st("FAILURE")
+ private val warn = st("WARN")
+ private val cliDownloadUrl = "https://s.apache.org/openwhisk-cli-download"
+ private val dockerUrl = "https://docs.docker.com/install/"
+
+ def run(): Unit = {
+ println(separator)
+ println("Running pre flight checks ...")
+ println()
+ checkForDocker()
+ checkForWsk()
+ println()
+ println(separator)
+ }
+
+ def checkForDocker() = {
+ val dockerExistsResult = Try("docker --version".!(noopLogger)).getOrElse(-1)
+ if (dockerExistsResult != 0) {
+ println(s"$failed 'docker' cli not found.")
+ println(s"\t Install docker from $dockerUrl")
+ } else {
+ println(s"$pass 'docker' cli found. $dockerVersion")
+ checkDockerIsRunning()
+ //Other things we can possibly check for
+ //1. add check for minimal supported docker version
+ //2. should we also run `docker run hello-world` to see if we can execute docker run command
+ //This command takes 2-4 secs. So running it by default for every run should be avoided
+ }
+ }
+
+ private def dockerVersion = version("docker --version '{{.Client.Version}}'")
+
+ private def version(cmd: String) = Try(cmd !! (noopLogger)).map(v => s"(${v.trim})").getOrElse("")
+
+ private def checkDockerIsRunning(): Unit = {
+ val dockerInfoResult = Try("docker info".!(noopLogger)).getOrElse(-1)
+ if (dockerInfoResult != 0) {
+ println(s"$failed 'docker' not found to be running. Failed to run 'docker info'")
+ } else {
+ println(s"$pass 'docker' is running.")
+ }
+ }
+
+ def checkForWsk(): Unit = {
+ val wskExistsResult = Try("wsk property get --cliversion".!(noopLogger)).getOrElse(-1)
+ if (wskExistsResult != 0) {
+ println(s"$failed 'wsk' cli not found.")
+ println(s"\tDownload the cli from $cliDownloadUrl")
+ } else {
+ println(s"$pass 'wsk' cli found. $wskCliVersion")
+ checkWskProps()
+ }
+ }
+
+ def checkWskProps(): Unit = {
+ val users = loadConfigOrThrow[Map[String, String]](loadConfig(), usersConfigKey)
+
+ val configuredAuth = "wsk property get --auth".!!.trim
+ val apihost = "wsk property get --apihost".!!.trim
+
+ val requiredHostValue = s"http://localhost:${conf.port()}"
+
+ //We can use -o option to get raw value. However as its a recent addition
+ //using a lazy approach where we check if output ends with one of the configured auth keys or
+ val matchedAuth = users.find { case (_, auth) => configuredAuth.endsWith(auth) }
+ val hostMatched = apihost.endsWith(requiredHostValue)
+
+ if (matchedAuth.isDefined && hostMatched) {
+ println(s"$pass 'wsk' configured for namespace [${matchedAuth.get._1}].")
+ println(s"$pass 'wsk' configured to connect to $requiredHostValue.")
+ } else {
+ val guestUser = users.find { case (ns, _) => ns == "guest" }
+ //Only if guest user is found suggest wsk command to use that. Otherwise user is using a non default setup
+ //which may not be used for wsk based access like for tests
+ guestUser match {
+ case Some((ns, guestAuth)) =>
+ println(s"$warn Configure wsk via below command to connect to this server as [$ns]")
+ println()
+ println(clr(s"wsk property set --apihost '$requiredHostValue' --auth '$guestAuth'", MAGENTA, clrEnabled))
+ case None =>
+ }
+ }
+ }
+
+ private def wskCliVersion = version("wsk property get --cliversion -o raw")
+
+ private def loadConfig(): Config = {
+ conf.configFile.toOption match {
+ case Some(f) =>
+ require(f.exists(), s"Config file $f does not exist")
+ ConfigFactory.parseFile(f)
+ case None =>
+ ConfigFactory.parseResources("standalone.conf")
+ }
+ }
+
+ private def st(level: String) = {
+ val maxLength = "FAILURE".length
+ val (msg, code) = level match {
+ case "OK" => (StringUtils.center("OK", maxLength), GREEN)
+ case "WARN" => (StringUtils.center("WARN", maxLength), MAGENTA)
+ case _ => ("FAILURE", RED)
+ }
+ s"[${clr(msg, code, clrEnabled)}]"
+ }
+}
diff --git a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala
new file mode 100644
index 0000000..ff0de51
--- /dev/null
+++ b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.standalone
+
+import java.io.{ByteArrayInputStream, File}
+import java.nio.charset.StandardCharsets.UTF_8
+import java.util.Properties
+
+import akka.actor.ActorSystem
+import akka.event.slf4j.SLF4JLogging
+import akka.stream.ActorMaterializer
+import org.apache.commons.io.{FileUtils, IOUtils}
+import org.apache.commons.lang3.SystemUtils
+import org.apache.openwhisk.common.{AkkaLogging, Config, Logging, TransactionId}
+import org.apache.openwhisk.core.cli.WhiskAdmin
+import org.apache.openwhisk.core.controller.Controller
+import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig}
+import org.apache.openwhisk.standalone.ColorOutput.clr
+import org.rogach.scallop.ScallopConf
+import pureconfig.loadConfigOrThrow
+
+import scala.collection.JavaConverters._
+import scala.concurrent.Await
+import scala.concurrent.duration._
+import scala.io.AnsiColor
+import scala.util.Try
+
+class Conf(arguments: Seq[String]) extends ScallopConf(arguments) {
+ banner(StandaloneOpenWhisk.banner)
+ footer("\nOpenWhisk standalone server")
+ StandaloneOpenWhisk.gitInfo.foreach(g => version(s"Git Commit - ${g.commitId}"))
+
+ this.printedName = "openwhisk"
+ val configFile =
+ opt[File](descr = "application.conf which overrides the default standalone.conf", validate = _.canRead)
+ val manifest = opt[File](descr = "Manifest json defining the supported runtimes", validate = _.canRead)
+ val port = opt[Int](descr = "Server port", default = Some(3233))
+
+ val verbose = tally()
+ val disableColorLogging = opt[Boolean](descr = "Disables colored logging", noshort = true)
+
+ verify()
+
+ val colorEnabled = !disableColorLogging()
+}
+
+case class GitInfo(commitId: String, commitTime: String)
+
+object StandaloneOpenWhisk extends SLF4JLogging {
+ val usersConfigKey = "whisk.users"
+
+ val banner =
+ """
+ | ____ ___ _ _ _ _ _
+ | /\ \ / _ \ _ __ ___ _ __ | | | | |__ (_)___| | __
+ | /\ /__\ \ | | | | '_ \ / _ \ '_ \| | | | '_ \| / __| |/ /
+ | / \____ \ / | |_| | |_) | __/ | | | |/\| | | | | \__ \ <
+ | \ \ / \/ \___/| .__/ \___|_| |_|__/\__|_| |_|_|___/_|\_\
+ | \___\/ tm |_|
+ """.stripMargin
+
+ val defaultRuntime = """{
+ | "runtimes": {
+ | "nodejs": [
+ | {
+ | "kind": "nodejs:10",
+ | "default": true,
+ | "image": {
+ | "prefix": "openwhisk",
+ | "name": "action-nodejs-v10",
+ | "tag": "latest"
+ | },
+ | "deprecated": false,
+ | "attached": {
+ | "attachmentName": "codefile",
+ | "attachmentType": "text/plain"
+ | },
+ | "stemCells": [
+ | {
+ | "count": 1,
+ | "memory": "256 MB"
+ | }
+ | ]
+ | }
+ | ]
+ | }
+ |}
+ |""".stripMargin
+
+ val gitInfo: Option[GitInfo] = loadGitInfo()
+
+ def main(args: Array[String]): Unit = {
+ val conf = new Conf(args)
+
+ printBanner(conf)
+ PreFlightChecks(conf).run()
+
+ configureLogging(conf)
+ initialize(conf)
+ //Create actor system only after initializing the config
+ implicit val actorSystem = ActorSystem("standalone-actor-system")
+ implicit val materializer = ActorMaterializer.create(actorSystem)
+ implicit val logger: Logging = createLogging(actorSystem, conf)
+
+ startServer()
+ }
+
+ def initialize(conf: Conf): Unit = {
+ configureBuildInfo()
+ configureServerPort(conf)
+ configureOSSpecificOpts()
+ initConfigLocation(conf)
+ configureRuntimeManifest(conf)
+ loadWhiskConfig()
+ }
+
+ def startServer()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer, logging: Logging): Unit = {
+ bootstrapUsers()
+ startController()
+ }
+
+ private def configureServerPort(conf: Conf) = {
+ val port = conf.port()
+ log.info(s"Starting OpenWhisk standalone on port $port")
+ System.setProperty(WhiskConfig.disableWhiskPropsFileRead, "true")
+ setConfigProp(WhiskConfig.servicePort, port.toString)
+ setConfigProp(WhiskConfig.wskApiPort, port.toString)
+ setConfigProp(WhiskConfig.wskApiProtocol, "http")
+ setConfigProp(WhiskConfig.wskApiHostname, localHostName)
+ }
+
+ private def initConfigLocation(conf: Conf): Unit = {
+ conf.configFile.toOption match {
+ case Some(f) =>
+ require(f.exists(), s"Config file $f does not exist")
+ System.setProperty("config.file", f.getAbsolutePath)
+ case None =>
+ System.setProperty("config.resource", "standalone.conf")
+ }
+ }
+
+ private def configKey(k: String): String = Config.prefix + k.replace('-', '.')
+
+ private def loadWhiskConfig(): Unit = {
+ val config = loadConfigOrThrow[Map[String, String]](ConfigKeys.whiskConfig)
+ config.foreach { case (k, v) => setConfigProp(k, v) }
+ }
+
+ private def configureRuntimeManifest(conf: Conf): Unit = {
+ val manifest = conf.manifest.toOption match {
+ case Some(file) =>
+ FileUtils.readFileToString(file, UTF_8)
+ case None => {
+ //Fallback to a default runtime in case resource not found. Say while running from IDE
+ Try(IOUtils.resourceToString("/runtimes.json", UTF_8)).getOrElse(defaultRuntime)
+ }
+ }
+ setConfigProp(WhiskConfig.runtimesManifest, manifest)
+ }
+
+ private def setConfigProp(key: String, value: String): Unit = {
+ setSysProp(configKey(key), value)
+ }
+
+ private def startController()(implicit actorSystem: ActorSystem, logger: Logging): Unit = {
+ Controller.start(Array("standalone"))
+ }
+
+ private def bootstrapUsers()(implicit actorSystem: ActorSystem,
+ materializer: ActorMaterializer,
+ logging: Logging): Unit = {
+ val users = loadConfigOrThrow[Map[String, String]](usersConfigKey)
+ implicit val userTid: TransactionId = TransactionId("userBootstrap")
+ users.foreach {
+ case (name, key) =>
+ val subject = name.replace('-', '.')
+ val conf = new org.apache.openwhisk.core.cli.Conf(Seq("user", "create", "--auth", key, subject))
+ val admin = WhiskAdmin(conf)
+ Await.ready(admin.executeCommand(), 60.seconds)
+ logging.info(this, s"Created user [$subject]")
+ }
+ }
+
+ private def configureOSSpecificOpts(): Unit = {
+ setSysProp(
+ "whisk.spi.ContainerFactoryProvider",
+ "org.apache.openwhisk.core.containerpool.docker.StandaloneDockerContainerFactoryProvider")
+
+ //Disable runc by default to keep things stable
+ setSysProp("whisk.docker.container-factory.use-runc", "False")
+
+ //Use cli based log store for all setups as its more stable to use
+ // and does not require root user access
+ setSysProp("whisk.spi.LogStoreProvider", "org.apache.openwhisk.core.containerpool.docker.DockerCliLogStoreProvider")
+ }
+
+ private def localHostName = {
+ //For connecting back to controller on container host following name needs to be used
+ // on Windows and Mac
+ // https://docs.docker.com/docker-for-windows/networking/#use-cases-and-workarounds
+ if (SystemUtils.IS_OS_MAC || SystemUtils.IS_OS_WINDOWS)
+ "host.docker.internal"
+ else "localhost"
+ }
+
+ private def loadGitInfo() = {
+ val info = loadPropResource("git.properties")
+ for {
+ commit <- info.get("git.commit.id.abbrev")
+ time <- info.get("git.commit.time")
+ } yield GitInfo(commit, time)
+ }
+
+ private def printBanner(conf: Conf) = {
+ val bannerTxt = clr(banner, AnsiColor.CYAN, conf.colorEnabled)
+ println(bannerTxt)
+ gitInfo.foreach(g => println(s"Git Commit: ${g.commitId}, Build Date: ${g.commitTime}"))
+ }
+
+ private def configureBuildInfo(): Unit = {
+ gitInfo.foreach { g =>
+ setSysProp("whisk.info.build-no", g.commitId)
+ setSysProp("whisk.info.date", g.commitTime)
+ }
+ }
+
+ private def setSysProp(key: String, value: String): Unit = {
+ Option(System.getProperty(key)) match {
+ case Some(x) if x != value =>
+ log.info(s"Founding existing value for system property '$key'- Going to set '$value' , found '$x'")
+ case _ =>
+ System.setProperty(key, value)
+ }
+ }
+
+ private def loadPropResource(name: String): Map[String, String] = {
+ Try {
+ val propString = IOUtils.resourceToString("/" + name, UTF_8)
+ val props = new Properties()
+ props.load(new ByteArrayInputStream(propString.getBytes(UTF_8)))
+ props.asScala.toMap
+ }.getOrElse(Map.empty)
+ }
+
+ private def configureLogging(conf: Conf): Unit = {
+ if (System.getProperty("logback.configurationFile") == null && !conf.disableColorLogging()) {
+ LogbackConfigurator.configureLogbackFromResource("logback-standalone.xml")
+ }
+ LogbackConfigurator.initLogging(conf)
+ }
+
+ private def createLogging(actorSystem: ActorSystem, conf: Conf): Logging = {
+ val adapter = akka.event.Logging.getLogger(actorSystem, this)
+ if (conf.disableColorLogging())
+ new AkkaLogging(adapter)
+ else
+ new ColoredAkkaLogging(adapter)
+ }
+}
diff --git a/settings.gradle b/settings.gradle
index d4ebf0b..40c59bb 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -19,6 +19,7 @@ include 'common:scala'
include 'core:controller'
include 'core:invoker'
+include 'core:standalone'
include 'tests'
include 'tests:performance:gatling_tests'
diff --git a/tests/build.gradle b/tests/build.gradle
index efeb83d..48fc925 100644
--- a/tests/build.gradle
+++ b/tests/build.gradle
@@ -36,17 +36,6 @@ install.dependsOn ':tools:admin:install'
project.archivesBaseName = "openwhisk-tests"
-tasks.withType(Test) {
- systemProperties(System.getProperties())
-
- testLogging {
- events "passed", "skipped", "failed"
- showStandardStreams = true
- exceptionFormat = 'full'
- }
- outputs.upToDateWhen { false } // force tests to run every time
-}
-
def leanExcludes = [
'**/MaxActionDurationTests*',
'invokerShoot/**'
@@ -250,6 +239,17 @@ gradle.projectsEvaluated {
task testCoverage(type: Test) {
classpath = getScoverageClasspath(project)
}
+ tasks.withType(Test) {
+ systemProperties(System.getProperties())
+ systemProperty("whisk.server.jar", getStandaloneJarFilePath())
+
+ testLogging {
+ events "passed", "skipped", "failed"
+ showStandardStreams = true
+ exceptionFormat = 'full'
+ }
+ outputs.upToDateWhen { false } // force tests to run every time
+ }
}
task copyMeasurementFiles() {
@@ -366,3 +366,7 @@ task testSwaggerCodegen(type: GradleBuild) {
buildFile = "${buildDir}/swagger-code-java/build.gradle"
tasks = ['build']
}
+
+def getStandaloneJarFilePath(){
+ project(":core:standalone").jar.archivePath
+}
diff --git a/tests/src/test/scala/common/FreePortFinder.scala b/tests/src/test/scala/common/FreePortFinder.scala
new file mode 100644
index 0000000..0ebd54a
--- /dev/null
+++ b/tests/src/test/scala/common/FreePortFinder.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package common
+
+import java.net.ServerSocket
+
+/**
+ * Utility to find a free port such that any launched service can be configured to use that.
+ * This helps in ensuring that test do not fail due to conflict with any existing service using a standard port
+ */
+object FreePortFinder {
+
+ def freePort(): Int = {
+ val socket = new ServerSocket(0)
+ try socket.getLocalPort
+ finally if (socket != null) socket.close()
+ }
+}
diff --git a/tests/src/test/scala/common/WhiskProperties.java b/tests/src/test/scala/common/WhiskProperties.java
index b53be03..1d442eb 100644
--- a/tests/src/test/scala/common/WhiskProperties.java
+++ b/tests/src/test/scala/common/WhiskProperties.java
@@ -34,7 +34,7 @@ public class WhiskProperties {
/**
* System property key which refers to OpenWhisk Edge Host url
*/
- private static final String WHISK_SERVER = "whisk.server";
+ public static final String WHISK_SERVER = "whisk.server";
/**
* System property key which refers to authentication key to be used for testing
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala
index 04577e3..ef0121e 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/mesos/test/MesosContainerFactoryTest.scala
@@ -98,7 +98,7 @@ class MesosContainerFactoryTest
}
override def afterAll(): Unit = {
- TestKit.shutdownActorSystem(system)
+ TestKit.shutdownActorSystem(system, verifySystemShutdown = true)
super.afterAll()
}
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala
index 2d8a1be..8ca2d7c 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala
@@ -39,6 +39,11 @@ trait MemoryArtifactStoreBehaviorBase extends FlatSpec with ArtifactStoreBehavio
MemoryArtifactStoreProvider.makeArtifactStore[WhiskAuth](getAttachmentStore[WhiskAuth]())
}
+ override protected def beforeAll(): Unit = {
+ MemoryArtifactStoreProvider.purgeAll()
+ super.beforeAll()
+ }
+
override lazy val entityStore =
MemoryArtifactStoreProvider.makeArtifactStore[WhiskEntity](getAttachmentStore[WhiskEntity]())(
classTag[WhiskEntity],
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala
index 32c157d..c10395d 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala
@@ -32,6 +32,11 @@ class MemoryAttachmentStoreTests extends FlatSpec with AttachmentStoreBehaviors
override def storeType: String = "Memory"
+ override protected def beforeAll(): Unit = {
+ MemoryArtifactStoreProvider.purgeAll()
+ super.beforeAll()
+ }
+
override def afterAll(): Unit = {
super.afterAll()
val count = store.asInstanceOf[MemoryAttachmentStore].attachmentCount
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala
index ee9f1fb..9226a4f 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala
@@ -22,7 +22,7 @@ import akka.stream.ActorMaterializer
import org.scalatest.FlatSpec
import org.apache.openwhisk.common.Logging
import org.apache.openwhisk.core.database.{AttachmentStore, DocumentSerializer}
-import org.apache.openwhisk.core.database.memory.MemoryArtifactStoreBehaviorBase
+import org.apache.openwhisk.core.database.memory.{MemoryArtifactStoreBehaviorBase, MemoryArtifactStoreProvider}
import org.apache.openwhisk.core.database.test.AttachmentStoreBehaviors
import org.apache.openwhisk.core.database.test.behavior.ArtifactStoreAttachmentBehaviors
import org.apache.openwhisk.core.entity.WhiskEntity
@@ -41,6 +41,11 @@ trait S3AttachmentStoreBehaviorBase
override val prefix = s"attachmentTCK_${Random.alphanumeric.take(4).mkString}"
+ override protected def beforeAll(): Unit = {
+ MemoryArtifactStoreProvider.purgeAll()
+ super.beforeAll()
+ }
+
override def getAttachmentStore[D <: DocumentSerializer: ClassTag](): AttachmentStore =
makeS3Store[D]()
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala
index 83f5d6a..bbd232b 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/core/database/test/behavior/ArtifactStoreCRUDBehaviors.scala
@@ -48,6 +48,20 @@ trait ArtifactStoreCRUDBehaviors extends ArtifactStoreBehaviorBase {
doc2.rev.empty shouldBe false
}
+ it should "put delete and then recreate document with same id with different rev" in {
+ implicit val tid: TransactionId = transid()
+ val auth = newAuth()
+ val doc = put(authStore, auth)
+
+ delete(authStore, doc) shouldBe true
+
+ val auth2 = auth.copy(namespaces = Set(wskNS("foo1")))
+ val doc2 = put(authStore, auth2)
+
+ doc2.rev should not be doc.rev
+ doc2.rev.empty shouldBe false
+ }
+
it should "throw DocumentConflictException when updated with old revision" in {
implicit val tid: TransactionId = transid()
val auth = newAuth()
diff --git a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala
new file mode 100644
index 0000000..df07b27
--- /dev/null
+++ b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.standalone
+
+import java.io.File
+import java.net.URI
+import java.nio.charset.StandardCharsets.UTF_8
+
+import com.google.common.base.Stopwatch
+import common.WhiskProperties.WHISK_SERVER
+import common.{FreePortFinder, StreamLogging, WhiskProperties}
+import io.restassured.RestAssured
+import org.apache.commons.io.FileUtils
+import org.apache.commons.lang3.SystemUtils
+import org.apache.openwhisk.core.WhiskConfig
+import org.apache.openwhisk.utils.retry
+import org.scalatest.{BeforeAndAfterAll, Pending, Suite, TestSuite}
+
+import scala.concurrent.duration._
+import scala.sys.process._
+import scala.util.control.NonFatal
+
+trait StandaloneServerFixture extends TestSuite with BeforeAndAfterAll with StreamLogging {
+ self: Suite =>
+
+ private val jarPathProp = "whisk.server.jar"
+ private var manifestFile: Option[File] = None
+ private var serverProcess: Process = _
+ protected val serverPort: Int = FreePortFinder.freePort()
+ protected var serverUrl: String = System.getProperty(WHISK_SERVER, s"http://localhost:$serverPort/")
+ private val disablePullConfig = "whisk.docker.standalone.container-factory.pull-standard-images"
+ private var serverStartedForTest = false
+
+ //Following tests always fail on Mac but pass when standalone server is running on Linux
+ //It looks related to how networking works on Mac for Docker container
+ //For now ignoring there failure
+ private val ignoredTestsOnMac = Set(
+ "Wsk Action REST should create, and invoke an action that utilizes a docker container",
+ "Wsk Action REST should create, and invoke an action that utilizes dockerskeleton with native zip",
+ "Wsk Action REST should create and invoke a blocking action resulting in an application error response",
+ "Wsk Action REST should create an action, and invoke an action that returns an empty JSON object")
+
+ override def beforeAll(): Unit = {
+ val serverUrlViaSysProp = Option(System.getProperty(WHISK_SERVER))
+ serverUrlViaSysProp match {
+ case Some(u) =>
+ serverUrl = u
+ println(s"Connecting to existing server at $serverUrl")
+ case None =>
+ System.setProperty(WHISK_SERVER, serverUrl)
+ //TODO avoid starting the server if url whisk.server property is predefined
+ super.beforeAll()
+ println(s"Running standalone server from ${standaloneServerJar.getAbsolutePath}")
+ manifestFile = getRuntimeManifest()
+ val args = Seq(
+ Seq(
+ "java",
+ s"-D$disablePullConfig=false",
+ "-jar",
+ standaloneServerJar.getAbsolutePath,
+ "--disable-color-logging"),
+ Seq("-p", serverPort.toString),
+ manifestFile.map(f => Seq("-m", f.getAbsolutePath)).getOrElse(Seq.empty)).flatten
+
+ serverProcess = args.run(ProcessLogger(s => printstream.println(s)))
+ val w = waitForServerToStart()
+ serverStartedForTest = true
+ println(s"Started test server at $serverUrl in [$w]")
+ }
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ if (serverStartedForTest) {
+ System.clearProperty(WHISK_SERVER)
+ manifestFile.foreach(FileUtils.deleteQuietly)
+ serverProcess.destroy()
+ }
+ }
+
+ override def withFixture(test: NoArgTest) = {
+ val outcome = super.withFixture(test)
+ if (outcome.isFailed) {
+ println(logLines.mkString("\n"))
+ }
+ stream.reset()
+ val result = if (outcome.isFailed && SystemUtils.IS_OS_MAC && ignoredTestsOnMac.contains(test.name)) {
+ println(s"Ignoring known failed test for Mac [${test.name}]")
+ Pending
+ } else outcome
+ result
+ }
+
+ def waitForServerToStart(): Stopwatch = {
+ val w = Stopwatch.createStarted()
+ try {
+ retry({
+ println(s"Waiting for OpenWhisk server to start since $w")
+ val response = RestAssured.get(new URI(serverUrl))
+ require(response.statusCode() == 200)
+ }, 30, Some(1.second))
+ } catch {
+ case NonFatal(e) =>
+ println(logLines.mkString("\n"))
+ throw e
+ }
+ w
+ }
+
+ private def getRuntimeManifest(): Option[File] = {
+ Option(WhiskProperties.getProperty(WhiskConfig.runtimesManifest)).map { json =>
+ val f = newFile()
+ FileUtils.write(f, json, UTF_8)
+ f
+ }
+ }
+
+ private def newFile(): File = File.createTempFile("whisktest", null, null)
+
+ private def standaloneServerJar: File = {
+ Option(System.getProperty(jarPathProp)) match {
+ case Some(p) =>
+ val jarFile = new File(p)
+ assert(
+ jarFile.canRead,
+ s"OpenWhisk standalone server jar file [$p] specified via system property [$jarPathProp] not found")
+ jarFile
+ case None =>
+ fail(s"No jar file specified via system property [$jarPathProp]")
+ }
+ }
+}
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerTests.scala
similarity index 54%
copy from tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala
copy to tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerTests.scala
index 32c157d..725033c 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/database/memory/MemoryAttachmentStoreTests.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerTests.scala
@@ -15,26 +15,14 @@
* limitations under the License.
*/
-package org.apache.openwhisk.core.database.memory
+package org.apache.openwhisk.standalone
-import common.WskActorSystem
+import common.WskProps
import org.junit.runner.RunWith
-import org.scalatest.FlatSpec
import org.scalatest.junit.JUnitRunner
-import org.apache.openwhisk.core.database.AttachmentStore
-import org.apache.openwhisk.core.database.test.AttachmentStoreBehaviors
-import org.apache.openwhisk.core.entity.WhiskEntity
+import system.basic.WskRestBasicTests
@RunWith(classOf[JUnitRunner])
-class MemoryAttachmentStoreTests extends FlatSpec with AttachmentStoreBehaviors with WskActorSystem {
-
- override val store: AttachmentStore = MemoryAttachmentStoreProvider.makeStore[WhiskEntity]()
-
- override def storeType: String = "Memory"
-
- override def afterAll(): Unit = {
- super.afterAll()
- val count = store.asInstanceOf[MemoryAttachmentStore].attachmentCount
- require(count == 0, s"AttachmentStore not empty after all runs - $count")
- }
+class StandaloneServerTests extends WskRestBasicTests with StandaloneServerFixture {
+ override implicit val wskprops = WskProps().copy(apihost = serverUrl)
}
diff --git a/tests/src/test/scala/system/basic/WskRestBasicTests.scala b/tests/src/test/scala/system/basic/WskRestBasicTests.scala
index 2155e7a..1348089 100644
--- a/tests/src/test/scala/system/basic/WskRestBasicTests.scala
+++ b/tests/src/test/scala/system/basic/WskRestBasicTests.scala
@@ -39,7 +39,7 @@ import org.apache.openwhisk.http.Messages
@RunWith(classOf[JUnitRunner])
class WskRestBasicTests extends TestHelpers with WskTestHelpers with WskActorSystem {
- implicit val wskprops = WskProps()
+ implicit def wskprops: WskProps = WskProps()
val wsk = new WskRestOperations
val defaultAction: Some[String] = Some(TestUtils.getTestActionFilename("hello.js"))
diff --git a/tools/build/redo b/tools/build/redo
index cb81b26..0534409 100755
--- a/tools/build/redo
+++ b/tools/build/redo
@@ -311,7 +311,13 @@ Components = [
'run units tests',
yaml = False,
tasks = 'testUnit',
- gradle = 'tests')
+ gradle = 'tests'),
+
+ makeComponent('standalone',
+ 'run standalone server',
+ yaml = False,
+ tasks = 'bootRun',
+ gradle = 'core:standalone')
]
def getComponent(component):
diff --git a/tools/travis/distDocker.sh b/tools/travis/distDocker.sh
index 7f40d0b..a6022fd 100755
--- a/tools/travis/distDocker.sh
+++ b/tools/travis/distDocker.sh
@@ -29,5 +29,6 @@ TERM=dumb ./gradlew distDocker -PdockerImagePrefix=testing $GRADLE_PROJS_SKIP
TERM=dumb ./gradlew :core:controller:distDockerCoverage -PdockerImagePrefix=testing
TERM=dumb ./gradlew :core:invoker:distDockerCoverage -PdockerImagePrefix=testing
+TERM=dumb ./gradlew :core:standalone:build
echo "Time taken for ${0##*/} is $SECONDS secs"