You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by dr...@apache.org on 2018/12/06 21:54:40 UTC
[incubator-openwhisk] branch master updated: tools/dev - updated
intellij script to start controller and invoker locally (#4142)
This is an automated email from the ASF dual-hosted git repository.
dragos pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 885f228 tools/dev - updated intellij script to start controller and invoker locally (#4142)
885f228 is described below
commit 885f228bd0417f134f7e9fe6a6e0c43d070e19fe
Author: ddragosd <dd...@gmail.com>
AuthorDate: Thu Dec 6 13:54:33 2018 -0800
tools/dev - updated intellij script to start controller and invoker locally (#4142)
* tools/dev - updated intellij script for docker-compose
* Added support for docker for mac
* log-limit is 0 to avoid collection logs
* updates for wskdev with ansible
---
ansible/group_vars/all | 2 +-
.../core/containerpool/docker/DockerClient.scala | 2 +-
.../docker/DockerContainerFactory.scala | 5 +-
.../docker/DockerForMacContainerFactory.scala | 75 ++++++++++++++++++++++
tools/dev/README.md | 26 ++++++--
tools/dev/build.gradle | 1 +
tools/dev/src/main/groovy/intellijRunConfig.groovy | 22 +++++--
7 files changed, 120 insertions(+), 13 deletions(-)
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index fb10427..9aeb5a5 100755
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -172,7 +172,7 @@ invoker:
port: 12001
heap: "{{ invoker_heap | default('2g') }}"
arguments: "{{ invoker_arguments | default('') }}"
- userMemory: "{{ invoker_user_memory | default('2048 m') }}"
+ userMemory: "{{ invoker_user_memory | default('2048m') }}"
instances: "{{ groups['invokers'] | length }}"
# Specify if it is allowed to deploy more than 1 invoker on a single machine.
allowMultipleInstances: "{{ invoker_allow_multiple_instances | default(false) }}"
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
index 4f493f2..355c3ab 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
@@ -178,7 +178,7 @@ class DockerClient(dockerHost: Option[String] = None,
def isOomKilled(id: ContainerId)(implicit transid: TransactionId): Future[Boolean] =
runCmd(Seq("inspect", id.asString, "--format", "{{.State.OOMKilled}}"), config.timeouts.inspect).map(_.toBoolean)
- private def runCmd(args: Seq[String], timeout: Duration)(implicit transid: TransactionId): Future[String] = {
+ protected def runCmd(args: Seq[String], timeout: Duration)(implicit transid: TransactionId): Future[String] = {
val cmd = dockerCmd ++ args
val start = transid.started(
this,
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
index e5f11b1..c4ceff3 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
@@ -18,6 +18,7 @@
package org.apache.openwhisk.core.containerpool.docker
import akka.actor.ActorSystem
+
import scala.concurrent.Await
import scala.concurrent.ExecutionContext
import scala.concurrent.Future
@@ -27,14 +28,16 @@ import org.apache.openwhisk.core.WhiskConfig
import org.apache.openwhisk.core.containerpool.Container
import org.apache.openwhisk.core.containerpool.ContainerFactory
import org.apache.openwhisk.core.containerpool.ContainerFactoryProvider
+import org.apache.openwhisk.core.containerpool.ContainerArgsConfig
import org.apache.openwhisk.core.entity.ByteSize
import org.apache.openwhisk.core.entity.ExecManifest
import org.apache.openwhisk.core.entity.InvokerInstanceId
+
import scala.concurrent.duration._
import java.util.concurrent.TimeoutException
+
import pureconfig._
import org.apache.openwhisk.core.ConfigKeys
-import org.apache.openwhisk.core.containerpool.ContainerArgsConfig
case class DockerContainerFactoryConfig(useRunc: Boolean)
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala
new file mode 100644
index 0000000..fb7bc2e
--- /dev/null
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.core.containerpool.docker
+
+import akka.actor.ActorSystem
+import org.apache.openwhisk.common.{Logging, TransactionId}
+import org.apache.openwhisk.core.WhiskConfig
+import org.apache.openwhisk.core.containerpool._
+import org.apache.openwhisk.core.entity.InvokerInstanceId
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+
+/**
+ * This factory provides a Docker for Mac client which exposes action container's ports on the host.
+ */
+object DockerForMacContainerFactoryProvider extends ContainerFactoryProvider {
+ override def instance(actorSystem: ActorSystem,
+ logging: Logging,
+ config: WhiskConfig,
+ instanceId: InvokerInstanceId,
+ parameters: Map[String, Set[String]]): ContainerFactory = {
+
+ new DockerContainerFactory(instanceId, parameters)(
+ actorSystem,
+ actorSystem.dispatcher,
+ logging,
+ new DockerForMacClient()(actorSystem.dispatcher)(logging, actorSystem),
+ new RuncClient()(actorSystem.dispatcher)(logging, actorSystem))
+ }
+
+}
+
+class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)(implicit log: Logging,
+ as: ActorSystem)
+ extends DockerClientWithFileAccess(dockerHost)(executionContext)
+ with DockerApiWithFileAccess {
+
+ implicit private val ec: ExecutionContext = executionContext
+
+ override def run(image: String, args: Seq[String] = Seq.empty[String])(
+ implicit transid: TransactionId): Future[ContainerId] = {
+ // b/c docker for mac doesn't have a routing to the action containers
+ // the port 8080 is exposed on the host on a random port number
+ val extraArgs: Seq[String] = Seq("-p", "0:8080") ++ args
+ super.run(image, extraArgs)
+ }
+ // See extended trait for description
+ override def inspectIPAddress(id: ContainerId, network: String)(
+ implicit transid: TransactionId): Future[ContainerAddress] = {
+ super
+ .runCmd(
+ Seq("inspect", "--format", """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}""", id.asString),
+ 10.seconds)
+ .flatMap {
+ case "<no value>" => Future.failed(new NoSuchElementException)
+ case stdout => Future.successful(ContainerAddress("localhost", stdout.toInt))
+ }
+ }
+}
diff --git a/tools/dev/README.md b/tools/dev/README.md
index ed3aa67..ffd3849 100644
--- a/tools/dev/README.md
+++ b/tools/dev/README.md
@@ -76,14 +76,28 @@ First setup OpenWhisk so that Controller and Invoker containers are up and runni
It would inspect the running docker containers and then generate the launch configs with name 'controller0'
and 'invoker0'.
-Key points to note:
+Now the docker container(s) (controller and/or invoker) can be stopped and they can be launched instead from within the IDE.
-1. Uses ~/tmp/openwhisk/controller (or invoker) as working directory.
-2. Changes the PORT to linked one. So controller gets started at 10001 only just like as its done in container.
+Key points to note:
-Now the docker container can be stopped and application can be launched from within the IDE.
+1. Controller uses port `10001` and Invoker uses port `12001`.
+2. Action activation logs are [disabled][2].
+3. SSL is disabled for Controller and Invoker.
+4. Make sure you have the loopback interface configured:
+ ```bash
+ sudo ifconfig lo0 alias 172.17.0.1/24
+ ```
+5. `~/.wskprops` must be updated with `APIHOST=http://localhost:10001` so that the `wsk` CLI communicates directly with the controller.
+6. On a MAC
+ * With Docker For Mac the invoker is configured to use a Container Factory that exposes ports for actions on the host,
+ as otherwise the invoker can't make HTTP requests to the actions.
+ You can read more at [docker/for-mac#171][7].
+
+ * When using [docker-compose][8] locally you have to update `/etc/hosts` with the line bellow:
+ ```
+ 127.0.0.1 kafka zookeeper kafka.docker zookeeper.docker db.docker controller whisk.controller
+ ```
-**Note** - Currently only the controller can be run from IDE. Invoker posses some [problems][2].
### Configuration
@@ -168,3 +182,5 @@ $ ./gradlew :tools:dev:renderModuleDetails
[4]: http://docs.groovy-lang.org/2.4.2/html/gapi/groovy/util/ConfigSlurper.html
[5]: https://developer.github.com/v3/search/
[6]: https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
+[7]: https://github.com/docker/for-mac/issues/171
+[8]: https://github.com/apache/incubator-openwhisk-devtools/tree/master/docker-compose
diff --git a/tools/dev/build.gradle b/tools/dev/build.gradle
index 0ce1183..745b91f 100644
--- a/tools/dev/build.gradle
+++ b/tools/dev/build.gradle
@@ -26,6 +26,7 @@ def owHome = project.projectDir.parentFile.parentFile
dependencies {
compile "org.codehaus.groovy:groovy-all:2.4.14"
compile 'org.apache.commons:commons-io:1.3.2'
+ compile 'org.apache.commons:commons-lang3:3.8.1'
}
task couchdbViews(type: JavaExec) {
diff --git a/tools/dev/src/main/groovy/intellijRunConfig.groovy b/tools/dev/src/main/groovy/intellijRunConfig.groovy
index ad2858b..cc41d87 100644
--- a/tools/dev/src/main/groovy/intellijRunConfig.groovy
+++ b/tools/dev/src/main/groovy/intellijRunConfig.groovy
@@ -16,6 +16,7 @@
*/
import groovy.json.JsonSlurper
import groovy.text.SimpleTemplateEngine
+import org.apache.commons.lang3.SystemUtils
assert args : "Expecting the OpenWhisk home directory to passed"
owHome = args[0]
@@ -26,7 +27,7 @@ def configTemplate = '''<component name="ProjectRunConfigurationManager">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" value="$main" />
<option name="VM_PARAMETERS" value="$sysProps" />
- <option name="PROGRAM_PARAMETERS" value="0" />
+ <option name="PROGRAM_PARAMETERS" value="$programParams" />
<option name="WORKING_DIRECTORY" value="$workingDir" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
@@ -57,13 +58,15 @@ Map controllerEnv = null
Map invokerEnv = null
containerNames.each{cn ->
+
//Inspect the specific container
def inspectResult = "docker inspect $cn".execute().text
def json = new JsonSlurper().parseText(inspectResult)
def imageName = json[0].'Config'.'Image'
if (imageName.contains("controller") || imageName.contains("invoker")){
- def mappedPort = json.'NetworkSettings'.'Ports'.'8080/tcp'[0][0].'HostPort'
+ // pre-configure the local ports for controller and invoker
+ def mappedPort = imageName.contains("controller") ? '10001' : '12001'
def envBaseMap = getEnvMap(json[0].'Config'.'Env')
String type
@@ -84,6 +87,14 @@ containerNames.each{cn ->
//Prepare system properties
def sysProps = getSysProps(envMap,type)
+ // disable log collection. See more at: https://github.com/apache/incubator-openwhisk/issues/3195
+ sysProps += " -Dwhisk.log-limit.max=0 -Dwhisk.log-limit.std=0"
+ // disable https protocol for controller and invoker
+ sysProps = sysProps.replaceAll("protocol=https", "protocol=http")
+ if (SystemUtils.IS_OS_MAC){
+ sysProps = sysProps.replaceAll("use-runc=True", "use-runc=False")
+ sysProps += " -Dwhisk.spi.ContainerFactoryProvider=org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider"
+ }
def templateBinding = [
main: meta[type].main,
@@ -92,7 +103,8 @@ containerNames.each{cn ->
env: encodeForXML(envMap),
sysProps : sysProps,
USER_HOME : '$USER_HOME$',
- workingDir : getWorkDir(type)
+ workingDir : getWorkDir(type),
+ programParams: imageName.contains("controller") ? '0' : '--id 0'
]
def engine = new SimpleTemplateEngine()
@@ -174,7 +186,7 @@ def getSysProps(def envMap, String type){
def props = config[type].props
def sysProps = transformEnv(envMap)
sysProps.putAll(props)
- sysProps.collect{k,v -> "-D$k='$v'"}.join(' ').replace('\'','')
+ sysProps.collect{k,v -> "-D$k='$v'"}.join(' ').replace('"','').replace('\'','')
}
//Implements the logic from transformEnvironment.sh
@@ -213,7 +225,7 @@ def transformEnv(Map<String, String> envMap){
* This method converts it to map and add provided overrides with overrides from config
*/
def getEnv(Map envMap, String type, Map overrides){
- def ignoredKeys = ['PATH']
+ def ignoredKeys = ['PATH','JAVA_HOME','JAVA_VERSION','JAVA_TOOL_OPTIONS']
def overridesFromConfig = config[type].env
Map sortedMap = new TreeMap(envMap)
sortedMap.putAll(overrides)