You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/12/06 21:54:37 UTC
[GitHub] ddragosd closed pull request #4142: tools/dev - updated intellij
script to start controller and invoker locally
ddragosd closed pull request #4142: tools/dev - updated intellij script to start controller and invoker locally
URL: https://github.com/apache/incubator-openwhisk/pull/4142
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index 85ee6ffbf3..a525afb9ad 100755
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -172,7 +172,7 @@ invoker:
port: 12001
heap: "{{ invoker_heap | default('2g') }}"
arguments: "{{ invoker_arguments | default('') }}"
- userMemory: "{{ invoker_user_memory | default('2048 m') }}"
+ userMemory: "{{ invoker_user_memory | default('2048m') }}"
instances: "{{ groups['invokers'] | length }}"
# Specify if it is allowed to deploy more than 1 invoker on a single machine.
allowMultipleInstances: "{{ invoker_allow_multiple_instances | default(false) }}"
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
index 4f493f2273..355c3abf52 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala
@@ -178,7 +178,7 @@ class DockerClient(dockerHost: Option[String] = None,
def isOomKilled(id: ContainerId)(implicit transid: TransactionId): Future[Boolean] =
runCmd(Seq("inspect", id.asString, "--format", "{{.State.OOMKilled}}"), config.timeouts.inspect).map(_.toBoolean)
- private def runCmd(args: Seq[String], timeout: Duration)(implicit transid: TransactionId): Future[String] = {
+ protected def runCmd(args: Seq[String], timeout: Duration)(implicit transid: TransactionId): Future[String] = {
val cmd = dockerCmd ++ args
val start = transid.started(
this,
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
index e5f11b1d82..c4ceff34c7 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala
@@ -18,6 +18,7 @@
package org.apache.openwhisk.core.containerpool.docker
import akka.actor.ActorSystem
+
import scala.concurrent.Await
import scala.concurrent.ExecutionContext
import scala.concurrent.Future
@@ -27,14 +28,16 @@ import org.apache.openwhisk.core.WhiskConfig
import org.apache.openwhisk.core.containerpool.Container
import org.apache.openwhisk.core.containerpool.ContainerFactory
import org.apache.openwhisk.core.containerpool.ContainerFactoryProvider
+import org.apache.openwhisk.core.containerpool.ContainerArgsConfig
import org.apache.openwhisk.core.entity.ByteSize
import org.apache.openwhisk.core.entity.ExecManifest
import org.apache.openwhisk.core.entity.InvokerInstanceId
+
import scala.concurrent.duration._
import java.util.concurrent.TimeoutException
+
import pureconfig._
import org.apache.openwhisk.core.ConfigKeys
-import org.apache.openwhisk.core.containerpool.ContainerArgsConfig
case class DockerContainerFactoryConfig(useRunc: Boolean)
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala
new file mode 100644
index 0000000000..fb7bc2e9b8
--- /dev/null
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.core.containerpool.docker
+
+import akka.actor.ActorSystem
+import org.apache.openwhisk.common.{Logging, TransactionId}
+import org.apache.openwhisk.core.WhiskConfig
+import org.apache.openwhisk.core.containerpool._
+import org.apache.openwhisk.core.entity.InvokerInstanceId
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+
+/**
+ * This factory provides a Docker for Mac client which exposes action container's ports on the host.
+ */
+object DockerForMacContainerFactoryProvider extends ContainerFactoryProvider {
+ override def instance(actorSystem: ActorSystem,
+ logging: Logging,
+ config: WhiskConfig,
+ instanceId: InvokerInstanceId,
+ parameters: Map[String, Set[String]]): ContainerFactory = {
+
+ new DockerContainerFactory(instanceId, parameters)(
+ actorSystem,
+ actorSystem.dispatcher,
+ logging,
+ new DockerForMacClient()(actorSystem.dispatcher)(logging, actorSystem),
+ new RuncClient()(actorSystem.dispatcher)(logging, actorSystem))
+ }
+
+}
+
+class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)(implicit log: Logging,
+ as: ActorSystem)
+ extends DockerClientWithFileAccess(dockerHost)(executionContext)
+ with DockerApiWithFileAccess {
+
+ implicit private val ec: ExecutionContext = executionContext
+
+ override def run(image: String, args: Seq[String] = Seq.empty[String])(
+ implicit transid: TransactionId): Future[ContainerId] = {
+ // b/c docker for mac doesn't have a routing to the action containers
+ // the port 8080 is exposed on the host on a random port number
+ val extraArgs: Seq[String] = Seq("-p", "0:8080") ++ args
+ super.run(image, extraArgs)
+ }
+ // See extended trait for description
+ override def inspectIPAddress(id: ContainerId, network: String)(
+ implicit transid: TransactionId): Future[ContainerAddress] = {
+ super
+ .runCmd(
+ Seq("inspect", "--format", """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}""", id.asString),
+ 10.seconds)
+ .flatMap {
+ case "<no value>" => Future.failed(new NoSuchElementException)
+ case stdout => Future.successful(ContainerAddress("localhost", stdout.toInt))
+ }
+ }
+}
diff --git a/tools/dev/README.md b/tools/dev/README.md
index ed3aa67b7e..ffd3849a4f 100644
--- a/tools/dev/README.md
+++ b/tools/dev/README.md
@@ -76,14 +76,28 @@ First setup OpenWhisk so that Controller and Invoker containers are up and runni
It would inspect the running docker containers and then generate the launch configs with name 'controller0'
and 'invoker0'.
-Key points to note:
+Now the docker container(s) (controller and/or invoker) can be stopped and they can be launched instead from within the IDE.
-1. Uses ~/tmp/openwhisk/controller (or invoker) as working directory.
-2. Changes the PORT to linked one. So controller gets started at 10001 only just like as its done in container.
+Key points to note:
-Now the docker container can be stopped and application can be launched from within the IDE.
+1. Controller uses port `10001` and Invoker uses port `12001`.
+2. Action activation logs are [disabled][2].
+3. SSL is disabled for Controller and Invoker.
+4. Make sure you have the loopback interface configured:
+ ```bash
+ sudo ifconfig lo0 alias 172.17.0.1/24
+ ```
+5. `~/.wskprops` must be updated with `APIHOST=http://localhost:10001` so that the `wsk` CLI communicates directly with the controller.
+6. On a MAC
+ * With Docker For Mac the invoker is configured to use a Container Factory that exposes ports for actions on the host,
+ as otherwise the invoker can't make HTTP requests to the actions.
+ You can read more at [docker/for-mac#171][7].
+
+ * When using [docker-compose][8] locally you have to update `/etc/hosts` with the line bellow:
+ ```
+ 127.0.0.1 kafka zookeeper kafka.docker zookeeper.docker db.docker controller whisk.controller
+ ```
-**Note** - Currently only the controller can be run from IDE. Invoker posses some [problems][2].
### Configuration
@@ -168,3 +182,5 @@ $ ./gradlew :tools:dev:renderModuleDetails
[4]: http://docs.groovy-lang.org/2.4.2/html/gapi/groovy/util/ConfigSlurper.html
[5]: https://developer.github.com/v3/search/
[6]: https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
+[7]: https://github.com/docker/for-mac/issues/171
+[8]: https://github.com/apache/incubator-openwhisk-devtools/tree/master/docker-compose
diff --git a/tools/dev/build.gradle b/tools/dev/build.gradle
index 0ce11834fc..745b91f01c 100644
--- a/tools/dev/build.gradle
+++ b/tools/dev/build.gradle
@@ -26,6 +26,7 @@ def owHome = project.projectDir.parentFile.parentFile
dependencies {
compile "org.codehaus.groovy:groovy-all:2.4.14"
compile 'org.apache.commons:commons-io:1.3.2'
+ compile 'org.apache.commons:commons-lang3:3.8.1'
}
task couchdbViews(type: JavaExec) {
diff --git a/tools/dev/src/main/groovy/intellijRunConfig.groovy b/tools/dev/src/main/groovy/intellijRunConfig.groovy
index ad2858b22f..cc41d878e8 100644
--- a/tools/dev/src/main/groovy/intellijRunConfig.groovy
+++ b/tools/dev/src/main/groovy/intellijRunConfig.groovy
@@ -16,6 +16,7 @@
*/
import groovy.json.JsonSlurper
import groovy.text.SimpleTemplateEngine
+import org.apache.commons.lang3.SystemUtils
assert args : "Expecting the OpenWhisk home directory to passed"
owHome = args[0]
@@ -26,7 +27,7 @@ def configTemplate = '''<component name="ProjectRunConfigurationManager">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" value="$main" />
<option name="VM_PARAMETERS" value="$sysProps" />
- <option name="PROGRAM_PARAMETERS" value="0" />
+ <option name="PROGRAM_PARAMETERS" value="$programParams" />
<option name="WORKING_DIRECTORY" value="$workingDir" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
@@ -57,13 +58,15 @@ Map controllerEnv = null
Map invokerEnv = null
containerNames.each{cn ->
+
//Inspect the specific container
def inspectResult = "docker inspect $cn".execute().text
def json = new JsonSlurper().parseText(inspectResult)
def imageName = json[0].'Config'.'Image'
if (imageName.contains("controller") || imageName.contains("invoker")){
- def mappedPort = json.'NetworkSettings'.'Ports'.'8080/tcp'[0][0].'HostPort'
+ // pre-configure the local ports for controller and invoker
+ def mappedPort = imageName.contains("controller") ? '10001' : '12001'
def envBaseMap = getEnvMap(json[0].'Config'.'Env')
String type
@@ -84,6 +87,14 @@ containerNames.each{cn ->
//Prepare system properties
def sysProps = getSysProps(envMap,type)
+ // disable log collection. See more at: https://github.com/apache/incubator-openwhisk/issues/3195
+ sysProps += " -Dwhisk.log-limit.max=0 -Dwhisk.log-limit.std=0"
+ // disable https protocol for controller and invoker
+ sysProps = sysProps.replaceAll("protocol=https", "protocol=http")
+ if (SystemUtils.IS_OS_MAC){
+ sysProps = sysProps.replaceAll("use-runc=True", "use-runc=False")
+ sysProps += " -Dwhisk.spi.ContainerFactoryProvider=org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider"
+ }
def templateBinding = [
main: meta[type].main,
@@ -92,7 +103,8 @@ containerNames.each{cn ->
env: encodeForXML(envMap),
sysProps : sysProps,
USER_HOME : '$USER_HOME$',
- workingDir : getWorkDir(type)
+ workingDir : getWorkDir(type),
+ programParams: imageName.contains("controller") ? '0' : '--id 0'
]
def engine = new SimpleTemplateEngine()
@@ -174,7 +186,7 @@ def getSysProps(def envMap, String type){
def props = config[type].props
def sysProps = transformEnv(envMap)
sysProps.putAll(props)
- sysProps.collect{k,v -> "-D$k='$v'"}.join(' ').replace('\'','')
+ sysProps.collect{k,v -> "-D$k='$v'"}.join(' ').replace('"','').replace('\'','')
}
//Implements the logic from transformEnvironment.sh
@@ -213,7 +225,7 @@ def transformEnv(Map<String, String> envMap){
* This method converts it to map and add provided overrides with overrides from config
*/
def getEnv(Map envMap, String type, Map overrides){
- def ignoredKeys = ['PATH']
+ def ignoredKeys = ['PATH','JAVA_HOME','JAVA_VERSION','JAVA_TOOL_OPTIONS']
def overridesFromConfig = config[type].env
Map sortedMap = new TreeMap(envMap)
sortedMap.putAll(overrides)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services