You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ra...@apache.org on 2017/11/27 17:12:38 UTC
[incubator-openwhisk] branch master updated: Configure components
via pureconfig. (#2576)
This is an automated email from the ASF dual-hosted git repository.
rabbah pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new aa697f6 Configure components via pureconfig. (#2576)
aa697f6 is described below
commit aa697f620ae5f07828da677d06fa0f62b4f60adc
Author: Markus Thömmes <ma...@me.com>
AuthorDate: Mon Nov 27 18:12:35 2017 +0100
Configure components via pureconfig. (#2576)
This adds the ability to configure the main components
(controller/invoker) via pureconfig case-classes vs. using
WhiskConfig.
Benefits are:
- Automatic type derivation
- Static loading of config vs. passing one instance around
- Proper scoping of configuration
- It's the standard in Scala (based on TypesafeConfig)
Implementation:
TypesafeConfig reads JSON-like configuration files, which we can nest
as we please. Those namespaces can be read by each component
independently. While TypesafeConfig supports expansion of environment
variables, one has to do so explicitly. Which System properties, one
can override specific values directly without the added overhead. To
achieve the best of both worlds (for containers we only have
environment variables available) there is a small script which
translates environment variables to system properties in Java.
---
ansible/roles/controller/tasks/deploy.yml | 24 +++++++-----
ansible/roles/invoker/tasks/deploy.yml | 2 +-
common/scala/Dockerfile | 3 ++
common/scala/build.gradle | 2 +
common/scala/src/main/resources/application.conf | 5 ---
common/scala/transformEnvironment.sh | 45 ++++++++++++++++++++++
core/controller/Dockerfile | 6 ++-
core/controller/init.sh | 6 +++
.../controller/src/main/resources/application.conf | 18 ++++-----
.../core/loadBalancer/LoadBalancerService.scala | 16 ++++----
core/invoker/Dockerfile | 4 ++
core/invoker/init.sh | 6 +++
docs/dev/configuration.md | 17 ++++++++
settings.gradle | 2 +-
14 files changed, 121 insertions(+), 35 deletions(-)
diff --git a/ansible/roles/controller/tasks/deploy.yml b/ansible/roles/controller/tasks/deploy.yml
index 2e5d276..3978316 100644
--- a/ansible/roles/controller/tasks/deploy.yml
+++ b/ansible/roles/controller/tasks/deploy.yml
@@ -75,27 +75,33 @@
"LIMITS_ACTIONS_INVOKES_CONCURRENTINSYSTEM": "{{ limits.concurrentInvocationsSystem }}"
"LIMITS_TRIGGERS_FIRES_PERMINUTE": "{{ limits.firesPerMinute }}"
"LIMITS_ACTIONS_SEQUENCE_MAXLENGTH": "{{ limits.sequenceMaxLength }}"
- "CONTROLLER_BLACKBOXFRACTION": "{{ controller.blackboxFraction }}"
- "LOADBALANCER_INVOKERBUSYTHRESHOLD": "{{ invoker.busyThreshold }}"
"RUNTIMES_MANIFEST": "{{ runtimesManifest | to_json }}"
"CONTROLLER_LOCALBOOKKEEPING": "{{ controller.localBookkeeping }}"
- "AKKA_CLUSTER_PORT": "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname) }}"
- "AKKA_CLUSTER_HOST": "{{ controller.akka.cluster.host[groups['controllers'].index(inventory_hostname)] }}"
"AKKA_CLUSTER_SEED_NODES": "{{seed_nodes_list | join(' ') }}"
- "AKKA_CLUSTER_BIND_PORT": "{{ controller.akka.cluster.bindPort }}"
- "AKKA_ACTOR_PROVIDER": "{{ controller.akka.provider }}"
"METRICS_KAMON": "{{ metrics.kamon.enabled }}"
"METRICS_LOG": "{{ metrics.log.enabled }}"
- "METRICS_KAMON_HOST": "{{ metrics.kamon.host }}"
- "METRICS_KAMON_PORT": "{{ metrics.kamon.port }}"
"CONTROLLER_HA": "{{ controller.ha }}"
+
+ "METRICS_KAMON": "{{ metrics.kamon.enabled }}"
+ "METRICS_LOG": "{{ metrics.log.enabled }}"
+
+ "CONFIG_whisk_loadbalancer_invokerBusyThreshold": "{{ invoker.busyThreshold }}"
+ "CONFIG_whisk_loadbalancer_blackboxFraction": "{{ controller.blackboxFraction }}"
+
+ "CONFIG_akka_actor_provider": "{{ controller.akka.provider }}"
+ "CONFIG_akka_remote_netty_tcp_hostname": "{{ controller.akka.cluster.host[groups['controllers'].index(inventory_hostname)] }}"
+ "CONFIG_akka_remote_netty_tcp_port": "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname) }}"
+ "CONFIG_akka_remote_netty_tcp_bindPort": "{{ controller.akka.cluster.bindPort }}"
+
+ "CONFIG_kamon_statsd_hostname": "{{ metrics.kamon.host }}"
+ "CONFIG_kamon_statsd_port": "{{ metrics.kamon.port }}"
volumes:
- "{{ whisk_logs_dir }}/controller{{ groups['controllers'].index(inventory_hostname) }}:/logs"
ports:
- "{{ controller.basePort + groups['controllers'].index(inventory_hostname) }}:8080"
- "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname) }}:{{ controller.akka.cluster.bindPort }}"
- command: /bin/sh -c "export CONTAINER_IP=$(hostname -I); controller/bin/controller {{ groups['controllers'].index(inventory_hostname) }} >> /logs/controller{{ groups['controllers'].index(inventory_hostname) }}_logs.log 2>&1"
+ command: /bin/sh -c "exec /init.sh {{ groups['controllers'].index(inventory_hostname) }} >> /logs/controller{{ groups['controllers'].index(inventory_hostname) }}_logs.log 2>&1"
- name: wait until the Controller in this host is up and running
uri:
diff --git a/ansible/roles/invoker/tasks/deploy.yml b/ansible/roles/invoker/tasks/deploy.yml
index 560d64a..94c06a3 100644
--- a/ansible/roles/invoker/tasks/deploy.yml
+++ b/ansible/roles/invoker/tasks/deploy.yml
@@ -161,7 +161,7 @@
-v {{ docker_sock | default('/var/run/docker.sock') }}:/var/run/docker.sock
-p {{ invoker.port + groups['invokers'].index(inventory_hostname) }}:8080
{{ docker_registry }}{{ docker.image.prefix }}/invoker:{{ docker.image.tag }}
- /bin/sh -c "exec /invoker/bin/invoker {{ groups['invokers'].index(inventory_hostname) }} >> /logs/invoker{{ groups['invokers'].index(inventory_hostname) }}_logs.log 2>&1"
+ /bin/sh -c "exec /init.sh {{ groups['invokers'].index(inventory_hostname) }} >> /logs/invoker{{ groups['invokers'].index(inventory_hostname) }}_logs.log 2>&1"
# todo: re-enable docker_container module once https://github.com/ansible/ansible-modules-core/issues/5054 is resolved
diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile
index 750b1d5..15e5bdf 100644
--- a/common/scala/Dockerfile
+++ b/common/scala/Dockerfile
@@ -29,3 +29,6 @@ RUN update-alternatives --install "/usr/bin/java" "java" "${JRE_HOME}/bin/java"
update-alternatives --set java "${JRE_HOME}/bin/java" && \
update-alternatives --set javac "${JAVA_HOME}/bin/javac" && \
mkdir /logs
+
+COPY transformEnvironment.sh /
+RUN chmod +x transformEnvironment.sh
\ No newline at end of file
diff --git a/common/scala/build.gradle b/common/scala/build.gradle
index 8777f47..31781f4 100644
--- a/common/scala/build.gradle
+++ b/common/scala/build.gradle
@@ -11,11 +11,13 @@ repositories {
dependencies {
compile "org.scala-lang:scala-library:${gradle.scala.version}"
+ compile 'com.github.pureconfig:pureconfig_2.11:0.7.2'
compile 'io.spray:spray-json_2.11:1.3.3'
compile 'com.typesafe.akka:akka-actor_2.11:2.5.6'
compile 'com.typesafe.akka:akka-stream_2.11:2.5.6'
compile 'com.typesafe.akka:akka-slf4j_2.11:2.5.6'
+
compile 'com.typesafe.akka:akka-http-core_2.11:10.0.10'
compile 'com.typesafe.akka:akka-http-spray-json_2.11:10.0.10'
diff --git a/common/scala/src/main/resources/application.conf b/common/scala/src/main/resources/application.conf
index 1968e90..41b41ce 100644
--- a/common/scala/src/main/resources/application.conf
+++ b/common/scala/src/main/resources/application.conf
@@ -21,11 +21,6 @@ kamon {
}
statsd {
- # Hostname and port in which your StatsD is running. Remember that StatsD packets are sent using UDP and
- # setting unreachable hosts and/or not open ports wont be warned by the Kamon, your data wont go anywhere.
- hostname = ${?METRICS_KAMON_HOST}
- port = ${?METRICS_KAMON_PORT}
-
# Interval between metrics data flushes to StatsD. It's value must be equal or greater than the
# kamon.metrics.tick-interval setting.
flush-interval = 1 second
diff --git a/common/scala/transformEnvironment.sh b/common/scala/transformEnvironment.sh
new file mode 100755
index 0000000..05039ff
--- /dev/null
+++ b/common/scala/transformEnvironment.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Transforms environment variables starting with `prefix` to kebab-cased JVM system properties
+#
+# "_" becomes "."
+# "camelCased" becomes "camel-cased"
+# "PascalCased" stays "PascalCased" -> classnames stay untouched
+#
+# Examples:
+# CONFIG_whisk_loadbalancer_invokerBusyThreshold -> -Dwhisk.loadbalancer.invoker-busy-threshold
+# CONFIG_akka_remote_netty_tcp_bindPort -> -Dakka.remote.netty.tcp.bind-port
+# CONFIG_whisk_spi_LogStoreProvider -> -Dwhisk.spi.LogStoreProvider
+#
+
+prefix="CONFIG_"
+configVariables=$(compgen -v | grep $prefix)
+
+props=()
+
+for var in $configVariables
+do
+ value=$(printenv "$var")
+ if [ ! -z "$value" ]
+ then
+ sansConfig=${var#$prefix} # remove the CONFIG_ prefix
+ parts=${sansConfig//_/ } # "split" the name by replacing '_' with ' '
+
+ transformedParts=()
+ for part in $parts
+ do
+ if [[ $part =~ ^[A-Z] ]] # if the current part starts with an uppercase letter (is PascalCased)
+ then
+ transformedParts+=($part) # leave it alone
+ else
+ transformedParts+=($(echo "$part" | sed -r 's/([a-z0-9])([A-Z])/\1-\L\2/g')) # rewrite camelCased to kebab-cased
+ fi
+ done
+
+ key=$(IFS=.; echo "${transformedParts[*]}") # reassemble the parts delimited by a '.'
+ props+=("-D$key='$value'") # assemble a JVM system property
+ fi
+done
+
+echo "${props[@]}"
\ No newline at end of file
diff --git a/core/controller/Dockerfile b/core/controller/Dockerfile
index ecfc0ec..51f673d 100644
--- a/core/controller/Dockerfile
+++ b/core/controller/Dockerfile
@@ -11,7 +11,11 @@ RUN wget --no-verbose https://github.com/swagger-api/swagger-ui/archive/v2.1.4.t
#
# Copy app jars
-COPY build/distributions/controller.tar ./
+COPY build/distributions/controller.tar /
RUN tar xf controller.tar
+COPY init.sh /
+RUN chmod +x init.sh
+
EXPOSE 8080
+CMD ["init.sh", "0"]
\ No newline at end of file
diff --git a/core/controller/init.sh b/core/controller/init.sh
new file mode 100644
index 0000000..232c405
--- /dev/null
+++ b/core/controller/init.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export CONTROLLER_OPTS
+CONTROLLER_OPTS="$CONTROLLER_OPTS -Dakka.remote.netty.tcp.bind-hostname=$(hostname -I) $(./transformEnvironment.sh)"
+
+exec controller/bin/controller "$@"
\ No newline at end of file
diff --git a/core/controller/src/main/resources/application.conf b/core/controller/src/main/resources/application.conf
index 6e13354..10615d3 100644
--- a/core/controller/src/main/resources/application.conf
+++ b/core/controller/src/main/resources/application.conf
@@ -2,6 +2,13 @@
include "logging"
include "akka-http-version"
+whisk {
+ loadbalancer {
+ invoker-busy-threshold: 4
+ blackbox-fraction: 10%
+ }
+}
+
# http://doc.akka.io/docs/akka-http/current/scala/http/configuration.html
# descriptions inlined below for convenience
akka.http {
@@ -59,26 +66,15 @@ akka.http {
# Check out all akka-remote-2.5.4 options here:
# http://doc.akka.io/docs/akka/2.5.4/scala/general/configuration.html#config-akka-remote
akka {
- actor {
- provider = ${?AKKA_ACTOR_PROVIDER}
- }
remote {
log-remote-lifecycle-events = DEBUG
log-received-messages = on
log-sent-messages = on
-
- netty.tcp {
- hostname = ${?AKKA_CLUSTER_HOST}
- port = ${?AKKA_CLUSTER_PORT}
- bind-port = ${?AKKA_CLUSTER_BIND_PORT}
- bind-hostname = ${?CONTAINER_IP}
- }
}
cluster {
# Disable legacy metrics in akka-cluster.
metrics.enabled=off
- auto-down-unreachable-after = ${?AUTO_DOWN_UNREACHABLE_AFTER}
#distributed-data.notify-subscribers-interval = 0.01
}
}
\ No newline at end of file
diff --git a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
index a8be73b..e38f69c 100644
--- a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
+++ b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
@@ -55,6 +55,10 @@ import whisk.core.entity.size._
import whisk.core.entity.types.EntityStore
import whisk.spi.SpiLoader
+import pureconfig._
+
+case class LoadbalancerConfig(blackboxFraction: Double, invokerBusyThreshold: Int)
+
trait LoadBalancer {
val activeAckTimeoutGrace = 1.minute
@@ -87,11 +91,13 @@ class LoadBalancerService(config: WhiskConfig, instance: InstanceId, entityStore
logging: Logging)
extends LoadBalancer {
+ private val lbConfig = loadConfigOrThrow[LoadbalancerConfig]("whisk.loadbalancer")
+
/** The execution context for futures */
implicit val executionContext: ExecutionContext = actorSystem.dispatcher
/** How many invokers are dedicated to blackbox images. We range bound to something sensical regardless of configuration. */
- private val blackboxFraction: Double = Math.max(0.0, Math.min(1.0, config.controllerBlackboxFraction))
+ private val blackboxFraction: Double = Math.max(0.0, Math.min(1.0, lbConfig.blackboxFraction))
logging.info(this, s"blackboxFraction = $blackboxFraction")(TransactionId.loadbalancer)
/** Feature switch for shared load balancer data **/
@@ -324,7 +330,7 @@ class LoadBalancerService(config: WhiskConfig, instance: InstanceId, entityStore
case (instance, state) => (instance, state, currentActivations.getOrElse(instance.toString, 0))
}
- LoadBalancerService.schedule(invokersWithUsage, config.loadbalancerInvokerBusyThreshold, hash) match {
+ LoadBalancerService.schedule(invokersWithUsage, lbConfig.invokerBusyThreshold, hash) match {
case Some(invoker) => Future.successful(invoker)
case None =>
logging.error(this, s"all invokers down")(TransactionId.invokerHealth)
@@ -347,11 +353,7 @@ object LoadBalancerService {
kafkaTopicsCompletedRetentionBytes -> 1024.MB.toBytes.toString,
kafkaTopicsCompletedRetentionMS -> 1.hour.toMillis.toString,
kafkaTopicsCompletedSegmentBytes -> 512.MB.toBytes.toString) ++
- Map(
- loadbalancerInvokerBusyThreshold -> null,
- controllerBlackboxFraction -> null,
- controllerLocalBookkeeping -> null,
- controllerSeedNodes -> null)
+ Map(controllerLocalBookkeeping -> null, controllerSeedNodes -> null)
/** Memoizes the result of `f` for later use. */
def memoize[I, O](f: I => O): I => O = new scala.collection.mutable.HashMap[I, O]() {
diff --git a/core/invoker/Dockerfile b/core/invoker/Dockerfile
index 3a7a53f..2a8b6ee 100644
--- a/core/invoker/Dockerfile
+++ b/core/invoker/Dockerfile
@@ -15,4 +15,8 @@ COPY build/distributions/invoker.tar ./
RUN tar xf invoker.tar && \
rm -f invoker.tar
+COPY init.sh /
+RUN chmod +x init.sh
+
EXPOSE 8080
+CMD ["init.sh", "0"]
\ No newline at end of file
diff --git a/core/invoker/init.sh b/core/invoker/init.sh
new file mode 100644
index 0000000..beb5e71
--- /dev/null
+++ b/core/invoker/init.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export INVOKER_OPTS
+INVOKER_OPTS="$INVOKER_OPTS $(./transformEnvironment.sh)"
+
+exec invoker/bin/invoker "$@"
\ No newline at end of file
diff --git a/docs/dev/configuration.md b/docs/dev/configuration.md
new file mode 100644
index 0000000..cedc31c
--- /dev/null
+++ b/docs/dev/configuration.md
@@ -0,0 +1,17 @@
+# Scala component configuration
+
+Our Scala components are in the process of switching from our own `WhiskConfig` to the TypesafeConfig based `pureconfig`. That will give us a more sane way of parsing environment values, overriding them and defining sensible defaults for them.
+
+TypesafeConfig is able to override its values via JVM system properties. Unfortunately, docker only supports passing environment variables in a comfortable way.
+
+To solve that mismatch, we use a script to transform environment variables into JVM system properties to spare the developer of manually overriding all the values via explicit environment -> TypesafeConfig value mapping. That script applies the following transformations to all environment variables starting with a defined prefix ("CONFIG_" in our case):
+
+- `_` becomes `.` (TypesafeConfig hierarchies are built using `.`)
+- `camelCased` becomes `camel-cased` (Kebabcase is usually used for TypesafeConfig keys)
+- `PascalCased` stays `PascalCased` (Defining classnames for overriding SPIs is crucial)
+
+### Examples:
+
+- `CONFIG_whisk_loadbalancer_invokerBusyThreshold` becomes `-Dwhisk.loadbalancer.invoker-busy-threshold`
+- `CONFIG_akka_remote_netty_tcp_bindPort` becomes `-Dakka.remote.netty.tcp.bind-port`
+- `CONFIG_whisk_spi_LogStoreProvider` becomes `-Dwhisk.spi.LogStoreProvider`
diff --git a/settings.gradle b/settings.gradle
index 19d3506..441c6b6 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -21,7 +21,7 @@ include 'tests:dat:blackbox:badproxy'
rootProject.name = 'openwhisk'
gradle.ext.scala = [
- version: '2.11.8',
+ version: '2.11.11',
compileFlags: ['-feature', '-unchecked', '-deprecation', '-Xfatal-warnings', '-Ywarn-unused-import']
]
--
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].