You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by cb...@apache.org on 2018/01/16 09:06:43 UTC
[incubator-openwhisk] branch master updated: Configure jmxremote.
(#3163)
This is an automated email from the ASF dual-hosted git repository.
cbickel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new fc98e9a Configure jmxremote. (#3163)
fc98e9a is described below
commit fc98e9acc91191c9b38790eb38f304002f5a264d
Author: Martin Henke <ma...@web.de>
AuthorDate: Tue Jan 16 10:06:40 2018 +0100
Configure jmxremote. (#3163)
* Configure jmxremote
* Address review comments
Signed-off-by: Martin Henke <ma...@de.ibm.com>
* use conf dir to move jmx user and pw files
---
ansible/environments/docker-machine/group_vars/all | 3 --
ansible/environments/local/group_vars/all | 3 --
ansible/group_vars/all | 30 +++++++++++++++
ansible/roles/controller/tasks/clean.yml | 6 +++
ansible/roles/controller/tasks/deploy.yml | 45 +++++++++++++++++++---
ansible/roles/invoker/tasks/clean.yml | 6 +++
ansible/roles/invoker/tasks/deploy.yml | 32 ++++++++++++++-
ansible/templates/jmxremote.access.j2 | 1 +
ansible/templates/jmxremote.password.j2 | 1 +
common/scala/.dockerignore | 1 +
common/scala/Dockerfile | 5 ++-
common/scala/copyJMXFiles.sh | 7 ++++
common/scala/src/main/resources/logback.xml | 1 +
core/controller/init.sh | 4 +-
core/invoker/init.sh | 4 +-
15 files changed, 134 insertions(+), 15 deletions(-)
diff --git a/ansible/environments/docker-machine/group_vars/all b/ansible/environments/docker-machine/group_vars/all
index a09f335..efd0b56 100644
--- a/ansible/environments/docker-machine/group_vars/all
+++ b/ansible/environments/docker-machine/group_vars/all
@@ -28,9 +28,6 @@ apigw_auth_user: ""
apigw_auth_pwd: ""
apigw_host_v2: "http://{{ groups['apigateway']|first }}:{{apigateway.port.api}}/v2"
-controller_arguments: '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098'
-invoker_arguments: "{{ controller_arguments }}"
-
invoker_allow_multiple_instances: true
# Set kafka configuration
diff --git a/ansible/environments/local/group_vars/all b/ansible/environments/local/group_vars/all
index 9a10b00..0d8dc06 100755
--- a/ansible/environments/local/group_vars/all
+++ b/ansible/environments/local/group_vars/all
@@ -20,9 +20,6 @@ apigw_auth_user: ""
apigw_auth_pwd: ""
apigw_host_v2: "http://{{ groups['apigateway']|first }}:{{apigateway.port.api}}/v2"
-controller_arguments: '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098'
-invoker_arguments: "{{ controller_arguments }}"
-
invoker_allow_multiple_instances: true
# Set kafka configuration
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index c6957fc..31c462e 100644
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -43,8 +43,14 @@ limits:
firesPerMinute: "{{ limit_fires_per_minute | default(60) }}"
sequenceMaxLength: "{{ limit_sequence_max_length | default(50) }}"
+controllerHostnameFromMap: "{{ groups['controllers'] | map('extract', hostvars, 'ansible_host') | list | first }}"
+controllerHostname: "{{ controllerHostnameFromMap | default(inventory_hostname) }}"
+
# port means outer port
controller:
+ dir:
+ become: "{{ controller_dir_become | default(false) }}"
+ confdir: "{{ config_root_dir }}/controller"
basePort: 10001
heap: "{{ controller_heap | default('2g') }}"
arguments: "{{ controller_arguments | default('') }}"
@@ -62,6 +68,20 @@ controller:
# We recommend to enable HA for the controllers only, if bookkeeping data are shared too. (localBookkeeping: false)
ha: "{{ controller_enable_ha | default(True) and groups['controllers'] | length > 1 }}"
loglevel: "{{ controller_loglevel | default(whisk_loglevel) | default('INFO') }}"
+ jmxremote:
+ jvmArgs: "{% if inventory_hostname in groups['controllers'] %}
+ {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ controllerHostname }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortController + groups['controllers'].index(inventory_hostname) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortController + groups['controllers'].index(inventory_hostname) }}
+ {% endif %}"
+
+jmx:
+ basePortController: 15000
+ rmiBasePortController: 16000
+ basePortInvoker: 17000
+ rmiBasePortInvoker: 18000
+ user: "{{ jmxuser | default('jmxuser') }}"
+ pass: "{{ jmxuser | default('jmxpass') }}"
+ jvmCommonArgs: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.password.file=/root/jmxremote.password -Dcom.sun.management.jmxremote.access.file=/root/jmxremote.access"
+ enabled: "{{ jmxremote_enabled | default('true') }}"
transactions:
stride: "{{ groups['controllers'] | length }}"
@@ -93,7 +113,13 @@ zookeeper_connect_string: "{% set ret = [] %}\
{% endfor %}\
{{ ret | join(',') }}"
+invokerHostnameFromMap: "{{ groups['invokers'] | map('extract', hostvars, 'ansible_host') | list | first }}"
+invokerHostname: "{{ invokerHostnameFromMap | default(inventory_hostname) }}"
+
invoker:
+ dir:
+ become: "{{ invoker_dir_become | default(false) }}"
+ confdir: "{{ config_root_dir }}/invoker"
port: 12001
heap: "{{ invoker_heap | default('2g') }}"
arguments: "{{ invoker_arguments | default('') }}"
@@ -108,6 +134,10 @@ invoker:
docker:
become: "{{ invoker_docker_become | default(false) }}"
loglevel: "{{ invoker_loglevel | default(whisk_loglevel) | default('INFO') }}"
+ jmxremote:
+ jvmArgs: "{% if inventory_hostname in groups['invokers'] %}
+ {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ invokerHostname }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }}
+ {% endif %}"
userLogs:
spi: "{{ userLogs_spi | default('whisk.core.containerpool.logging.DockerToActivationLogStoreProvider') }}"
diff --git a/ansible/roles/controller/tasks/clean.yml b/ansible/roles/controller/tasks/clean.yml
index e3bbe07..231198a 100644
--- a/ansible/roles/controller/tasks/clean.yml
+++ b/ansible/roles/controller/tasks/clean.yml
@@ -13,3 +13,9 @@
path: "{{ whisk_logs_dir }}/controller{{ groups['controllers'].index(inventory_hostname) }}"
state: absent
become: "{{ logs.dir.become }}"
+
+- name: remove controller conf directory
+ file:
+ path: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}"
+ state: absent
+ become: "{{ controller.dir.become }}"
diff --git a/ansible/roles/controller/tasks/deploy.yml b/ansible/roles/controller/tasks/deploy.yml
index 2400bf8..671aa17 100644
--- a/ansible/roles/controller/tasks/deploy.yml
+++ b/ansible/roles/controller/tasks/deploy.yml
@@ -16,6 +16,27 @@
mode: 0777
become: "{{ logs.dir.become }}"
+- name: ensure controller config directory is created with permissions
+ file:
+ path: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}"
+ state: directory
+ mode: 0777
+ become: "{{ controller.dir.become }}"
+
+- name: copy jmxremote password file
+ when: jmx.enabled
+ template:
+ src: "jmxremote.password.j2"
+ dest: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}/jmxremote.password"
+ mode: 0777
+
+- name: copy jmxremote access file
+ when: jmx.enabled
+ template:
+ src: "jmxremote.access.j2"
+ dest: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}/jmxremote.access"
+ mode: 0777
+
- name: check, that required databases exist
include: "{{ openwhisk_home }}/ansible/tasks/db/checkDb.yml"
vars:
@@ -25,6 +46,20 @@
- "{{ db.whisk.auth }}"
- "{{ db.whisk.activations }}"
+- name: prepare controller ports
+ set_fact:
+ ports_to_expose: ["{{ controller.basePort + groups['controllers'].index(inventory_hostname) }}:8080", "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname) }}:{{ controller.akka.cluster.bindPort }}"]
+
+- name: expose additional ports if jmxremote is enabled
+ when: jmx.enabled
+ set_fact:
+ ports_to_expose: "{{ ports_to_expose }} + [ \"{{ jmx.basePortController + groups['controllers'].index(inventory_hostname) }}:{{ jmx.basePortController + groups['controllers'].index(inventory_hostname) }}\" ] + [ \"{{ jmx.rmiBasePortController + groups['controllers'].index(inventory_hostname) }}:{{ jmx.rmiBasePortController + groups['controllers'].index(inventory_hostname) }}\" ]"
+
+- name: add additional jvm params if jmxremote is enabled
+ when: jmx.enabled
+ set_fact:
+ controller_args: "{{ controller.arguments }} {{ controller.jmxremote.jvmArgs }}"
+
- name: create seed nodes list
set_fact:
seed_nodes_list: "{{ seed_nodes_list | default([]) }} + [ \"{{item.1}}:{{controller.akka.cluster.basePort+item.0}}\" ]"
@@ -41,8 +76,9 @@
hostname: "controller{{ groups['controllers'].index(inventory_hostname) }}"
env:
"JAVA_OPTS": "-Xmx{{ controller.heap }} -XX:+CrashOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:ErrorFile=/logs/java_error.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs"
- "CONTROLLER_OPTS": "{{ controller.arguments }}"
+ "CONTROLLER_OPTS": "{{ controller_args | default(controller.arguments) }}"
"CONTROLLER_INSTANCES": "{{ controller.instances }}"
+ "JMX_REMOTE": "{{ jmx.enabled }}"
"COMPONENT_NAME": "controller{{ groups['controllers'].index(inventory_hostname) }}"
"PORT": 8080
@@ -112,9 +148,8 @@
"CONFIG_whisk_transactions_stride": "{{ transactions.stride | default() }}"
volumes:
- "{{ whisk_logs_dir }}/controller{{ groups['controllers'].index(inventory_hostname) }}:/logs"
- ports:
- - "{{ controller.basePort + groups['controllers'].index(inventory_hostname) }}:8080"
- - "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname) }}:{{ controller.akka.cluster.bindPort }}"
+ - "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}:/conf"
+ ports: "{{ ports_to_expose }}"
command: /bin/sh -c "exec /init.sh {{ groups['controllers'].index(inventory_hostname) }} >> /logs/controller{{ groups['controllers'].index(inventory_hostname) }}_logs.log 2>&1"
- name: wait until the Controller in this host is up and running
@@ -123,4 +158,4 @@
register: result
until: result.status == 200
retries: 12
- delay: 5
+ delay: 5
\ No newline at end of file
diff --git a/ansible/roles/invoker/tasks/clean.yml b/ansible/roles/invoker/tasks/clean.yml
index 4028269..ae5b83b 100644
--- a/ansible/roles/invoker/tasks/clean.yml
+++ b/ansible/roles/invoker/tasks/clean.yml
@@ -39,6 +39,12 @@
state: absent
become: "{{ logs.dir.become }}"
+- name: remove invoker conf directory
+ file:
+ path: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}"
+ state: absent
+ become: "{{ invoker.dir.become }}"
+
# Workaround for orphaned ifstate.veth* files on Ubuntu 14.04
# See https://github.com/moby/moby/issues/22513
# Remove inactive files older than 60 minutes
diff --git a/ansible/roles/invoker/tasks/deploy.yml b/ansible/roles/invoker/tasks/deploy.yml
index 7199125..bd05da2 100644
--- a/ansible/roles/invoker/tasks/deploy.yml
+++ b/ansible/roles/invoker/tasks/deploy.yml
@@ -53,6 +53,13 @@
mode: 0777
become: "{{ logs.dir.become }}"
+- name: ensure invoker config directory is created with permissions
+ file:
+ path: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}"
+ state: directory
+ mode: 0777
+ become: "{{ invoker.dir.become }}"
+
- name: check, that required databases exist
include: "{{ openwhisk_home }}/ansible/tasks/db/checkDb.yml"
vars:
@@ -103,6 +110,25 @@
with_items: "{{ invokerInfo }}"
when: not invoker.allowMultipleInstances and item.Names[0] != "/invoker{{ groups['invokers'].index(inventory_hostname) }}"
+- name: copy jmxremote password file
+ when: jmx.enabled
+ template:
+ src: "jmxremote.password.j2"
+ dest: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}/jmxremote.password"
+ mode: 0777
+
+- name: copy jmxremote access file
+ when: jmx.enabled
+ template:
+ src: "jmxremote.access.j2"
+ dest: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}/jmxremote.access"
+ mode: 0777
+
+- name: add additional jvm params if jmxremote is enabled
+ when: jmx.enabled
+ set_fact:
+ invoker_args: "{{ invoker.arguments }} {{ invoker.jmxremote.jvmArgs }}"
+
- name: start invoker using docker cli
shell: >
docker run -d
@@ -114,7 +140,8 @@
--hostname invoker{{ groups['invokers'].index(inventory_hostname) }}
--restart {{ docker.restart.policy }}
-e JAVA_OPTS='-Xmx{{ invoker.heap }} -XX:+CrashOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:ErrorFile=/logs/java_error.log'
- -e INVOKER_OPTS='{{ invoker.arguments }}'
+ -e INVOKER_OPTS='{{ invoker_args | default(invoker.arguments) }}'
+ -e JMX_REMOTE='{{ jmx.enabled }}'
-e COMPONENT_NAME='invoker{{ groups['invokers'].index(inventory_hostname) }}'
-e PORT='8080'
-e KAFKA_HOSTS='{{ kafka_connect_string }}'
@@ -161,9 +188,12 @@
-v /sys/fs/cgroup:/sys/fs/cgroup
-v /run/runc:/run/runc
-v {{ whisk_logs_dir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}:/logs
+ -v {{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}:/conf
-v {{ dockerInfo["DockerRootDir"] }}/containers/:/containers
-v {{ docker_sock | default('/var/run/docker.sock') }}:/var/run/docker.sock
-p {{ invoker.port + groups['invokers'].index(inventory_hostname) }}:8080
+ {% if jmx.enabled %} -p {{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }}:{{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }} {% endif %}
+ {% if jmx.enabled %} -p {{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }}:{{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }} {% endif %}
{{ docker_registry }}{{ docker.image.prefix }}/invoker:{{ docker.image.tag }}
/bin/sh -c "exec /init.sh {{ groups['invokers'].index(inventory_hostname) }} >> /logs/invoker{{ groups['invokers'].index(inventory_hostname) }}_logs.log 2>&1"
diff --git a/ansible/templates/jmxremote.access.j2 b/ansible/templates/jmxremote.access.j2
new file mode 100644
index 0000000..9d79568
--- /dev/null
+++ b/ansible/templates/jmxremote.access.j2
@@ -0,0 +1 @@
+{{ jmx.user }} readwrite
diff --git a/ansible/templates/jmxremote.password.j2 b/ansible/templates/jmxremote.password.j2
new file mode 100644
index 0000000..5d9c51b
--- /dev/null
+++ b/ansible/templates/jmxremote.password.j2
@@ -0,0 +1 @@
+{{ jmx.user }} {{ jmx.pass }}
diff --git a/common/scala/.dockerignore b/common/scala/.dockerignore
index eed7a81..8f456fd 100644
--- a/common/scala/.dockerignore
+++ b/common/scala/.dockerignore
@@ -1,3 +1,4 @@
*
!transformEnvironment.sh
+!copyJMXFiles.sh
!build/distributions
\ No newline at end of file
diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile
index 15e5bdf..114077c 100644
--- a/common/scala/Dockerfile
+++ b/common/scala/Dockerfile
@@ -31,4 +31,7 @@ RUN update-alternatives --install "/usr/bin/java" "java" "${JRE_HOME}/bin/java"
mkdir /logs
COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
\ No newline at end of file
+RUN chmod +x transformEnvironment.sh
+
+COPY copyJMXFiles.sh /
+RUN chmod +x copyJMXFiles.sh
\ No newline at end of file
diff --git a/common/scala/copyJMXFiles.sh b/common/scala/copyJMXFiles.sh
new file mode 100644
index 0000000..fc5004f
--- /dev/null
+++ b/common/scala/copyJMXFiles.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ $( ls /conf/jmxremote.* 2> /dev/null ) ]]
+then
+ mv /conf/jmxremote.* /root
+ chmod 600 /root/jmxremote.*
+fi
\ No newline at end of file
diff --git a/common/scala/src/main/resources/logback.xml b/common/scala/src/main/resources/logback.xml
index 50d6ee1..c268a7e 100644
--- a/common/scala/src/main/resources/logback.xml
+++ b/common/scala/src/main/resources/logback.xml
@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
+ <jmxConfigurator></jmxConfigurator>
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}] [%p] %msg%n</pattern>
diff --git a/core/controller/init.sh b/core/controller/init.sh
index 232c405..ec7d5e8 100644
--- a/core/controller/init.sh
+++ b/core/controller/init.sh
@@ -1,6 +1,8 @@
#!/bin/bash
+./copyJMXFiles.sh
+
export CONTROLLER_OPTS
CONTROLLER_OPTS="$CONTROLLER_OPTS -Dakka.remote.netty.tcp.bind-hostname=$(hostname -I) $(./transformEnvironment.sh)"
-exec controller/bin/controller "$@"
\ No newline at end of file
+exec controller/bin/controller "$@"
diff --git a/core/invoker/init.sh b/core/invoker/init.sh
index beb5e71..cfcf056 100644
--- a/core/invoker/init.sh
+++ b/core/invoker/init.sh
@@ -1,6 +1,8 @@
#!/bin/bash
+./copyJMXFiles.sh
+
export INVOKER_OPTS
INVOKER_OPTS="$INVOKER_OPTS $(./transformEnvironment.sh)"
-exec invoker/bin/invoker "$@"
\ No newline at end of file
+exec invoker/bin/invoker "$@"
--
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].