[GitHub] ningyougang closed pull request #3432: Introduce jmx for kafka

[GitBox <gi...@apache.org>]

ningyougang closed pull request #3432: Introduce jmx for kafka
URL: https://github.com/apache/incubator-openwhisk/pull/3432
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index f0dca5fac1..89487fee8c 100644
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -91,6 +91,8 @@ jmx:
   rmiBasePortController: 16000
   basePortInvoker: 17000
   rmiBasePortInvoker: 18000
+  basePortKafka: 19000
+  rmiBasePortKafka: 20000
   user: "{{ jmxuser | default('jmxuser') }}"
   pass: "{{ jmxuser | default('jmxpass') }}"
   jvmCommonArgs: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.password.file=/root/jmxremote.password -Dcom.sun.management.jmxremote.access.file=/root/jmxremote.access"
@@ -103,6 +105,7 @@ registry:
   confdir: "{{ config_root_dir }}/registry"
 
 kafka:
+  confdir: "{{ config_root_dir }}/kafka"
   ssl:
     client_authentication: required
     keystore:
@@ -135,9 +138,6 @@ zookeeper_connect_string: "{% set ret = [] %}\
                            {% endfor %}\
                            {{ ret | join(',') }}"
 
-invokerHostnameFromMap: "{{ groups['invokers'] | map('extract', hostvars, 'ansible_host') | list | first }}"
-invokerHostname: "{{ invokerHostnameFromMap | default(inventory_hostname) }}"
-
 invoker:
   dir:
     become: "{{ invoker_dir_become | default(false) }}"
@@ -158,7 +158,7 @@ invoker:
   loglevel: "{{ invoker_loglevel | default(whisk_loglevel) | default('INFO') }}"
   jmxremote:
     jvmArgs: "{% if inventory_hostname in groups['invokers'] %}
-    {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ invokerHostname }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }}
+    {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ ansible_host }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }}
     {% endif %}"
 
 userLogs:
diff --git a/ansible/roles/controller/tasks/deploy.yml b/ansible/roles/controller/tasks/deploy.yml
index 68d7efa572..59f12e594c 100644
--- a/ansible/roles/controller/tasks/deploy.yml
+++ b/ansible/roles/controller/tasks/deploy.yml
@@ -91,7 +91,7 @@
 - name: add additional jvm params if jmxremote is enabled
   when: jmx.enabled
   set_fact:
-    controller_args: "{{ controller.arguments }} {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ inventory_hostname }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortController + (controller_index | int) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortController + (controller_index | int) }}"
+    controller_args: "{{ controller.arguments }} {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ ansible_host }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortController + (controller_index | int) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortController + (controller_index | int) }}"
 
 - name: create seed nodes list
   set_fact:
diff --git a/ansible/roles/kafka/tasks/clean.yml b/ansible/roles/kafka/tasks/clean.yml
index b9d593381c..d86d0f02d1 100644
--- a/ansible/roles/kafka/tasks/clean.yml
+++ b/ansible/roles/kafka/tasks/clean.yml
@@ -16,3 +16,8 @@
     keep_volumes: False
     state: absent
   ignore_errors: True
+
+- name: remove kafka conf directory
+  file:
+    path: "{{ kafka.confdir }}/kafka{{ groups['kafkas'].index(inventory_hostname) }}"
+    state: absent
diff --git a/ansible/roles/kafka/tasks/deploy.yml b/ansible/roles/kafka/tasks/deploy.yml
index f7c3270711..d5c3c65925 100644
--- a/ansible/roles/kafka/tasks/deploy.yml
+++ b/ansible/roles/kafka/tasks/deploy.yml
@@ -18,6 +18,51 @@
     src: "files/{{ kafka.ssl.keystore.name }}"
     dest: "{{ config_root_dir }}/kafka/certs"
 
+- name: ensure kafka config directory is created with permissions
+  file:
+    path: "{{ kafka.confdir }}/kafka{{ groups['kafkas'].index(inventory_hostname) }}"
+    state: directory
+    mode: 0777
+
+- name: copy jmxremote password file
+  when: jmx.enabled
+  template:
+    src: "jmxremote.password.j2"
+    dest: "{{ kafka.confdir }}/kafka{{ groups['kafkas'].index(inventory_hostname) }}/jmxremote.password"
+    mode: 0600
+
+- name: copy jmxremote access file
+  when: jmx.enabled
+  template:
+    src: "jmxremote.access.j2"
+    dest: "{{ kafka.confdir }}/kafka{{ groups['kafkas'].index(inventory_hostname) }}/jmxremote.access"
+    mode: 0600
+
+- name: "prepare the volume_dir"
+  set_fact:
+    volume_dir: ["{{ config_root_dir }}/kafka/certs:/config"]
+
+- name: "add additional jmx volume"
+  when: jmx.enabled
+  set_fact:
+    volume_dir: "{{ volume_dir }} + [ \"{{ kafka.confdir }}/kafka{{ groups['kafkas'].index(inventory_hostname) }}:/root\" ]"
+
+- name: prepare kafka ports
+  set_fact:
+    ports_to_expose: ["{{ kafka.port + groups['kafkas'].index(inventory_hostname) }}:{{ kafka.port + groups['kafkas'].index(inventory_hostname) }}", "{{ kafka.advertisedPort + groups['kafkas'].index(inventory_hostname) }}:{{ kafka.advertisedPort + groups['kafkas'].index(inventory_hostname) }}"]
+
+- name: expose additional ports if jmxremote is enabled
+  when: jmx.enabled
+  set_fact:
+    ports_to_expose: "{{ ports_to_expose }} + [ \"{{ jmx.basePortKafka + groups['kafkas'].index(inventory_hostname) }}:{{ jmx.basePortKafka + groups['kafkas'].index(inventory_hostname) }}\" ] + [ \"{{ jmx.rmiBasePortKafka + groups['kafkas'].index(inventory_hostname) }}:{{ jmx.rmiBasePortKafka + groups['kafkas'].index(inventory_hostname) }}\" ]"
+
+- name: add kafka jmx env vars if jmxremote is enabled
+  when: jmx.enabled
+  set_fact:
+    kafka_jmx_env_vars:
+      "KAFKA_JMX_OPTS": "{{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ ansible_host }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortKafka + groups['kafkas'].index(inventory_hostname) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortKafka + groups['kafkas'].index(inventory_hostname) }}"
+      "JMX_PORT": "{{ jmx.basePortKafka + groups['kafkas'].index(inventory_hostname) }}"
+
 - name: add kafka default env vars
   set_fact:
     kafka_env_vars:
@@ -67,6 +112,11 @@
   set_fact:
     kafka_env_vars: "{{ kafka_env_vars | combine(kafka_non_ssl_vars) }}"
 
+- name: "join kafka jmx env vars if jmxremote is enabled"
+  when: jmx.enabled
+  set_fact:
+    kafka_env_vars: "{{ kafka_env_vars | combine(kafka_jmx_env_vars) }}"
+
 - name: (re)start kafka
   vars:
     zookeeper_idx: "{{ groups['kafkas'].index(inventory_hostname) % (groups['zookeepers'] | length) }}"
@@ -77,11 +127,8 @@
     recreate: true
     restart_policy: "{{ docker.restart.policy }}"
     env: "{{ kafka_env_vars }}"
-    ports:
-      - "{{ kafka.port + groups['kafkas'].index(inventory_hostname) }}:{{ kafka.port + groups['kafkas'].index(inventory_hostname) }}"
-      - "{{ kafka.advertisedPort  + groups['kafkas'].index(inventory_hostname) }}:{{ kafka.advertisedPort + groups['kafkas'].index(inventory_hostname) }}"
-    volumes:
-      - "{{ config_root_dir }}/kafka/certs:/config"
+    ports: "{{ ports_to_expose }}"
+    volumes: "{{ volume_dir }}"
 
 - name: wait until the kafka server started up
   shell: docker logs kafka{{ groups['kafkas'].index(inventory_hostname) }}


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services