You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by kt...@apache.org on 2021/11/01 13:50:19 UTC
[fluo-muchos] branch main updated: Add optional Application
Insights on azure (#414)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/main by this push:
new 7decbf3 Add optional Application Insights on azure (#414)
7decbf3 is described below
commit 7decbf3069e592414cfd184b98e424a503c09b6e
Author: Brian Loss <br...@apache.org>
AuthorDate: Mon Nov 1 09:50:13 2021 -0400
Add optional Application Insights on azure (#414)
This PR allows for the Azure Application Insights Java agent to be
configured to run with the manager and tablet servers. When specified,
this will create an Application Insights object along with the cluster,
and metrics will be sent there. The default configuration sends
micrometer metrics, but the user could configure differently (e.g., to
send certain JMX metrics).
* Create workbook if either oms integration or app insights are used.
* Create application insights resource as part of azure cluster setup
* Configure Accumulo to run the application insights Java agent
* Clean up parameter to azureDeployLinuxCounters.json template to remove
duplication of the creation of the log analytics workspace name.
* Save the log analytics workspace resource ID in muchos.props along
with the workspace ID and key, since this value is needed to set up
the application insights resource.
* Fix CI failure--missing newline at EOF.
* PR feedback
* Activate insights agent on the gc process
* Improve regex when az_appinsights_connection_string
is substituted in muchos.props.
* Ensure az_appinsights_connection_string is added after
az_app_insights_version in muchos.props, or after the
[azure] section header if az_app_insights_version isn't
there.
---
README.md | 4 ++
ansible/azure_terminate.yml | 14 +++++-
ansible/common.yml | 2 +
ansible/roles/accumulo/templates/accumulo-env.sh | 18 ++++++++
..._common.yml => application_insights_common.yml} | 28 +++---------
.../azure/tasks/create_application_insights.yml | 52 ++++++++++++++++++++++
.../roles/azure/tasks/create_log_analytics_ws.yml | 22 +++++++--
.../roles/azure/tasks/log_analytics_ws_common.yml | 12 ++---
ansible/roles/azure/tasks/main.yml | 10 +++--
.../azure/templates/azureDeployLinuxCounters.json | 36 +++++++--------
.../common/tasks/azure_application_insights.yml | 22 +++++++++
.../tasks/templates/applicationinsights.json | 14 ++++++
ansible/roles/proxy/tasks/download.yml | 32 +++++++++----
conf/muchos.props.example | 14 +++++-
lib/muchos/config/azure.py | 26 +++++++++++
15 files changed, 239 insertions(+), 67 deletions(-)
diff --git a/README.md b/README.md
index 14b9be9..babde0e 100644
--- a/README.md
+++ b/README.md
@@ -190,6 +190,10 @@ Under the `azure` section, edit following values as per your configuration:
[Create Log Analytics workspace](https://docs.microsoft.com/en-us/azure/azure-monitor/learn/quick-create-workspace).
[Create and Share dashboards](https://docs.microsoft.com/en-us/azure/azure-portal/azure-portal-dashboards).
[Azure Monitor Workbooks](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/workbooks-overview).
+* `az_use_app_insights` to configure an Azure Application Insights with your setup, and activate the application insights
+ [Java agent](https://docs.microsoft.com/en-us/azure/azure-monitor/app/java-in-process-agent) with the manager and tablet
+ servers. Customize [applicationinsights.json](./ansible/roles/common/tasks/templates/applicationinsights.json) to meet
+ your needs before executing muchos setup.
Please refer to the [muchos.props] example for the full list of Azure-specific configurations - some of which have supplementary comments.
diff --git a/ansible/azure_terminate.yml b/ansible/azure_terminate.yml
index b7edc43..2771abf 100644
--- a/ansible/azure_terminate.yml
+++ b/ansible/azure_terminate.yml
@@ -20,16 +20,26 @@
tasks:
- block:
- import_tasks: roles/azure/tasks/log_analytics_ws_common.yml
+ - import_tasks: roles/azure/tasks/application_insights_common.yml
+
+ - name: Delete application insights
+ azure_rm_resource:
+ resource_group: "{{ resource_group }}"
+ provider: insights
+ resource_type: components
+ resource_name: "{{ app_insights_name }}"
+ api_version: '2020-02-02'
+ state: absent
- name: Delete workbook
azure_rm_resource:
resource_group: "{{ resource_group }}"
provider: insights
resource_type: workbooks
- resource_name: "{{ workbook_exists|join('') }}"
+ resource_name: "{{ workbook_name }}"
api_version: '2020-02-12'
state: absent
- when: workbook_exists is defined
+ when: workbook_exists
- name: Delete dashboard
azure_rm_resource:
diff --git a/ansible/common.yml b/ansible/common.yml
index 74c753f..02649b1 100644
--- a/ansible/common.yml
+++ b/ansible/common.yml
@@ -41,6 +41,8 @@
when: az_oms_integration_needed
- import_tasks: roles/common/tasks/azure_oms_selinux.yml
when: az_oms_integration_needed
+ - import_tasks: roles/common/tasks/azure_application_insights.yml
+ when: az_use_app_insights
when: cluster_type == 'azure'
- import_tasks: roles/common/tasks/ec2.yml
diff --git a/ansible/roles/accumulo/templates/accumulo-env.sh b/ansible/roles/accumulo/templates/accumulo-env.sh
index ea225c1..1200c51 100755
--- a/ansible/roles/accumulo/templates/accumulo-env.sh
+++ b/ansible/roles/accumulo/templates/accumulo-env.sh
@@ -34,6 +34,12 @@ export ACCUMULO_OTHER_OPTS="-Xmx256m -Xms64m"
export ACCUMULO_KILL_CMD='kill -9 %p'
export NUM_TSERVERS=1
export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1}
+{% if cluster_type == 'azure' and az_use_app_insights %}
+# Activate the Application Insights agent
+export ACCUMULO_TSERVER_OPTS="$ACCUMULO_TSERVER_OPTS -javaagent:{{ az_app_insights_home }}/appinsights-agent.jar"
+export ACCUMULO_MASTER_OPTS="$ACCUMULO_MASTER_OPTS -javaagent:{{ az_app_insights_home }}/appinsights-agent.jar"
+export ACCUMULO_GC_OPTS="$ACCUMULO_GC_OPTS -javaagent:{{ az_app_insights_home }}/appinsights-agent.jar"
+{% endif %}
{% else %}
@@ -121,6 +127,18 @@ case "$cmd" in
true
;;
esac
+{% if cluster_type == 'azure' and az_use_app_insights %}
+
+# Enable the application insights agent for tablet servers, manager, and gc processes
+case "$cmd" in
+ tserver|master|manager|gc)
+ JAVA_OPTS=("${JAVA_OPTS[@]}" '-javaagent:{{ az_app_insights_home }}/appinsights-agent.jar')
+ ;;
+ *)
+ true
+ ;;
+esac
+{% endif %}
export JAVA_OPTS
export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1}
diff --git a/ansible/roles/azure/tasks/log_analytics_ws_common.yml b/ansible/roles/azure/tasks/application_insights_common.yml
similarity index 51%
copy from ansible/roles/azure/tasks/log_analytics_ws_common.yml
copy to ansible/roles/azure/tasks/application_insights_common.yml
index 10ff083..70d913c 100644
--- a/ansible/roles/azure/tasks/log_analytics_ws_common.yml
+++ b/ansible/roles/azure/tasks/application_insights_common.yml
@@ -16,34 +16,16 @@
# limitations under the License.
#
#
-# This file is imported in create_log_analytics_ws.yml and terminate_cluster.yml
+# This file is imported in create_app_insights.yml and azure_terminate.yml
#
-- name: Generate name for Workspace, Dashboard and Workbook
+- name: Generate name for Application Insights
shell: set -o pipefail && echo -n {{ resource_group + vmss_name + location }} | md5sum | tr -cd "[:alnum:]" | cut -c 1-48
args:
executable: bash
+ changed_when: true
register: monitor_name
-- name: Set name for log analytics workspace
+- name: Set fact for Application Insights name
set_fact:
- log_workspace_name: "{{ monitor_name.stdout + '-la' }}"
-
-- name: Set name for dashboard
- set_fact:
- dashboard_name: "{{ monitor_name.stdout + '-db' }}"
-
-- name: Set name for workbook
- set_fact:
- workbook_name: "{{ monitor_name.stdout + '-wb' }}"
-
-- name: Query all the resources in the resource group
- azure_rm_resource_info:
- resource_group: "{{ resource_group }}"
- resource_type: resources
- register: rgfacts
-
-- name: Retrieve workbook name
- set_fact:
- workbook_exists: "{{ rgfacts.response|selectattr('type', 'equalto', 'microsoft.insights/workbooks')|map(attribute='name')|list }}"
- when: workbook_name in rgfacts.response|map(attribute='tags')|selectattr('hidden-title','defined')|map(attribute='hidden-title')|list
+ app_insights_name: "{{ monitor_name.stdout + '-insights' }}"
diff --git a/ansible/roles/azure/tasks/create_application_insights.yml b/ansible/roles/azure/tasks/create_application_insights.yml
new file mode 100644
index 0000000..5a47433
--- /dev/null
+++ b/ansible/roles/azure/tasks/create_application_insights.yml
@@ -0,0 +1,52 @@
+---
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# These Ansible tasks only run on the client machine where Muchos runs
+# At a high level, the various sections in this file do the following:
+# 1. Create an Azure Application Insights resource.
+# 2. Update the az_appinsights_connection_string property in muchos.props
+
+- import_tasks: log_analytics_ws_common.yml
+- import_tasks: application_insights_common.yml
+
+- name: Create Application Insights resource using REST API
+ azure_rm_resource:
+ resource_group: "{{ resource_group }}"
+ provider: insights
+ resource_type: components
+ resource_name: "{{ app_insights_name }}"
+ api_version: "2020-02-02"
+ idempotency: yes
+ state: present
+ body:
+ location: "{{ location }}"
+ kind: java
+ properties:
+ ApplicationId: "{{ app_insights_name }}"
+ Application_Type: other
+ Flow_Type: Bluefield
+ Request_Source: rest
+ WorkspaceResourceId: "{{ az_logs_resource_id if az_logs_resource_id else az_logs_ws_resource_id }}"
+ register: app_insights
+
+- name: Update az_appinsights_connection_string in muchos.props
+ lineinfile:
+ path: "{{ deploy_path }}/conf/muchos.props"
+ regexp: '^\s*[#]?\s*az_appinsights_connection_string\s*='
+ line: "az_appinsights_connection_string = {{ app_insights.response.properties.ConnectionString }}"
+ insertafter: '\[azure\]|az_app_insights_version\s*='
diff --git a/ansible/roles/azure/tasks/create_log_analytics_ws.yml b/ansible/roles/azure/tasks/create_log_analytics_ws.yml
index b7dbbc3..d7114b8 100644
--- a/ansible/roles/azure/tasks/create_log_analytics_ws.yml
+++ b/ansible/roles/azure/tasks/create_log_analytics_ws.yml
@@ -20,7 +20,7 @@
# At a high level, the various sections in this file do the following:
# 1. Create Log Analytics workspace
# 2. Create dashboard and workbook using json templates
-# 3. Update az_logs_id and az_logs_key in muchos.props
+# 3. Update az_logs_resource_id, az_logs_id, and az_logs_key in muchos.props
#
#
@@ -32,8 +32,8 @@
state: present
location: "{{ location }}"
parameters:
- deployment-prefix:
- value: "{{ monitor_name.stdout }}"
+ la-workspace-name:
+ value: "{{ log_workspace_name }}"
template: "{{ lookup('file', 'roles/azure/templates/azureDeployLinuxCounters.json') }}"
- name: Gather information about log analytics workspace
@@ -46,6 +46,10 @@
until: logs_workspace.workspaces|map(attribute='customer_id')|list is defined
register: logs_workspace
+- name: Restrieve the workspace resource ID
+ set_fact:
+ az_logs_ws_resource_id: "{{ logs_workspace.workspaces|map(attribute='id')|list|join('') }}"
+
- name: Retrieve the workspace ID
set_fact:
az_logs_ws_id: "{{ logs_workspace.workspaces|map(attribute='customer_id')|list|join('') }}"
@@ -54,17 +58,26 @@
set_fact:
az_logs_ws_key: "{{ logs_workspace.workspaces|map(attribute='shared_keys')|map(attribute='primary_shared_key')|list|join('') }}"
+- name: Update az_logs_resource_id in muchos.props
+ lineinfile:
+ path: "{{ deploy_path }}/conf/muchos.props"
+ regexp: '^az_logs_resource_id\s*=\s*|^[#]az_logs_resource_id\s*=\s*'
+ line: "az_logs_resource_id = {{ az_logs_ws_resource_id }}"
+ insertafter: '\[azure\]'
+
- name: Update az_logs_id in muchos.props
lineinfile:
path: "{{ deploy_path }}/conf/muchos.props"
regexp: '^az_logs_id\s*=\s*|^[#]az_logs_id\s*=\s*'
line: "az_logs_id = {{ az_logs_ws_id }}"
+ insertafter: '^az_logs_resource_id'
- name: Update az_logs_key in muchos.props
lineinfile:
path: "{{ deploy_path }}/conf/muchos.props"
regexp: '^az_logs_key\s*=\s*|^[#]az_logs_key\s*=\s*'
line: "az_logs_key = {{ az_logs_ws_key }}"
+ insertafter: '^az_logs_id'
- name: Create Dashboard
azure_rm_deployment:
@@ -77,6 +90,7 @@
DashboardName:
value: "{{ dashboard_name }}"
template: "{{ lookup('file', 'roles/azure/templates/dashboardTemplate.json') }}"
+ when: az_oms_integration_needed and not dashboard_exists
- name: Create Workbook
azure_rm_deployment:
@@ -89,4 +103,4 @@
workbookDisplayName:
value: "{{ workbook_name }}"
template: "{{ lookup('file', 'roles/azure/templates/workbookTemplate.json') }}"
- when: workbook_exists is undefined
+ when: az_oms_integration_needed and not workbook_exists
diff --git a/ansible/roles/azure/tasks/log_analytics_ws_common.yml b/ansible/roles/azure/tasks/log_analytics_ws_common.yml
index 10ff083..84ca59f 100644
--- a/ansible/roles/azure/tasks/log_analytics_ws_common.yml
+++ b/ansible/roles/azure/tasks/log_analytics_ws_common.yml
@@ -23,6 +23,7 @@
shell: set -o pipefail && echo -n {{ resource_group + vmss_name + location }} | md5sum | tr -cd "[:alnum:]" | cut -c 1-48
args:
executable: bash
+ changed_when: false
register: monitor_name
- name: Set name for log analytics workspace
@@ -33,9 +34,9 @@
set_fact:
dashboard_name: "{{ monitor_name.stdout + '-db' }}"
-- name: Set name for workbook
+- name: Set display name for workbook
set_fact:
- workbook_name: "{{ monitor_name.stdout + '-wb' }}"
+ workbook_title: "{{ monitor_name.stdout + '-wb' }}"
- name: Query all the resources in the resource group
azure_rm_resource_info:
@@ -43,7 +44,8 @@
resource_type: resources
register: rgfacts
-- name: Retrieve workbook name
+- name: Retrieve workbook and dashboard info (name, existance)
set_fact:
- workbook_exists: "{{ rgfacts.response|selectattr('type', 'equalto', 'microsoft.insights/workbooks')|map(attribute='name')|list }}"
- when: workbook_name in rgfacts.response|map(attribute='tags')|selectattr('hidden-title','defined')|map(attribute='hidden-title')|list
+ workbook_name: "{{ rgfacts.response|selectattr('type', 'equalto', 'microsoft.insights/workbooks')|selectattr('tags.hidden-title', 'equalto', workbook_title)|map(attribute='name')|join('') }}"
+ workbook_exists: "{{ rgfacts.response|selectattr('type', 'equalto', 'microsoft.insights/workbooks')|map(attribute='tags')|selectattr('hidden-title','equalto', workbook_title)|length > 0 }}"
+ dashboard_exists: "{{ rgfacts.response|selectattr('type', 'equalto', 'Microsoft.Portal/dashboards')|selectattr('name', 'equalto', dashboard_name)|length > 0 }}"
diff --git a/ansible/roles/azure/tasks/main.yml b/ansible/roles/azure/tasks/main.yml
index bd0e447..0a62898 100644
--- a/ansible/roles/azure/tasks/main.yml
+++ b/ansible/roles/azure/tasks/main.yml
@@ -22,9 +22,9 @@
quiet: yes
that: ansible_version.full is version('2.9.0', '>=')
fail_msg: "Ansible 2.9 or above is required for using Azure ADLS Gen2 in Muchos.
- Please use the requirements.txt file to install the currently supported
- version of Ansible, and also execute the ./scripts/install-ansible-for-azure
- script to install the Ansible collection for Azure and other pre-requisites."
+ Please use the requirements.txt file to install the currently supported
+ version of Ansible, and also execute the ./scripts/install-ansible-for-azure
+ script to install the Ansible collection for Azure and other pre-requisites."
when: use_adlsg2
- import_tasks: create_common_resources.yml
@@ -40,4 +40,6 @@
- import_tasks: assign_msi_multiple_vmss.yml
when: use_multiple_vmss and use_adlsg2
- import_tasks: create_log_analytics_ws.yml
- when: az_oms_integration_needed and (az_logs_id is not defined or (not az_logs_id) or az_logs_id == None)
+ when: (az_oms_integration_needed or az_use_app_insights) and (az_logs_id is not defined or (not az_logs_id) or az_logs_id == None)
+- import_tasks: create_application_insights.yml
+ when: az_use_app_insights
diff --git a/ansible/roles/azure/templates/azureDeployLinuxCounters.json b/ansible/roles/azure/templates/azureDeployLinuxCounters.json
index dbbe253..36d378c 100644
--- a/ansible/roles/azure/templates/azureDeployLinuxCounters.json
+++ b/ansible/roles/azure/templates/azureDeployLinuxCounters.json
@@ -2,11 +2,10 @@
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
- "deployment-prefix": {
+ "la-workspace-name": {
"type": "string",
- "defaultValue": "[parameters('deployment-prefix')]",
"metadata": {
- "description": "Deployment prefix. E.g. Organization-DeploymentType"
+ "description": "Log analytics workspace name"
}
},
"service-tier": {
@@ -31,7 +30,6 @@
}
},
"variables": {
- "la-workspace-name": "[concat(parameters('deployment-prefix'), '-la')]",
"log-analytics-search-version": 1
},
"resources": [
@@ -39,7 +37,7 @@
"apiVersion": "2017-03-15-preview",
"type": "Microsoft.OperationalInsights/workspaces",
"location": "[parameters('location')]",
- "name": "[variables('la-workspace-name')]",
+ "name": "[parameters('la-workspace-name')]",
"properties": {
"features": {
"searchVersion": "[variables('log-analytics-search-version')]"
@@ -54,7 +52,7 @@
"type": "datasources",
"name": "sampleSyslog1",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxSyslog",
"properties": {
@@ -81,9 +79,9 @@
{
"apiVersion": "2015-11-01-preview",
"type": "datasources",
- "name": "sampleSyslog2",
+ "name": "sampleSyslog2",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxSyslog",
"properties": {
@@ -101,7 +99,7 @@
"type": "datasources",
"name": "sampleSyslogCollection1",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxSyslogCollection",
"properties": {
@@ -113,7 +111,7 @@
"type": "datasources",
"name": "sampleLinuxPerf1",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxPerformanceObject",
"properties": {
@@ -137,7 +135,7 @@
"counterName": "Disk Writes/sec"
}
],
- "objectName": "Logical Disk",
+ "objectName": "Logical Disk",
"instanceName": "*",
"intervalSeconds": 60
}
@@ -147,7 +145,7 @@
"type": "datasources",
"name": "sampleLinuxPerf2",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxPerformanceObject",
"properties": {
@@ -172,7 +170,7 @@
"type": "datasources",
"name": "sampleLinuxPerf3",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxPerformanceObject",
"properties": {
@@ -200,7 +198,7 @@
"type": "datasources",
"name": "sampleLinuxPerf4",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxPerformanceObject",
"properties": {
@@ -231,7 +229,7 @@
"type": "datasources",
"name": "sampleLinuxPerfCollection1",
"dependsOn": [
- "[concat('Microsoft.OperationalInsights/workspaces/', variables('la-workspace-name'))]"
+ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('la-workspace-name'))]"
],
"kind": "LinuxPerformanceCollection",
"properties": {
@@ -244,7 +242,7 @@
"outputs": {
"la-resourceId": {
"type": "string",
- "value": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('la-workspace-name'))]"
+ "value": "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('la-workspace-name'))]"
},
"la-workspace-resourceGroup": {
"type": "string",
@@ -252,15 +250,15 @@
},
"la-workspace-name": {
"type": "string",
- "value": "[variables('la-workspace-name')]"
+ "value": "[parameters('la-workspace-name')]"
},
"la-workspace-id": {
"type": "string",
- "value": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', variables('la-workspace-name')), '2015-03-20').customerId]"
+ "value": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', parameters('la-workspace-name')), '2015-03-20').customerId]"
},
"la-primary-shared-key": {
"type": "string",
- "value": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', variables('la-workspace-name')), '2015-03-20').primarySharedKey]"
+ "value": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', parameters('la-workspace-name')), '2015-03-20').primarySharedKey]"
}
}
}
diff --git a/ansible/roles/common/tasks/azure_application_insights.yml b/ansible/roles/common/tasks/azure_application_insights.yml
new file mode 100644
index 0000000..c57b2e3
--- /dev/null
+++ b/ansible/roles/common/tasks/azure_application_insights.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "create application insights dir"
+ file: state=directory mode=0755 path={{ az_app_insights_home }}
+- name: "copy application insights agent jar"
+ copy: src={{ tarballs_dir }}/appinsights-agent.jar dest={{ az_app_insights_home }}/appinsights-agent.jar mode=0644
+- name: "copy application insights configuration"
+ template: src=applicationinsights.json dest={{ az_app_insights_home }}/applicationinsights.json mode=0644
diff --git a/ansible/roles/common/tasks/templates/applicationinsights.json b/ansible/roles/common/tasks/templates/applicationinsights.json
new file mode 100644
index 0000000..624a77e
--- /dev/null
+++ b/ansible/roles/common/tasks/templates/applicationinsights.json
@@ -0,0 +1,14 @@
+{
+ "connectionString": "{{ az_appinsights_connection_string }}",
+ "role": {
+ "name": "{{ 'manager' if (inventory_hostname in groups.accumulomaster) else 'tserver' }}"
+ },
+ "instrumentation": {
+ "micrometer": {
+ "enabled": true
+ }
+ },
+ "preview": {
+ "metricIntervalSeconds": 10
+ }
+}
diff --git a/ansible/roles/proxy/tasks/download.yml b/ansible/roles/proxy/tasks/download.yml
index 798b665..d15c20d 100644
--- a/ansible/roles/proxy/tasks/download.yml
+++ b/ansible/roles/proxy/tasks/download.yml
@@ -39,12 +39,26 @@
mode: 0644
when: hadoop_major_version == '3' and java_product_version == 11
-- name: "download omsagent installer to proxy"
- shell: set -o pipefail && eval $(curl -s https://raw.githubusercontent.com/Microsoft/OMS-Agent-for-Linux/master/installer/scripts/onboard_agent.sh | grep -e 'GITHUB_RELEASE_X64=' -e 'BUNDLE_X64=') && wget ${GITHUB_RELEASE_X64}${BUNDLE_X64} -O {{ tarballs_dir }}/omsagent.x64.sh
- args:
- executable: bash
- register: result
- until: result is not failed
- retries: 3
- delay: 2
- when: cluster_type == 'azure' and az_oms_integration_needed
+- block:
+ - name: "download omsagent installer to proxy"
+ shell: set -o pipefail && eval $(curl -s https://raw.githubusercontent.com/Microsoft/OMS-Agent-for-Linux/master/installer/scripts/onboard_agent.sh | grep -e 'GITHUB_RELEASE_X64=' -e 'BUNDLE_X64=') && wget ${GITHUB_RELEASE_X64}${BUNDLE_X64} -O {{ tarballs_dir }}/omsagent.x64.sh
+ args:
+ executable: bash
+ register: result
+ until: result is not failed
+ retries: 3
+ delay: 2
+ when: az_oms_integration_needed
+
+ - name: "check if Application Insights Agent jar was uploaded to proxy"
+ stat: path={{ tarballs_dir }}/appinsights-agent.jar
+ register: appinsights
+ when: az_use_app_insights
+ - name: "download Application Insights Agent jar to proxy"
+ get_url: url=https://github.com/microsoft/ApplicationInsights-Java/releases/download/{{ az_app_insights_version }}/applicationinsights-agent-{{ az_app_insights_version }}.jar dest={{ tarballs_dir }}/appinsights-agent.jar force=no
+ register: gresult
+ until: "'OK' in gresult.msg or 'file already exists' in gresult.msg"
+ retries: 3
+ when: az_use_app_insights and not appinsights.stat.exists
+ when: cluster_type == 'azure'
+
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index f46e54b..b78ed27 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -195,12 +195,24 @@ principal_id =
#azure_fileshare_username = fs_username
#azure_fileshare_password = fs_password
# Optional integration with Azure Log Analytics
-# Workspace ID and key must be updated to enable this.
+# A workspace will be created and az_logs_resource_id, az_logs_id, az_logs_key
+# will be updated here by the launch command. If you wish to use an existing
+# workspace in another resource group, then fill in all 3 of these values before
+# running launch.
# For details on how to get a workspace ID and key, see the link below
# https://docs.microsoft.com/en-us/azure/azure-monitor/learn/quick-collect-linux-computer#obtain-workspace-id-and-key
az_oms_integration_needed = False
+#az_logs_resource_id =
#az_logs_id =
#az_logs_key =
+# Set to true to enable Azure Application Insights integration
+# An Application Insights instance will be created by the launch command and its
+# connection string will be updated here. If you wish to use a different
+# Application Insights instance, then set az_appinsights_connection_string to
+# its connection string before invoking launch.
+az_use_app_insights = False
+az_app_insights_version = 3.2.1
+#az_appinsights_connection_string =
[existing]
# Root of data dirs
diff --git a/lib/muchos/config/azure.py b/lib/muchos/config/azure.py
index 1bb0acc..6833426 100644
--- a/lib/muchos/config/azure.py
+++ b/lib/muchos/config/azure.py
@@ -202,6 +202,11 @@ class AzureDeployConfig(BaseConfig):
def omsIntegrationNeeded(self):
return self.getboolean("azure", "az_oms_integration_needed")
+ @ansible_host_var(name="az_logs_resource_id")
+ @default(None)
+ def logs_resource_id(self):
+ return self.get("azure", "az_logs_resource_id")
+
@ansible_host_var(name="az_logs_id")
@default(None)
def logs_id(self):
@@ -212,6 +217,27 @@ class AzureDeployConfig(BaseConfig):
def logs_key(self):
return self.get("azure", "az_logs_key")
+ @ansible_host_var(name="az_use_app_insights")
+ @default(False)
+ @is_valid(is_in([True, False]))
+ def az_use_app_insights(self):
+ return self.getboolean("azure", "az_use_app_insights")
+
+ @ansible_host_var(name="az_appinsights_connection_string")
+ @default(None)
+ def az_appinsights_connection_string(self):
+ return self.get("azure", "az_appinsights_connection_string")
+
+ @ansible_host_var(name="az_app_insights_version")
+ @default("3.2.1")
+ def az_app_insights_version(self):
+ return self.getboolean("azure", "az_app_insights_version")
+
+ @ansible_host_var(name="az_app_insights_home")
+ @default("{{ install_dir }}/appinsights-{{ az_app_insights_version }}")
+ def az_app_insights_home(self):
+ return self.getboolean("azure", "az_app_insights_home")
+
@ansible_host_var(name="use_adlsg2")
@is_valid(is_in([True, False]))
@default(False)