You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by ar...@apache.org on 2020/01/29 19:05:24 UTC
[fluo-muchos] branch master updated: Add conditions for Azure
specific tasks (#315)
This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new a2f06c4 Add conditions for Azure specific tasks (#315)
a2f06c4 is described below
commit a2f06c4ae90a394479a252e249f50033061c5551
Author: Arvind Shyamsundar <ar...@microsoft.com>
AuthorDate: Wed Jan 29 11:05:16 2020 -0800
Add conditions for Azure specific tasks (#315)
* Add conditions for Azure specific tasks
Fixes #314
* Simplify boolean conditions for ADLS Gen2 tasks
Eliminates the extra == True expressions
---
ansible/accumulo.yml | 4 ++--
ansible/roles/accumulo/templates/accumulo-env.sh | 4 ++--
ansible/roles/accumulo/templates/accumulo.properties | 2 +-
ansible/roles/azure/tasks/main.yml | 2 +-
ansible/roles/azure/tasks/terminate_cluster.yml | 4 ++--
ansible/roles/azure/tasks/wipe_adlsg2.yml | 4 ++--
ansible/roles/hadoop/tasks/main.yml | 2 +-
ansible/roles/hadoop/templates/core-site.xml | 2 +-
ansible/roles/hadoop/templates/mapred-site.xml | 2 +-
ansible/roles/hadoop/templates/yarn-site.xml | 2 +-
10 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/ansible/accumulo.yml b/ansible/accumulo.yml
index 2352c85..b31919f 100644
--- a/ansible/accumulo.yml
+++ b/ansible/accumulo.yml
@@ -30,11 +30,11 @@
- hosts: all:!{{ azure_proxy_host }}
tasks:
- import_tasks: roles/accumulo/tasks/add-adlsgen2.yml
- when: accumulo_major_version == '2' and use_adlsg2 == True
+ when: cluster_type == 'azure' and accumulo_major_version == '2' and use_adlsg2
- hosts: accumulomaster[0]
tasks:
- import_tasks: roles/accumulo/tasks/init-adlsgen2.yml
- when: accumulo_major_version == '2' and use_adlsg2 == True
+ when: cluster_type == 'azure' and accumulo_major_version == '2' and use_adlsg2
handlers:
- import_tasks: roles/accumulo/handlers/init-adlsgen2.yml
- hosts: accumulo
diff --git a/ansible/roles/accumulo/templates/accumulo-env.sh b/ansible/roles/accumulo/templates/accumulo-env.sh
index 083007b..71fb224 100755
--- a/ansible/roles/accumulo/templates/accumulo-env.sh
+++ b/ansible/roles/accumulo/templates/accumulo-env.sh
@@ -41,7 +41,7 @@ export HADOOP_HOME={{ hadoop_home }}
export HADOOP_CONF_DIR="$HADOOP_HOME/etc/hadoop"
CLASSPATH="${conf}:${lib}/*:${HADOOP_CONF_DIR}:${ZOOKEEPER_HOME}/*:${HADOOP_HOME}/share/hadoop/client/*"
-{% if use_adlsg2 == True %}
+{% if cluster_type == 'azure' and use_adlsg2 %}
CLASSPATH="${CLASSPATH}:${HADOOP_HOME}/share/hadoop/tools/lib/*"
CLASSPATH="${CLASSPATH}:${HADOOP_HOME}/share/hadoop/common/lib/*"
{% endif %}
@@ -54,7 +54,7 @@ JAVA_OPTS=("${ACCUMULO_JAVA_OPTS[@]}"
'-XX:OnOutOfMemoryError=kill -9 %p'
'-XX:-OmitStackTraceInFastThrow'
'-Djava.net.preferIPv4Stack=true'
-{% if use_adlsg2 == True %}
+{% if cluster_type == 'azure' and use_adlsg2 %}
'-Dorg.wildfly.openssl.path=/usr/lib64'
{% endif %}
"-Daccumulo.native.lib.path=${lib}/native")
diff --git a/ansible/roles/accumulo/templates/accumulo.properties b/ansible/roles/accumulo/templates/accumulo.properties
index eac3ddf..582cb4b 100644
--- a/ansible/roles/accumulo/templates/accumulo.properties
+++ b/ansible/roles/accumulo/templates/accumulo.properties
@@ -43,7 +43,7 @@ tserver.server.threads.minimum=64
## The maximum size for each write-ahead log
tserver.walog.max.size=512M
-{% if use_adlsg2 == True %}
+{% if cluster_type == 'azure' and use_adlsg2 %}
general.volume.chooser=org.apache.accumulo.server.fs.PreferredVolumeChooser
general.custom.volume.preferred.default={{ instance_volumes_preferred }}
general.custom.volume.preferred.logger={{ hdfs_root }}/accumulo
diff --git a/ansible/roles/azure/tasks/main.yml b/ansible/roles/azure/tasks/main.yml
index a846779..37df069 100644
--- a/ansible/roles/azure/tasks/main.yml
+++ b/ansible/roles/azure/tasks/main.yml
@@ -20,4 +20,4 @@
# tasks file for azure
- import_tasks: create_vmss.yml
- import_tasks: create_adlsgen2.yml
- when: use_adlsg2 == True
+ when: use_adlsg2
diff --git a/ansible/roles/azure/tasks/terminate_cluster.yml b/ansible/roles/azure/tasks/terminate_cluster.yml
index 822656d..e1e1e9e 100644
--- a/ansible/roles/azure/tasks/terminate_cluster.yml
+++ b/ansible/roles/azure/tasks/terminate_cluster.yml
@@ -27,7 +27,7 @@
resource_name: "{{ user_assigned_identity if user_assigned_identity !='' else vmss_name + '-ua-msi' }}"
api_version: '2018-11-30'
state: absent
- when: use_adlsg2 == True
+ when: use_adlsg2
- name: Delete ADLS Gen2 storage Account
azure_rm_storageaccount:
@@ -37,7 +37,7 @@
state: absent
loop:
"{{ instance_volumes_preferred.split(',') }}"
- when: use_adlsg2 == True
+ when: use_adlsg2
- name: Delete VM Scale Set
azure_rm_virtualmachinescaleset:
diff --git a/ansible/roles/azure/tasks/wipe_adlsg2.yml b/ansible/roles/azure/tasks/wipe_adlsg2.yml
index 6278adc..47c32d7 100644
--- a/ansible/roles/azure/tasks/wipe_adlsg2.yml
+++ b/ansible/roles/azure/tasks/wipe_adlsg2.yml
@@ -26,7 +26,7 @@
force: yes
loop:
"{{ instance_volumes_preferred.split(',') }}"
- when: cluster_type == 'azure' and use_adlsg2 == True
+ when: cluster_type == 'azure' and use_adlsg2
- name: Create container/Filesystem on ADLS Gen2
azure_rm_storageblob:
@@ -39,4 +39,4 @@
until: result is succeeded and (result.changed == False or (result.changed == True and result.container|length > 0))
loop:
"{{ instance_volumes_preferred.split(',') }}"
- when: cluster_type == 'azure' and use_adlsg2 == True
+ when: cluster_type == 'azure' and use_adlsg2
diff --git a/ansible/roles/hadoop/tasks/main.yml b/ansible/roles/hadoop/tasks/main.yml
index a6733a9..45d72ec 100644
--- a/ansible/roles/hadoop/tasks/main.yml
+++ b/ansible/roles/hadoop/tasks/main.yml
@@ -62,4 +62,4 @@
block: |
export HADOOP_OPTIONAL_TOOLS=hadoop-azure
export HADOOP_OPTS="-Dorg.wildfly.openssl.path=/usr/lib64 ${HADOOP_OPTS}"
- when: hadoop_major_version == '3' and use_adlsg2 == True
+ when: cluster_type == 'azure' and hadoop_major_version == '3' and use_adlsg2
diff --git a/ansible/roles/hadoop/templates/core-site.xml b/ansible/roles/hadoop/templates/core-site.xml
index 85b88a0..583d42e 100644
--- a/ansible/roles/hadoop/templates/core-site.xml
+++ b/ansible/roles/hadoop/templates/core-site.xml
@@ -40,7 +40,7 @@
<value>{{ zookeeper_connect }}</value>
</property>
{% endif %}
-{% if use_adlsg2 == True %}
+{% if cluster_type == 'azure' and use_adlsg2 %}
<property>
<name>fs.azure.account.auth.type</name>
<value>OAuth</value>
diff --git a/ansible/roles/hadoop/templates/mapred-site.xml b/ansible/roles/hadoop/templates/mapred-site.xml
index 90024ad..04c9219 100644
--- a/ansible/roles/hadoop/templates/mapred-site.xml
+++ b/ansible/roles/hadoop/templates/mapred-site.xml
@@ -54,7 +54,7 @@
<value>HADOOP_MAPRED_HOME={{ hadoop_home }}</value>
</property>
{% endif %}
-{% if use_adlsg2 == True %}
+{% if cluster_type == 'azure' and use_adlsg2 %}
<property>
<name>mapreduce.application.classpath</name>
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/common/*,$HADOOP_MAPRED_HOME/share/hadoop/common/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/yarn/*,$HADOOP_MAPRED_HOME/share/hadoop/yarn/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/hdfs/*,$HADOOP_MAPRED_HOME/share/hadoop/hdfs/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/tools/lib/*,${HADOOP_HOME}/share/hadoop/client/*</value>
diff --git a/ansible/roles/hadoop/templates/yarn-site.xml b/ansible/roles/hadoop/templates/yarn-site.xml
index afce378..5d9c4d0 100644
--- a/ansible/roles/hadoop/templates/yarn-site.xml
+++ b/ansible/roles/hadoop/templates/yarn-site.xml
@@ -80,7 +80,7 @@
<name>twill.java.reserved.memory.mb</name>
<value>{{ twill_reserve_mem_mb }}</value>
</property>
- {% if use_adlsg2 == True %}
+ {% if cluster_type == 'azure' and use_adlsg2 %}
<property>
<name>yarn.application.classpath</name>
<value>${HADOOP_HOME}/share/hadoop/tools/lib/*,${HADOOP_HOME}/share/hadoop/hdfs/lib/*,${HADOOP_HOME}/share/hadoop/common/lib/*,${HADOOP_HOME}/share/hadoop/yarn/*,${HADOOP_HOME}/share/hadoop/yarn/lib/*,${HADOOP_HOME}/share/hadoop/hdfs/*,${HADOOP_HOME}/share/hadoop/common/*,${HADOOP_HOME}/share/hadoop/mapreduce/*,${HADOOP_HOME}/share/hadoop/mapreduce/lib/*,${HADOOP_HOME}/share/hadoop/client/*</value>