You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by ar...@apache.org on 2020/07/14 17:50:13 UTC
[fluo-muchos] branch master updated: Normalize hdfs-site.xml across
HA and non-HA cases (#356)
This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 35f925d Normalize hdfs-site.xml across HA and non-HA cases (#356)
35f925d is described below
commit 35f925d93dc3331f207ec05bf13be0475519541a
Author: Arvind Shyamsundar <ar...@apache.org>
AuthorDate: Tue Jul 14 10:50:08 2020 -0700
Normalize hdfs-site.xml across HA and non-HA cases (#356)
* Explicitly specify the data directory for the non-HA checkpoint
(secondary namenode) to use (in a non-HA config)
* Rearrange some elements in the hdfs-site.xml file in a more logical order
* Use HDFS namespace (using nameservice_id) in non-HA cases as well (HA
config already uses it). This change allows using the nameservice_id
as a stable and simple way to reference namenodes regardless of
whether HA is used / or not
* Configure the use of ConfiguredFailoverProxyProvider in non-HA cases
as well (HA config already used it), so that namespace can be resolved
to physical namenodes in all cases
[Reference from Hadoop docs](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/Federation.html#Federation_Configuration) - though this is a doc for HDFS federation
the specific change in this PR is orthogonal and merely aims to
normalize hdfs-site.xml across both HA / standalone configurations.
---
ansible/roles/hadoop/templates/hdfs-site.xml | 53 +++++++++++++++++-----------
1 file changed, 32 insertions(+), 21 deletions(-)
diff --git a/ansible/roles/hadoop/templates/hdfs-site.xml b/ansible/roles/hadoop/templates/hdfs-site.xml
index ff9fe2b..99fe332 100644
--- a/ansible/roles/hadoop/templates/hdfs-site.xml
+++ b/ansible/roles/hadoop/templates/hdfs-site.xml
@@ -30,6 +30,12 @@
<name>dfs.namenode.name.dir</name>
<value>{{ worker_data_dirs[0] }}/hadoop/name</value>
</property>
+{% if not hdfs_ha %}
+ <property>
+ <name>dfs.namenode.checkpoint.dir</name>
+ <value>{{ worker_data_dirs[0] }}/hadoop/namesecondary</value>
+ </property>
+{% endif%}
<property>
<name>dfs.datanode.data.dir</name>
<value>{% for dir in worker_data_dirs -%}
@@ -37,48 +43,61 @@
{%- if not loop.last -%} , {%- endif -%}
{%- endfor %}</value>
</property>
-{% if not hdfs_ha %}
<property>
- <name>dfs.namenode.secondary.http-address</name>
- <value>{{ groups['namenode'][0] }}:50090</value>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
</property>
<property>
- <name>dfs.namenode.secondary.https-address</name>
- <value>{{ groups['namenode'][0] }}:50091</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
-{% else %}
<property>
<name>dfs.nameservices</name>
<value>{{ nameservice_id }}</value>
</property>
+{% if not hdfs_ha %}
+ <property>
+ <name>dfs.namenode.secondary.http-address.{{ nameservice_id }}</name>
+ <value>{{ groups['namenode'][0] }}:50090</value>
+ </property>
+ <property>
+ <name>dfs.namenode.secondary.https-address.{{ nameservice_id }}</name>
+ <value>{{ groups['namenode'][0] }}:50091</value>
+ </property>
+{% else %}
{% set nn_list = [] %}
{% for item in groups['namenode'] %}{{ nn_list.append('nn' + loop.index|string() ) }}{% endfor %}
<property>
<name>dfs.ha.namenodes.{{ nameservice_id }}</name>
<value>{{ nn_list | join(',') }}</value>
</property>
+{% endif %}
{% for nn_host in groups['namenode'] %}{% set nn_id = 'nn' + loop.index|string() %}
+{% if hdfs_ha %}{% set nnid_suffix = '.' + nn_id %}
+{% else %}{% set nnid_suffix = '' %}
+{% endif %}
<property>
- <name>dfs.namenode.rpc-address.{{ nameservice_id }}.{{ nn_id }}</name>
+ <name>dfs.namenode.rpc-address.{{ nameservice_id }}{{ nnid_suffix }}</name>
<value>{{ nn_host }}:8020</value>
</property>
<property>
- <name>dfs.namenode.http-address.{{ nameservice_id }}.{{ nn_id }}</name>
+ <name>dfs.namenode.http-address.{{ nameservice_id }}{{ nnid_suffix }}</name>
<value>{{ nn_host }}:50070</value>
</property>
<property>
- <name>dfs.namenode.https-address.{{ nameservice_id }}.{{ nn_id }}</name>
+ <name>dfs.namenode.https-address.{{ nameservice_id }}{{ nnid_suffix }}</name>
<value>{{ nn_host }}:50071</value>
</property>
{% endfor %}
<property>
- <name>dfs.namenode.shared.edits.dir</name>
- <value>qjournal://{{ journal_quorum }}/{{ nameservice_id }}</value>
- </property>
- <property>
<name>dfs.client.failover.proxy.provider.{{ nameservice_id }}</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
+{% if hdfs_ha %}
+ <property>
+ <name>dfs.namenode.shared.edits.dir</name>
+ <value>qjournal://{{ journal_quorum }}/{{ nameservice_id }}</value>
+ </property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/usr/bin/true)</value>
@@ -92,12 +111,4 @@
<value>true</value>
</property>
{% endif %}
- <property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- </property>
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
</configuration>