You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2023/05/09 13:09:56 UTC
[hbase-operator-tools] 06/06: HBASE-27834 Introduce ha-hdfs overlay
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch 27834-introduce-ha-hdfs-overlay
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
commit 4974ad0c5cbfa2d8b0f10e22033a7b5a200be944
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Tue May 2 14:19:04 2023 +0200
HBASE-27834 Introduce ha-hdfs overlay
---
.../overlays/ha-hdfs/dn-statefulset-patch.yaml | 25 ++
.../overlays/ha-hdfs/hdfs-site.xml | 325 +++++++++++++++++++++
.../overlays/ha-hdfs/jn-service.yaml | 124 ++++++++
.../overlays/ha-hdfs/jn-statefulset.yaml | 244 ++++++++++++++++
.../overlays/ha-hdfs/kustomization.yaml | 62 ++++
.../overlays/ha-hdfs/nn-statefulset-patch.yaml | 144 +++++++++
.../overlays_ha-hdfs/00-assert-hdfs.yaml | 42 +++
.../integration/overlays_ha-hdfs/00-kustomize.yaml | 21 ++
.../overlays_ha-hdfs/kustomization.yaml | 22 ++
.../unit/overlays_ha-hdfs/00-assert-hdfs.yaml | 123 ++++++++
.../tests/unit/overlays_ha-hdfs/00-kustomize.yaml | 21 ++
.../tests/unit/overlays_ha-hdfs/kustomization.yaml | 21 ++
12 files changed, 1174 insertions(+)
diff --git a/hbase-kubernetes-deployment/overlays/ha-hdfs/dn-statefulset-patch.yaml b/hbase-kubernetes-deployment/overlays/ha-hdfs/dn-statefulset-patch.yaml
new file mode 100644
index 0000000..3df748a
--- /dev/null
+++ b/hbase-kubernetes-deployment/overlays/ha-hdfs/dn-statefulset-patch.yaml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- op: replace
+ # Using replace. 'add' seems to replace anyways.
+ path: /spec/template/spec/affinity/podAntiAffinity
+ value:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ role: datanode
+ topologyKey: kubernetes.io/hostname
diff --git a/hbase-kubernetes-deployment/overlays/ha-hdfs/hdfs-site.xml b/hbase-kubernetes-deployment/overlays/ha-hdfs/hdfs-site.xml
new file mode 100644
index 0000000..05386b1
--- /dev/null
+++ b/hbase-kubernetes-deployment/overlays/ha-hdfs/hdfs-site.xml
@@ -0,0 +1,325 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <!--https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsBlockPlacementPolicies.html-->
+ <name>dfs.block.replicator.classname</name>
+ <value>org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant</value>
+ </property>
+ <property>
+ <name>dfs.blocksize</name>
+ <value>64m</value>
+ </property>
+ <property>
+ <name>dfs.datanode.address</name>
+ <value>0.0.0.0:9866</value>
+ </property>
+ <property>
+ <name>dfs.datanode.balance.bandwidthPerSec</name>
+ <value>20m</value>
+ </property>
+ <property>
+ <name>dfs.datanode.balance.max.concurrent.moves</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>${env.DATANODE_DATA_DIR}</value>
+ </property>
+ <property>
+ <name>dfs.datanode.failed.volumes.tolerated</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dfs.datanode.du.reserved</name>
+ <value>1073741824</value>
+ </property>
+ <property>
+ <name>dfs.datanode.fileio.profiling.sampling.percentage</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>dfs.datanode.http.address</name>
+ <value>0.0.0.0:9864</value>
+ </property>
+ <property>
+ <name>dfs.datanode.https.address</name>
+ <value>0.0.0.0:9865</value>
+ </property>
+ <property>
+ <name>dfs.datanode.ipc.address</name>
+ <value>0.0.0.0:9867</value>
+ </property>
+ <property>
+ <name>dfs.datanode.max.locked.memory</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dfs.datanode.peer.stats.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.encrypt.data.transfer</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>dfs.encrypt.data.transfer.algorithm</name>
+ <value>rc4</value>
+ </property>
+ <property>
+ <name>dfs.ha.automatic-failover.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.ha.fencing.methods</name>
+ <value>shell(/usr/bin/true)</value>
+ </property>
+ <property>
+ <name>dfs.journalnode.edits.dir</name>
+ <value>${env.JOURNALNODE_DATA_DIR}</value>
+ </property>
+ <property>
+ <name>dfs.journalnode.http-address</name>
+ <value>0.0.0.0:8480</value>
+ </property>
+ <property>
+ <name>dfs.journalnode.https-address</name>
+ <value>0.0.0.0:8481</value>
+ </property>
+ <property>
+ <name>dfs.journalnode.rpc-address</name>
+ <value>0.0.0.0:8485</value>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>64</value>
+ </property>
+ <!--
+ <property>
+ <name>dfs.hosts</name>
+ <value>/tmp/scratch/hosts.json</value>
+ </property>
+ <property>
+ <name>dfs.namenode.hosts.provider.classname</name>
+ <value>org.apache.hadoop.hdfs.server.blockmanagement.CombinedHostFileManager</value>
+ </property>
+ -->
+ <property>
+ <name>dfs.namenode.http-bind-host</name>
+ <value>0.0.0.0</value>
+ </property>
+ <property>
+ <name>dfs.namenode.https-bind-host</name>
+ <value>0.0.0.0</value>
+ </property>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>${env.NAMENODE_DATA_DIR}</value>
+ </property>
+ <property>
+ <name>dfs.namenode.replication.max-streams</name>
+ <value>20</value>
+ </property>
+ <property>
+ <name>dfs.namenode.replication.max-streams-hard-limit</name>
+ <value>40</value>
+ </property>
+ <property>
+ <name>dfs.namenode.replication.min</name>
+ <value>3</value>
+ </property>
+ <property>
+ <name>dfs.namenode.replication.work.multiplier.per.iteration</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>dfs.namenode.safemode.threshold-pct</name>
+ <value>0.9</value>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>64</value>
+ </property>
+ <property>
+ <name>dfs.reformat.disabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>dfs.replication</name>
+ <value>3</value>
+ </property>
+ <property>
+ <name>dfs.replication.max</name>
+ <value>512</value>
+ </property>
+ <property>
+ <name>ipc.8020.callqueue.impl</name>
+ <value>org.apache.hadoop.ipc.FairCallQueue</value>
+ </property>
+ <property>
+ <name>ipc.8020.scheduler.impl</name>
+ <value>org.apache.hadoop.ipc.DecayRpcScheduler</value>
+ </property>
+ <property>
+ <name>zk-dt-secret-manager.zkAuthType</name>
+ <value>digest</value>
+ </property>
+ <property>
+ <name>zk-dt-secret-manager.digest.auth</name>
+ <value>@/etc/hadoop/zookeeper/auth/zk-auth.txt</value>
+ </property>
+ <property>
+ <name>zk-dt-secret-manager.zkConnectionString</name>
+ <value>TODO</value>
+ </property>
+ <property>
+ <name>zk-dt-secret-manager.znodeWorkingPath</name>
+ <value>TODO</value>
+ </property>
+ <property>
+ <name>dfs.client.failover.proxy.provider.hadoop</name>
+ <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+ </property>
+
+ <property>
+ <name>dfs.client.https.keystore.resource</name>
+ <value>ssl-client.xml</value>
+ </property>
+ <property>
+ <name>dfs.client.https.need-auth</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>dfs.http.policy</name>
+ <value>${env.HTTP_POLICY}</value>
+ </property>
+ <property>
+ <name>dfs.https.enable</name>
+ <value>${env.DFS_HTTPS_ENABLE}</value>
+ </property>
+ <property>
+ <name>dfs.https.server.keystore.resource</name>
+ <value>ssl-server.xml</value>
+ </property>
+ <property>
+ <name>dfs.namenode.acls.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <!--From https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsMultihoming.html -->
+ <name>dfs.datanode.use.datanode.hostname</name>
+ <value>true</value>
+ </property>
+ <property>
+ <!--https://log.rowanto.com/posts/why-datanode-is-denied-communication-with-namenode/-->
+ <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>dfs.namenode.shared.edits.dir</name>
+ <value>${env.QJOURNAL}</value>
+ </property>
+ <property>
+ <name>dfs.nameservices</name>
+ <value>${env.HADOOP_SERVICE}</value>
+ </property>
+ <property>
+ <name>dfs.ha.namenodes.hadoop</name>
+ <value>namenode-0,namenode-1,namenode-2</value>
+ </property>
+ <property>
+ <name>dfs.namenode.http-address.hadoop.namenode-0</name>
+ <value>namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9870</value>
+ </property>
+ <property>
+ <name>dfs.namenode.https-address.hadoop.namenode-0</name>
+ <value>namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9871</value>
+ </property>
+ <property>
+ <name>dfs.namenode.rpc-address.hadoop.namenode-0</name>
+ <value>namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8020</value>
+ <description>RPC address that handles all clients requests. In the case of
+ HA/Federation where multiple namenodes exist, the name service id is added
+ to the name e.g. dfs.namenode.rpc-address.ns1 dfs.namenode.rpc-address.EXAMPLENAMESERVICE
+ The value of this property will take the form of nn-host1:rpc-port.
+
+ Uses the value here to find its local name. The value here must be useable
+ making a resolvable inetsocketaddress and then pass the is local test.
+ </description>
+ </property>
+ <property>
+ <name>dfs.namenode.servicerpc-address.hadoop.namenode-0</name>
+ <!--Service name-->
+ <value>namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8022</value>
+ </property>
+ <property>
+ <name>dfs.namenode.lifeline.rpc-address.hadoop.namenode-0</name>
+ <value>namenode-0.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8050</value>
+ </property>
+ <property>
+ <name>dfs.namenode.http-address.hadoop.namenode-1</name>
+ <value>namenode-1.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9870</value>
+ </property>
+ <property>
+ <name>dfs.namenode.https-address.hadoop.namenode-1</name>
+ <value>namenode-1.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9871</value>
+ </property>
+ <property>
+ <name>dfs.namenode.rpc-address.hadoop.namenode-1</name>
+ <value>namenode-1.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8020</value>
+ </property>
+ <property>
+ <name>dfs.namenode.servicerpc-address.hadoop.namenode-1</name>
+ <!--Service name-->
+ <value>namenode-1.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8022</value>
+ </property>
+ <property>
+ <name>dfs.namenode.lifeline.rpc-address.hadoop.namenode-1</name>
+ <value>namenode-1.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8050</value>
+ </property>
+ <property>
+ <name>dfs.namenode.http-address.hadoop.namenode-2</name>
+ <value>namenode-2.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9870</value>
+ </property>
+ <property>
+ <name>dfs.namenode.https-address.${HADOOP_SERVICE}.namenode-2</name>
+ <value>namenode-2.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:9871</value>
+ </property>
+ <property>
+ <name>dfs.namenode.rpc-address.hadoop.namenode-2</name>
+ <value>namenode-2.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8020</value>
+ </property>
+ <property>
+ <name>dfs.namenode.servicerpc-address.hadoop.namenode-2</name>
+ <!--Service nme-->
+ <value>namenode-2.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8022</value>
+ </property>
+ <property>
+ <name>dfs.namenode.lifeline.rpc-address.hadoop.namenode-2</name>
+ <value>namenode-2.${env.HADOOP_SERVICE}.${env.POD_NAMESPACE}.${env.DOMAIN}:8050</value>
+ </property>
+ <property>
+ <name>dfs.blockreport.intervalMsec</name>
+ <value>900000</value>
+ <description>Determines block reporting interval in milliseconds.
+ Report frequently else around recovery storms, the NN gets convinced
+ there is no block space left because of 'scheduled space' reserved.
+ </description>
+ </property>
+</configuration>
diff --git a/hbase-kubernetes-deployment/overlays/ha-hdfs/jn-service.yaml b/hbase-kubernetes-deployment/overlays/ha-hdfs/jn-service.yaml
new file mode 100644
index 0000000..60abb8e
--- /dev/null
+++ b/hbase-kubernetes-deployment/overlays/ha-hdfs/jn-service.yaml
@@ -0,0 +1,124 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Put a Service in front of each journalnode so ip doesn't change on nn.
+# The service is at journal-?.svc.domain. The pod
+# it is proxying will will be at journal-?.${env.HADOOP_SERVICE}.NAMESPACE.svc.domain.
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: journalnode-0
+ labels:
+ # So helm prometheus standalone install will scrape this jn
+ jmxexporter: enabled
+spec:
+ ports:
+ - port: 8480
+ name: http
+ - port: 8481
+ name: https
+ - port: 8485
+ name: rpc
+ - port: 8000
+ name: jmxexporter
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-0
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: journalnode-1
+ labels:
+ # So helm prometheus standalone install will scrape this jn
+ jmxexporter: enabled
+spec:
+ ports:
+ - port: 8480
+ name: http
+ - port: 8481
+ name: https
+ - port: 8485
+ name: rpc
+ - port: 8000
+ name: jmxexporter
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-1
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: journalnode-2
+ labels:
+ # So helm prometheus standalone install will scrape this jn
+ jmxexporter: enabled
+spec:
+ ports:
+ - port: 8480
+ name: http
+ - port: 8481
+ name: https
+ - port: 8485
+ name: rpc
+ - port: 8000
+ name: jmxexporter
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-2
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: journalnode-3
+ labels:
+ # So helm prometheus standalone install will scrape this jn
+ jmxexporter: enabled
+spec:
+ ports:
+ - port: 8480
+ name: http
+ - port: 8481
+ name: https
+ - port: 8485
+ name: rpc
+ - port: 8000
+ name: jmxexporter
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-3
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: journalnode-4
+ labels:
+ # So helm prometheus standalone install will scrape this jn
+ jmxexporter: enabled
+spec:
+ ports:
+ - port: 8480
+ name: http
+ - port: 8481
+ name: https
+ - port: 8485
+ name: rpc
+ - port: 8000
+ name: jmxexporter
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-4
diff --git a/hbase-kubernetes-deployment/overlays/ha-hdfs/jn-statefulset.yaml b/hbase-kubernetes-deployment/overlays/ha-hdfs/jn-statefulset.yaml
new file mode 100644
index 0000000..d9e6182
--- /dev/null
+++ b/hbase-kubernetes-deployment/overlays/ha-hdfs/jn-statefulset.yaml
@@ -0,0 +1,244 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+ name: journalnode
+spec:
+ minAvailable: 1
+ selector:
+ matchLabels:
+ role: journalnode
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: journalnode
+spec:
+ podManagementPolicy: Parallel
+ replicas: 5
+ selector:
+ matchLabels:
+ role: journalnode
+ serviceName: hadoop
+ template:
+ metadata:
+ labels:
+ role: journalnode
+ spec:
+ affinity:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - podAffinityTerm:
+ labelSelector:
+ matchLabels:
+ role: journalnode
+ topologyKey: kubernetes.io/hostname
+ weight: 30
+ containers:
+ - image: hadoop
+ name: journalnode
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ - -c
+ - |-
+ # Shell context so we can pull in the environment variables set in the container and
+ # via the env and envFrom.
+ # See https://stackoverflow.com/questions/57885828/netty-cannot-access-class-jdk-internal-misc-unsafe
+ HADOOP_LOGFILE="hdfs-${HOSTNAME}.log" \
+ HDFS_JOURNALNODE_OPTS=" \
+ -XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \
+ -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \
+ -javaagent:${JMX_PROMETHEUS_JAR}=8000:/tmp/scratch/jmxexporter.yaml \
+ -Djava.security.properties=/tmp/scratch/java.security \
+ -Djava.library.path=${HADOOP_HOME}/lib/native \
+ --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \
+ -Dio.netty.tryReflectionSetAccessible=true \
+ -Xlog:gc:/var/log/hadoop/gc.log:time,uptime:filecount=10,filesize=100M" \
+ hdfs journalnode
+ # For now, just fetch local /jmx
+ # Says kubelet only exposes failures, not success: https://stackoverflow.com/questions/34455040/kubernetes-liveness-probe-logging
+ # Do better. Check this DN successfully registered w/ NN. TODO.
+ livenessProbe:
+ httpGet:
+ # Could look to see if jn is 'formatted': JournalsStatus" : "{\"hadoop\":{\"Formatted\":\"true\"}}""
+ path: /jmx?qry=Hadoop:service=JournalNode,name=JournalNodeInfo
+ port: 8480
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ failureThreshold: 3
+ startupProbe:
+ exec:
+ command:
+ - /bin/bash
+ - -c
+ - |-
+ log="./startProbe.log"
+ probeResult=0
+ if [ ! -z "$( ls -A ${JOURNALNODE_DATA_DIR} )" ]; then
+ # The data dir is not empty. The NN has formatted. The JN has been restarted. In this case, I wanted
+ # to check the logs for ‘Preallocated 1048576 bytes at the end of the edit log (offset 0)‘.
+ # Instances of this message are present when journalnode has successfully opened a journal.
+ # (There may be other indicators: e.g. 'Updating lastPromisedEpoch from 2 to 3 for client /10.244.3.64 ; journal id: hadoop')
+ # Before this point, it will 'JournalOutOfSyncException: Can't write, no segment open ; journal id: hadoop'
+ # which the NN gets and considers a failure; too many replicas in this state and
+ # NN exits. This happens across a rolling restart of JNs. I'd rather hold up the startup
+ # until the journal is open before letting startup proceed. Unfortunately, I have to let
+ # journalnode go so it shows up and joins the cluster... and finds out what Journal to open.
+ # Because of this I can't hold here in startup phase (nor in readiness phase... same issue happens
+ # when readiness holds the JN offline preventing it from joining cluster waiting on 'Preallocated' in
+ # logs). Instead, wait as long as possible, wait till after webserver is up and it is ready listening
+ # before letting startup proceed.
+ echo "`date` ${JOURNALNODE_DATA_DIR} is NOT empty" >> $log
+ grep -q -e "IPC Server listener on .*: starting" ${HADOOP_LOG_DIR}/hdfs*.log
+ probeResult=$?
+ fi
+ # Just fall thought... we are initializing hdfs.
+ echo "`date` probeResult=$probeResult" >> $log
+ exit $probeResult
+ initialDelaySeconds: 1
+ failureThreshold: 30
+ periodSeconds: 10
+ resources:
+ limits:
+ cpu: "1.0"
+ memory: 1.5Gi
+ requests:
+ cpu: "0.5"
+ memory: 1Gi
+ envFrom:
+ - configMapRef:
+ name: environment
+ env:
+ # The 'node' this container is running on, not hdfs namenode.
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ ports:
+ - containerPort: 8480
+ name: http
+ - containerPort: 8481
+ name: https
+ - containerPort: 8480
+ name: jmx
+ - containerPort: 8485
+ name: rpc
+ volumeMounts:
+ - mountPath: /etc/hadoop
+ name: hadoop-configuration
+ - mountPath: /var/log/hadoop
+ name: hadoop-logs
+ - mountPath: /tmp/scratch
+ name: scratch
+ - mountPath: /tmp/scripts
+ name: scripts
+ - mountPath: /data00
+ name: data00
+ initContainers:
+ - image: hadoop
+ name: bootstrapper
+ imagePullPolicy: IfNotPresent
+ command:
+ # Check if we need to cleanup our data dir. Then load up
+ # the certificate, key, and keystores into the /tmp/scratch directory
+ # for use when main container launches.
+ - /bin/bash
+ - -c
+ - |-
+ set -ex
+ mkdir -p ${HADOOP_LOG_DIR} || echo $?
+ chown -R ${USER} ${HADOOP_LOG_DIR}
+ # If format-hdfs configmap present, format.
+ if /tmp/scripts/exists_configmap.sh format-hdfs; then
+ rm -rf ${JOURNALNODE_DATA_DIR}
+ fi
+ mkdir -p ${JOURNALNODE_DATA_DIR}
+ chown -R ${USER} ${JOURNALNODE_DATA_DIR}
+ cp /tmp/global-files/* /tmp/scratch/
+ securityContext:
+ # Run bootstrapper as root so can set ${USER} owner on data volume
+ allowPrivilegeEscalation: false
+ runAsUser: 0
+ resources:
+ requests:
+ cpu: '0.2'
+ memory: 256Mi
+ limits:
+ cpu: '0.5'
+ memory: 512Mi
+ envFrom:
+ - configMapRef:
+ name: environment
+ env:
+ # Used by scripts that run during bootstrap
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ volumeMounts:
+ - mountPath: /data00
+ name: data00
+ - mountPath: /tmp/scripts
+ name: scripts
+ # Scratch dir is a location where init containers place items for later use
+ # by the main containers when they run.
+ - mountPath: /tmp/scratch
+ name: scratch
+ - mountPath: /tmp/global-files
+ name: global-files
+ serviceAccountName: hadoop
+ volumes:
+ - configMap:
+ name: hadoop-configuration
+ name: hadoop-configuration
+ - configMap:
+ name: global-files
+ name: global-files
+ - emptyDir: {}
+ name: hadoop-logs
+ - configMap:
+ name: scripts
+ defaultMode: 0555
+ name: scripts
+ # Scratch dir is a location where init containers place items for later use
+ # by the main containers when they run.
+ - emptyDir: {}
+ name: scratch
+ updateStrategy:
+ type: RollingUpdate
+ volumeClaimTemplates:
+ - metadata:
+ name: data00
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 2Gi
diff --git a/hbase-kubernetes-deployment/overlays/ha-hdfs/kustomization.yaml b/hbase-kubernetes-deployment/overlays/ha-hdfs/kustomization.yaml
new file mode 100644
index 0000000..a33bdf1
--- /dev/null
+++ b/hbase-kubernetes-deployment/overlays/ha-hdfs/kustomization.yaml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+replicas:
+- name: datanode
+ count: 5
+- name: namenode
+ count: 3
+
+commonLabels:
+ app: hadoop
+
+configMapGenerator:
+- name: hadoop-configuration
+ # Put in place HA config in place of base hdfs-site.xml
+ behavior: merge
+ files:
+ - hdfs-site.xml
+- name: environment
+ behavior: merge
+ literals:
+ - DFS_REPLICATION=3
+ # Hard-coded. Five QJMs. Here we point at 5 Service instances defined in adjacent jn-service.yaml.
+ - QJOURNAL="qjournal://journalnode-0:8485;journalnode-1:8485;journalnode-2:8485;journalnode-3:8485;journalnode-4:8485/hadoop"
+
+patches:
+- target:
+ group: "apps"
+ version: v1
+ kind: StatefulSet
+ name: namenode
+ path: nn-statefulset-patch.yaml
+- target:
+ group: "apps"
+ version: v1
+ kind: StatefulSet
+ name: datanode
+ path: dn-statefulset-patch.yaml
+
+components:
+- ../../components/zookeeper/ha-ensemble
+
+resources:
+- jn-statefulset.yaml
+- jn-service.yaml
+- ../hdfs
diff --git a/hbase-kubernetes-deployment/overlays/ha-hdfs/nn-statefulset-patch.yaml b/hbase-kubernetes-deployment/overlays/ha-hdfs/nn-statefulset-patch.yaml
new file mode 100644
index 0000000..ffc74d2
--- /dev/null
+++ b/hbase-kubernetes-deployment/overlays/ha-hdfs/nn-statefulset-patch.yaml
@@ -0,0 +1,144 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- op: add
+ path: /spec/template/spec/containers/-
+ value:
+ image: hadoop
+ name: zkfc
+ command:
+ - /bin/bash
+ - -c
+ - |-
+ # Shell context so we can pull in the environment variables set in the container and
+ # via the env and envFrom.
+ # See https://stackoverflow.com/questions/57885828/netty-cannot-access-class-jdk-internal-misc-unsafe
+ # https://stackoverflow.com/questions/33311585/how-does-hadoop-namenode-failover-process-works
+ # https://github.com/c9n/hadoop/blob/master/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+ HDFS_NAMENODE_OPTS=" \
+ -XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \
+ -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \
+ -Djava.security.properties=/tmp/scratch/java.security \
+ -javaagent:${JMX_PROMETHEUS_JAR}=8000:/tmp/scratch/jmxexporter.yaml \
+ -Djava.library.path=${HADOOP_HOME}/lib/native \
+ --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \
+ -Dio.netty.tryReflectionSetAccessible=true \
+ -Xlog:gc:/var/log/hadoop/gc.log:time,uptime:filecount=10,filesize=100M" \
+ hdfs zkfc
+ resources:
+ requests:
+ cpu: '0.2'
+ memory: 768Mi
+ limits:
+ cpu: '0.5'
+ memory: 1Gi
+ envFrom:
+ - configMapRef:
+ name: environment
+ - configMapRef:
+ name: zookeeper-quorum
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ volumeMounts:
+ - mountPath: /etc/hadoop
+ name: hadoop-configuration
+ - mountPath: /var/log/hadoop
+ name: hadoop-logs
+ - mountPath: /tmp/scratch
+ name: scratch
+ - mountPath: /tmp/scripts
+ name: scripts
+ - mountPath: /etc/hadoop/zookeeper/auth
+ name: zookeeper-credentials
+ readOnly: true
+ # Mount the data00 dir here in case nn doesn't come up.
+ # Doing this, I can look at the data00 if zkfc container is up.
+ - mountPath: /data00
+ name: data00
+- op: add
+ path: /spec/template/spec/initContainers/-
+ value:
+ image: hadoop
+ name: format-zkfc
+ command:
+ # Runs as the image/hdfs user.
+ - /bin/bash
+ - -c
+ - |-
+ set -xe
+ ordinal=$(echo $POD_NAME | sed -e 's/^[^-]*-\(.*\)/\1/')
+ case $ordinal in
+ 0)
+ # Fails if dir exists up in zk already. Ignore
+ hdfs zkfc -formatZK -force -nonInteractive || echo $?
+ ;;
+ *)
+ ;;
+ esac
+ resources:
+ requests:
+ cpu: '0.2'
+ memory: 256Mi
+ limits:
+ cpu: '0.5'
+ memory: 512Mi
+ envFrom:
+ - configMapRef:
+ name: environment
+ - configMapRef:
+ name: zookeeper-quorum
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ volumeMounts:
+ - mountPath: /etc/hadoop
+ name: hadoop-configuration
+ - mountPath: /var/log/hadoop
+ name: hadoop-logs
+ - mountPath: /data00
+ name: data00
+ - mountPath: /etc/hadoop/zookeeper/auth
+ name: zookeeper-credentials
+ readOnly: true
+ # Scratch dir is a location where init containers place items for later use
+ # by the main containers when they run.
+ - mountPath: /tmp/scratch
+ name: scratch
+ #- mountPath: /tmp/hadoop-crt
+ #name: hadoop-crt
+ - mountPath: /tmp/global-files
+ name: global-files
diff --git a/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/00-assert-hdfs.yaml b/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/00-assert-hdfs.yaml
new file mode 100644
index 0000000..3ccbf3c
--- /dev/null
+++ b/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/00-assert-hdfs.yaml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Asserts on the HDFS portion of the deployment.
+#
+---
+# assert that there is a `StatefulSet` named "namenode" that has 3 live instance
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: namenode
+status:
+ availableReplicas: 3
+---
+# assert that there is a `StatefulSet` named "journalnode" that has 5 live instance
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: journalnode
+status:
+ availableReplicas: 5
+---
+# assert that there is a `StatefulSet` named "datanode" that has one 5 instance
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: datanode
+status:
+ availableReplicas: 5
diff --git a/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/00-kustomize.yaml b/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/00-kustomize.yaml
new file mode 100644
index 0000000..b365471
--- /dev/null
+++ b/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/00-kustomize.yaml
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+
+commands:
+- script: ../../bin/kustomize_into_tmpdir.sh
diff --git a/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/kustomization.yaml b/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/kustomization.yaml
new file mode 100644
index 0000000..791d243
--- /dev/null
+++ b/hbase-kubernetes-deployment/tests/integration/overlays_ha-hdfs/kustomization.yaml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../test_base
+- ../../../overlays/ha-hdfs
diff --git a/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/00-assert-hdfs.yaml b/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/00-assert-hdfs.yaml
new file mode 100644
index 0000000..a581a77
--- /dev/null
+++ b/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/00-assert-hdfs.yaml
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Asserts on the HDFS portion of the deployment.
+#
+---
+# assert that there is a `StatefulSet` named "namenode" that:
+# - has 3 replicas
+# - has a container named "zkfc"
+# - an init container named "format-zkfc"
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: namenode
+spec:
+ replicas: 3
+ template:
+ spec:
+ containers:
+ - name: namenode
+ - name: zkfc
+ initContainers:
+ - name: bootstrapper
+ - name: format-hdfs
+ - name: format-zkfc
+---
+# assert that there is a `StatefulSet` named "datanode" that has 5 replicas
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: datanode
+spec:
+ replicas: 5
+---
+# assert that there is a `PodDisruptionBudget` named "journalnode"
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+ name: journalnode
+---
+# assert that there is a `StatefulSet` named "journalnode" that it provides pods labeled role:journalnode
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: journalnode
+spec:
+ template:
+ metadata:
+ labels:
+ role: journalnode
+---
+# assert that there is a `Service` named "journalnode-0" that:
+# - points to pods labeled role:journalnode
+# - points the pod of the same name
+apiVersion: v1
+kind: Service
+metadata:
+ name: journalnode-0
+spec:
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-0
+---
+# assert that there is a `Service` named "journalnode-1" that:
+# - points to pods labeled role:journalnode
+# - points the pod of the same name
+apiVersion: v1
+kind: Service
+metadata:
+ name: journalnode-1
+spec:
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-1
+---
+# assert that there is a `Service` named "journalnode-2" that:
+# - points to pods labeled role:journalnode
+# - points the pod of the same name
+apiVersion: v1
+kind: Service
+metadata:
+ name: journalnode-2
+spec:
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-2
+---
+# assert that there is a `Service` named "journalnode-3" that:
+# - points to pods labeled role:journalnode
+# - points the pod of the same name
+apiVersion: v1
+kind: Service
+metadata:
+ name: journalnode-3
+spec:
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-3
+---
+# assert that there is a `Service` named "journalnode-4" that:
+# - points to pods labeled role:journalnode
+# - points the pod of the same name
+apiVersion: v1
+kind: Service
+metadata:
+ name: journalnode-4
+spec:
+ selector:
+ role: journalnode
+ statefulset.kubernetes.io/pod-name: journalnode-4
diff --git a/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/00-kustomize.yaml b/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/00-kustomize.yaml
new file mode 100644
index 0000000..b365471
--- /dev/null
+++ b/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/00-kustomize.yaml
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+
+commands:
+- script: ../../bin/kustomize_into_tmpdir.sh
diff --git a/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/kustomization.yaml b/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/kustomization.yaml
new file mode 100644
index 0000000..7929c6c
--- /dev/null
+++ b/hbase-kubernetes-deployment/tests/unit/overlays_ha-hdfs/kustomization.yaml
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+ - ../../../overlays/ha-hdfs