You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "Jyotirmoy Sinha (Jira)" <ji...@apache.org> on 2023/09/27 07:25:00 UTC
[jira] [Updated] (HDDS-9356) Ozone debug chunkinfo shows incorrect block path for some nodes in a phatcat cluster
[ https://issues.apache.org/jira/browse/HDDS-9356?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jyotirmoy Sinha updated HDDS-9356:
----------------------------------
Description:
Steps :
* Create volume/bucket/key
* Execute 'ozone debug chunkInfo volume/bucket/key'
Expected output - Block file path should be correct for all nodes
Observed output - Block file path is incorrect for one node. All paths are same (in same data disks) in all the nodes, but in actual one of the nodes have the block path in different data disk
Console output -
{code:java}
# ozone debug chunkinfo vol1/buck1/key1
23/09/26 22:07:41 WARN impl.MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-xceiverclientmetrics.properties,hadoop-metrics2.properties
23/09/26 22:07:42 INFO impl.MetricsSystemImpl: Scheduled Metric snapshot period at 10 second(s).
23/09/26 22:07:42 INFO impl.MetricsSystemImpl: XceiverClientMetrics metrics system started
{
"KeyLocations": [
[
{
"Datanode-HostName": "Host1",
"Datanode-IP": "ip1",
"Container-ID": 8026,
"Block-ID": 111677748019213027,
"Locations": {
"files": [
"/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
],
"pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
}
},
{
"Datanode-HostName": "Host2",
"Datanode-IP": "ip2",
"Container-ID": 8026,
"Block-ID": 111677748019213027,
"Locations": {
"files": [
"/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
],
"pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
}
},
{
"Datanode-HostName": "Host3",
"Datanode-IP": "ip3",
"Container-ID": 8026,
"Block-ID": 111677748019213027,
"Locations": {
"files": [
"/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
],
"pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
}
}
]
]
} {code}
Actual path in node *host2* -
{code:java}
[host2]# ls /data/3/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026
chunks metadata {code}
It is in disk 3 instead of disk 2 (other nodes are in disk 2)
Disks structure for all nodes -
{code:java}
# df -kh
Filesystem Size Used Avail Use% Mounted on
devtmpfs 63G 0 63G 0% /dev
tmpfs 63G 0 63G 0% /dev/shm
tmpfs 63G 1.8G 62G 3% /run
tmpfs 63G 0 63G 0% /sys/fs/cgroup
/dev/sda1 745G 26G 719G 4% /
/dev/sdd1 1.9T 9.4G 1.9T 1% /data/3
/dev/sdb1 1.9T 1.4G 1.9T 1% /data/1
/dev/sdc1 1.9T 13G 1.9T 1% /data/2
/dev/sda3 1.1T 2.9G 1.1T 1% /var
/dev/shm 63G 14M 63G 1% /tmp
tmpfs 13G 0 13G 0% /run/user/1573
cm_processes 63G 44M 63G 1% /run/cloudera-scm-agent/process {code}
Phatcat clusters have 3 data disk partitions configured.
was:
Steps :
* Create volume/bucket/key
* Execute 'ozone debug chunkInfo volume/bucket/key'
Expected output - Block file path should be correct for all nodes
Observed output - Block file path is incorrect for one node. All paths are same (in same data disks) in all the nodes, but in actual one of the nodes have the block path in different data disk
Console output -
{code:java}
[root@vc0719 ~]# ozone debug chunkinfo vol1/buck1/key1
23/09/26 22:07:41 WARN impl.MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-xceiverclientmetrics.properties,hadoop-metrics2.properties
23/09/26 22:07:42 INFO impl.MetricsSystemImpl: Scheduled Metric snapshot period at 10 second(s).
23/09/26 22:07:42 INFO impl.MetricsSystemImpl: XceiverClientMetrics metrics system started
{
"KeyLocations": [
[
{
"Datanode-HostName": "vc0729.halxg.cloudera.com",
"Datanode-IP": "10.17.213.39",
"Container-ID": 8026,
"Block-ID": 111677748019213027,
"Locations": {
"files": [
"/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
],
"pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
}
},
{
"Datanode-HostName": "vc0728.halxg.cloudera.com",
"Datanode-IP": "10.17.213.38",
"Container-ID": 8026,
"Block-ID": 111677748019213027,
"Locations": {
"files": [
"/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
],
"pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
}
},
{
"Datanode-HostName": "vc0719.halxg.cloudera.com",
"Datanode-IP": "10.17.213.29",
"Container-ID": 8026,
"Block-ID": 111677748019213027,
"Locations": {
"files": [
"/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
],
"pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
}
}
]
]
} {code}
Actual path in node *vc0728.halxg.cloudera.com* -
{code:java}
[root@vc0728 ~]# ls /data/3/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026
chunks metadata {code}
It is in disk 3 instead of disk 2 (other nodes are in disk 2)
Disks structure for all nodes -
{code:java}
[root@vc0719 ~]# df -kh
Filesystem Size Used Avail Use% Mounted on
devtmpfs 63G 0 63G 0% /dev
tmpfs 63G 0 63G 0% /dev/shm
tmpfs 63G 1.8G 62G 3% /run
tmpfs 63G 0 63G 0% /sys/fs/cgroup
/dev/sda1 745G 26G 719G 4% /
/dev/sdd1 1.9T 9.4G 1.9T 1% /data/3
/dev/sdb1 1.9T 1.4G 1.9T 1% /data/1
/dev/sdc1 1.9T 13G 1.9T 1% /data/2
/dev/sda3 1.1T 2.9G 1.1T 1% /var
/dev/shm 63G 14M 63G 1% /tmp
tmpfs 13G 0 13G 0% /run/user/1573
cm_processes 63G 44M 63G 1% /run/cloudera-scm-agent/process {code}
Phatcat clusters have 3 data disk partitions configured.
> Ozone debug chunkinfo shows incorrect block path for some nodes in a phatcat cluster
> ------------------------------------------------------------------------------------
>
> Key: HDDS-9356
> URL: https://issues.apache.org/jira/browse/HDDS-9356
> Project: Apache Ozone
> Issue Type: Bug
> Components: Ozone CLI
> Reporter: Jyotirmoy Sinha
> Priority: Major
>
> Steps :
> * Create volume/bucket/key
> * Execute 'ozone debug chunkInfo volume/bucket/key'
> Expected output - Block file path should be correct for all nodes
> Observed output - Block file path is incorrect for one node. All paths are same (in same data disks) in all the nodes, but in actual one of the nodes have the block path in different data disk
> Console output -
> {code:java}
> # ozone debug chunkinfo vol1/buck1/key1
> 23/09/26 22:07:41 WARN impl.MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-xceiverclientmetrics.properties,hadoop-metrics2.properties
> 23/09/26 22:07:42 INFO impl.MetricsSystemImpl: Scheduled Metric snapshot period at 10 second(s).
> 23/09/26 22:07:42 INFO impl.MetricsSystemImpl: XceiverClientMetrics metrics system started
> {
> "KeyLocations": [
> [
> {
> "Datanode-HostName": "Host1",
> "Datanode-IP": "ip1",
> "Container-ID": 8026,
> "Block-ID": 111677748019213027,
> "Locations": {
> "files": [
> "/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
> ],
> "pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
> }
> },
> {
> "Datanode-HostName": "Host2",
> "Datanode-IP": "ip2",
> "Container-ID": 8026,
> "Block-ID": 111677748019213027,
> "Locations": {
> "files": [
> "/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
> ],
> "pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
> }
> },
> {
> "Datanode-HostName": "Host3",
> "Datanode-IP": "ip3",
> "Container-ID": 8026,
> "Block-ID": 111677748019213027,
> "Locations": {
> "files": [
> "/data/2/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026/chunks/111677748019213027.block"
> ],
> "pipelineID": "d2923f6a-4b40-4d9a-9aaa-337511de9345"
> }
> }
> ]
> ]
> } {code}
> Actual path in node *host2* -
> {code:java}
> [host2]# ls /data/3/hadoop-ozone/datanode/data/hdds/CID-f23bf195-38c8-4562-bf8a-f15d9f52dc5c/current/containerDir15/8026
> chunks metadata {code}
> It is in disk 3 instead of disk 2 (other nodes are in disk 2)
> Disks structure for all nodes -
> {code:java}
> # df -kh
> Filesystem Size Used Avail Use% Mounted on
> devtmpfs 63G 0 63G 0% /dev
> tmpfs 63G 0 63G 0% /dev/shm
> tmpfs 63G 1.8G 62G 3% /run
> tmpfs 63G 0 63G 0% /sys/fs/cgroup
> /dev/sda1 745G 26G 719G 4% /
> /dev/sdd1 1.9T 9.4G 1.9T 1% /data/3
> /dev/sdb1 1.9T 1.4G 1.9T 1% /data/1
> /dev/sdc1 1.9T 13G 1.9T 1% /data/2
> /dev/sda3 1.1T 2.9G 1.1T 1% /var
> /dev/shm 63G 14M 63G 1% /tmp
> tmpfs 13G 0 13G 0% /run/user/1573
> cm_processes 63G 44M 63G 1% /run/cloudera-scm-agent/process {code}
> Phatcat clusters have 3 data disk partitions configured.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org