You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "Attila Doroszlai (Jira)" <ji...@apache.org> on 2023/05/05 06:11:00 UTC
[jira] [Reopened] (HDDS-8441) [EC] Unhealthy replica not getting replaced by healthy replica when hdds.datanode.dir path is deleted
[ https://issues.apache.org/jira/browse/HDDS-8441?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Attila Doroszlai reopened HDDS-8441:
------------------------------------
> [EC] Unhealthy replica not getting replaced by healthy replica when hdds.datanode.dir path is deleted
> -----------------------------------------------------------------------------------------------------
>
> Key: HDDS-8441
> URL: https://issues.apache.org/jira/browse/HDDS-8441
> Project: Apache Ozone
> Issue Type: Bug
> Reporter: Arun Sarin
> Priority: Major
>
> Scenario Repro :
> # Close container of a key
> # Simulate one unhealthy replica in above container by deleting hdds.datanode.dir
> # New replica is added for the unhealthy replica but after successful addition the unhealthy replica is not removed
> Final state of container - 2 replicas on replica index 1
> {code:java}
> {
> "containerInfo": {
> "state": "CLOSED",
> "replicationConfig": {
> "data": 3,
> "parity": 2,
> "ecChunkSize": 1048576,
> "codec": "RS",
> "replicationType": "EC",
> "requiredNodes": 5
> },
> "usedBytes": 54525952,
> "numberOfKeys": 1,
> "lastUsed": "2023-04-11T11:10:42.412Z",
> "stateEnterTime": "2023-04-11T10:54:42.254Z",
> "owner": "om3",
> "containerID": 8001,
> "deleteTransactionId": 0,
> "sequenceId": 0,
> "deleted": "False",
> "open": "False"
> },
> "pipeline": {
> "id": {
> "id": "10064036-6251-40a9-b4fa-e1edeae388e5"
> },
> "replicationConfig": {
> "data": 3,
> "parity": 2,
> "ecChunkSize": 1048576,
> "codec": "RS",
> "replicationType": "EC",
> "requiredNodes": 5
> },
> "nodesInOrder": [
> {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> }
> ],
> "creationTimestamp": "2023-04-11T11:10:42.344Z",
> "closed": "False",
> "allocationTimeout": "False",
> "healthy": "True",
> "pipelineState": "ALLOCATED",
> "nodeSet": [
> {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> }
> ],
> "leaderNode": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "firstNode": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "closestNode": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "nodes": [
> {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> }
> ],
> "empty": "False",
> "type": "EC",
> "open": "False"
> },
> "replicas": [
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "placeOfBirth": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 5
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> "placeOfBirth": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 1
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> "placeOfBirth": "74078315-7c10-412b-97a2-f520ad02b91e",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 4
> },
> {
> "containerID": 8001,
> "state": "UNHEALTHY",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> "placeOfBirth": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 1
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> "placeOfBirth": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 53477376,
> "replicaIndex": 3
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> },
> "placeOfBirth": "1548294e-6375-41e1-9324-0646a8163294",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 53477376,
> "replicaIndex": 2
> }
> ],
> "Datanodes": [
> "hostname-2.hostname.root.hwx.site",
> "hostname-4.hostname.root.hwx.site",
> "hostname-7.hostname.root.hwx.site",
> "hostname-9.hostname.root.hwx.site",
> "hostname-3.hostname.root.hwx.site",
> "hostname-6.hostname.root.hwx.site"
> ]
> } {code}
> Error Message from TC:
> {code:java}
> replace_status = Ozone.check_replica_present_in_container(dn_i, container_list[0], False)
> > assert replace_status, "Unhealthy replica not replace by healthy replica"E AssertionError: Unhealthy replica not replace by healthy replica
> E assert False{code}
> DN didn't realised that the container is not there any more (eg the full volume has failed)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org