You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ambari.apache.org by Alejandro Fernandez <af...@hortonworks.com> on 2015/03/27 01:09:45 UTC
Review Request 32557: Full Delete of Host : Switch host-related
tables to
use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/
-----------------------------------------------------------
Review request for Ambari.
Bugs: AMBARI-10168
https://issues.apache.org/jira/browse/AMBARI-10168
Repository: ambari
Description
-------
See p;arent Jira AMBARI-10167
Deleting a host from the UI only removes it from the mapping to a cluster.
The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
This causes two problems.
1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
Expectation in Ambari 2.1.0
Deleting a host should be clean, i.e., it should delete all configuration and historic records.
Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
Tables:
10 tables have foreign keys to the host_name in the hosts table:
1. hostcomponentstate (fixed in this patch)
2. hostcomponentdesiredstate (fixed in this patch)
3. hoststate (fixed in this patch)
4. host_version (future patch)
5. host_role_command (future patch)
6. request_operation_level (future patch)
7. hostconfigmapping (future patch)
8. configgrouphostmapping (future patch)
9. kerberos_principal_host (future patch)
10. clusterhostmapping (already fixed)
Diffs
-----
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
Diff: https://reviews.apache.org/r/32557/diff/
Testing
-------
-
--- Fix hoststate
SELECT COUNT(*) FROM hoststate;
ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hoststate DROP COLUMN host_name;
ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
-- Fix hostcomponentstate
SELECT COUNT(*) FROM hostcomponentstate;
ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentstate DROP COLUMN host_name;
ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
-- Fix hostcomponentdesiredstate
SELECT COUNT(*) FROM hostcomponentdesiredstate;
ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
Thanks,
Alejandro Fernandez
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review77983
-----------------------------------------------------------
ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java
<https://reviews.apache.org/r/32557/#comment126365>
The UpgradeCatalog210Test file was becoming spaghetti, so I created this interface so it's easier to break up the verification based on the section/feature.
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
<https://reviews.apache.org/r/32557/#comment126366>
It's much cleaner with just 2 lines.
- Alejandro Fernandez
On March 27, 2015, 12:10 a.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 27, 2015, 12:10 a.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests are in-progress.
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Jonathan Hurley <jh...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78661
-----------------------------------------------------------
Ship it!
Ship It!
- Jonathan Hurley
On April 1, 2015, 9:54 p.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated April 1, 2015, 9:54 p.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java 1f3c09a
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java 3691af2
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java fcf4f51
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog151.java afe27b5
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java a1f11d8
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog161.java 4ab0cb6
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java d3b062a
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java 23ed67d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql 7593e07
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 0562bc6
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql 44fde5f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 74ea215
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql 2a7d3f9
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java 3093c2f
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java c2ab35f
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java ba19154
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 47cadf3
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b0c8fd9
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> The most recent unit test run failed on 3 unrelated python tests,
> ERROR: test_remove_repo_redhat (TestRepositoryResource.TestRepositoryResource)
> FAIL: test_setup_suse (TestMonitorWebserverResource.TestMonitorWebserverResource)
> FAIL: test_stop_suse (TestMonitorWebserverResource.TestMonitorWebserverResource)
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/
-----------------------------------------------------------
(Updated April 2, 2015, 1:54 a.m.)
Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
Changes
-------
Unit tests passed on relevant tests, but failed on 3 python tests that are unrelated.
Bugs: AMBARI-10168
https://issues.apache.org/jira/browse/AMBARI-10168
Repository: ambari
Description
-------
See p;arent Jira AMBARI-10167
Deleting a host from the UI only removes it from the mapping to a cluster.
The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
This causes two problems.
1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
Expectation in Ambari 2.1.0
Deleting a host should be clean, i.e., it should delete all configuration and historic records.
Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
Tables:
10 tables have foreign keys to the host_name in the hosts table:
1. hostcomponentstate (fixed in this patch)
2. hostcomponentdesiredstate (fixed in this patch)
3. hoststate (fixed in this patch)
4. host_version (future patch)
5. host_role_command (future patch)
6. request_operation_level (future patch)
7. hostconfigmapping (future patch)
8. configgrouphostmapping (future patch)
9. kerberos_principal_host (future patch)
10. clusterhostmapping (already fixed)
Diffs
-----
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java 1f3c09a
ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java 3691af2
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java fcf4f51
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog151.java afe27b5
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java a1f11d8
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog161.java 4ab0cb6
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java d3b062a
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java 23ed67d
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql 7593e07
ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 0562bc6
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql 44fde5f
ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 74ea215
ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql 2a7d3f9
ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java 3093c2f
ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java c2ab35f
ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java ba19154
ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 47cadf3
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b0c8fd9
Diff: https://reviews.apache.org/r/32557/diff/
Testing (updated)
-------
Unit tests passed,
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Ambari Main ....................................... SUCCESS [4.737s]
[INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
[INFO] Ambari Web ........................................ SUCCESS [38.862s]
[INFO] Ambari Views ...................................... SUCCESS [2.078s]
[INFO] Ambari Admin View ................................. SUCCESS [9.289s]
[INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
[INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
[INFO] Ambari Agent ...................................... SUCCESS [21.818s]
[INFO] Ambari Client ..................................... SUCCESS [0.033s]
[INFO] Ambari Python Client .............................. SUCCESS [0.254s]
[INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
[INFO] Ambari Shell ...................................... SUCCESS [0.032s]
[INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
[INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 35:45.473s
[INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
[INFO] Final Memory: 66M/500M
[INFO] ------------------------------------------------------------------------
The most recent unit test run failed on 3 unrelated python tests,
ERROR: test_remove_repo_redhat (TestRepositoryResource.TestRepositoryResource)
FAIL: test_setup_suse (TestMonitorWebserverResource.TestMonitorWebserverResource)
FAIL: test_stop_suse (TestMonitorWebserverResource.TestMonitorWebserverResource)
I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
-- For Postgres
--- Fix hoststate
SELECT COUNT(*) FROM hoststate;
ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hoststate DROP COLUMN host_name;
ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
-- Fix hostcomponentstate
SELECT COUNT(*) FROM hostcomponentstate;
ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentstate DROP COLUMN host_name;
ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
-- Fix hostcomponentdesiredstate
SELECT COUNT(*) FROM hostcomponentdesiredstate;
ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
Thanks,
Alejandro Fernandez
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/
-----------------------------------------------------------
(Updated April 2, 2015, 1:16 a.m.)
Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
Changes
-------
Rebased patch. Agreed to fix 1.5 - 2.0 UpgradeCatalogs in a separate Jira
Bugs: AMBARI-10168
https://issues.apache.org/jira/browse/AMBARI-10168
Repository: ambari
Description
-------
See p;arent Jira AMBARI-10167
Deleting a host from the UI only removes it from the mapping to a cluster.
The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
This causes two problems.
1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
Expectation in Ambari 2.1.0
Deleting a host should be clean, i.e., it should delete all configuration and historic records.
Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
Tables:
10 tables have foreign keys to the host_name in the hosts table:
1. hostcomponentstate (fixed in this patch)
2. hostcomponentdesiredstate (fixed in this patch)
3. hoststate (fixed in this patch)
4. host_version (future patch)
5. host_role_command (future patch)
6. request_operation_level (future patch)
7. hostconfigmapping (future patch)
8. configgrouphostmapping (future patch)
9. kerberos_principal_host (future patch)
10. clusterhostmapping (already fixed)
Diffs (updated)
-----
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java 1f3c09a
ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java 3691af2
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java fcf4f51
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog151.java afe27b5
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java a1f11d8
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog161.java 4ab0cb6
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java d3b062a
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java 23ed67d
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql 7593e07
ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 0562bc6
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql 44fde5f
ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 74ea215
ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql 2a7d3f9
ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java 3093c2f
ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java c2ab35f
ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java ba19154
ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 47cadf3
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b0c8fd9
Diff: https://reviews.apache.org/r/32557/diff/
Testing
-------
Unit tests passed,
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Ambari Main ....................................... SUCCESS [4.737s]
[INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
[INFO] Ambari Web ........................................ SUCCESS [38.862s]
[INFO] Ambari Views ...................................... SUCCESS [2.078s]
[INFO] Ambari Admin View ................................. SUCCESS [9.289s]
[INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
[INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
[INFO] Ambari Agent ...................................... SUCCESS [21.818s]
[INFO] Ambari Client ..................................... SUCCESS [0.033s]
[INFO] Ambari Python Client .............................. SUCCESS [0.254s]
[INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
[INFO] Ambari Shell ...................................... SUCCESS [0.032s]
[INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
[INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 35:45.473s
[INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
[INFO] Final Memory: 66M/500M
[INFO] ------------------------------------------------------------------------
I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
-- For Postgres
--- Fix hoststate
SELECT COUNT(*) FROM hoststate;
ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hoststate DROP COLUMN host_name;
ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
-- Fix hostcomponentstate
SELECT COUNT(*) FROM hostcomponentstate;
ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentstate DROP COLUMN host_name;
ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
-- Fix hostcomponentdesiredstate
SELECT COUNT(*) FROM hostcomponentdesiredstate;
ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
Thanks,
Alejandro Fernandez
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/
-----------------------------------------------------------
(Updated March 30, 2015, 11:43 p.m.)
Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
Changes
-------
Addressed incompatible UpgradeCatalogs, so Ambari 2.1.0 cannot be used to perform UpgradeCatalogs on versions before 2.1.0
Fixed double quote issue. UQ constraint on the host_name was already added in patch revision #1.
Bugs: AMBARI-10168
https://issues.apache.org/jira/browse/AMBARI-10168
Repository: ambari
Description
-------
See p;arent Jira AMBARI-10167
Deleting a host from the UI only removes it from the mapping to a cluster.
The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
This causes two problems.
1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
Expectation in Ambari 2.1.0
Deleting a host should be clean, i.e., it should delete all configuration and historic records.
Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
Tables:
10 tables have foreign keys to the host_name in the hosts table:
1. hostcomponentstate (fixed in this patch)
2. hostcomponentdesiredstate (fixed in this patch)
3. hoststate (fixed in this patch)
4. host_version (future patch)
5. host_role_command (future patch)
6. request_operation_level (future patch)
7. hostconfigmapping (future patch)
8. configgrouphostmapping (future patch)
9. kerberos_principal_host (future patch)
10. clusterhostmapping (already fixed)
Diffs (updated)
-----
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java 1f3c09a
ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java 3691af2
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java fcf4f51
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog151.java afe27b5
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java a1f11d8
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog161.java 4ab0cb6
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java d3b062a
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql 26ff9bb
ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql 622fca3
ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b0c8fd9
Diff: https://reviews.apache.org/r/32557/diff/
Testing
-------
Unit tests passed,
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Ambari Main ....................................... SUCCESS [4.737s]
[INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
[INFO] Ambari Web ........................................ SUCCESS [38.862s]
[INFO] Ambari Views ...................................... SUCCESS [2.078s]
[INFO] Ambari Admin View ................................. SUCCESS [9.289s]
[INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
[INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
[INFO] Ambari Agent ...................................... SUCCESS [21.818s]
[INFO] Ambari Client ..................................... SUCCESS [0.033s]
[INFO] Ambari Python Client .............................. SUCCESS [0.254s]
[INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
[INFO] Ambari Shell ...................................... SUCCESS [0.032s]
[INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
[INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 35:45.473s
[INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
[INFO] Final Memory: 66M/500M
[INFO] ------------------------------------------------------------------------
I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
-- For Postgres
--- Fix hoststate
SELECT COUNT(*) FROM hoststate;
ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hoststate DROP COLUMN host_name;
ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
-- Fix hostcomponentstate
SELECT COUNT(*) FROM hostcomponentstate;
ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentstate DROP COLUMN host_name;
ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
-- Fix hostcomponentdesiredstate
SELECT COUNT(*) FROM hostcomponentdesiredstate;
ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
Thanks,
Alejandro Fernandez
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Nate Cole <nc...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78251
-----------------------------------------------------------
Ship it!
Just one fix, no need for me to re-review.
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
<https://reviews.apache.org/r/32557/#comment126772>
Double semicolons?
- Nate Cole
On March 27, 2015, 8:45 p.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 27, 2015, 8:45 p.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
> On March 30, 2015, 8:42 p.m., Jonathan Hurley wrote:
> > ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java, lines 105-106
> > <https://reviews.apache.org/r/32557/diff/2/?file=908680#file908680line105>
> >
> > Just a thought; although this should never happen, auto boxing could cause NPEs around this. Should we return 0L instead?
I think it ought to be fine. I only call getHostId() if hostEntity is not null, and this returns a "L"ong type that can be null.
> On March 30, 2015, 8:42 p.m., Jonathan Hurley wrote:
> > ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java, line 793
> > <https://reviews.apache.org/r/32557/diff/2/?file=908686#file908686line793>
> >
> > This might be a problem since the actual DDL hasn't changed yet (that happens in Upgrade210). You're using relationship classes that assume the database structure has already changed.
For now, this explicitly requires upgrading to 2.0.0 first, and then using the 2.1.0 jar to perform the next upgrade. There may be a bug in UpgradeCatalog210 because it uses the newer HostEntity while upgrading the schema that still exists in 2.0.0
- Alejandro
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78252
-----------------------------------------------------------
On March 30, 2015, 11:43 p.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 30, 2015, 11:43 p.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java 1f3c09a
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java 3691af2
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java fcf4f51
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog151.java afe27b5
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java a1f11d8
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog161.java 4ab0cb6
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java d3b062a
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql 26ff9bb
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql 622fca3
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b0c8fd9
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Jonathan Hurley <jh...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78252
-----------------------------------------------------------
I'm mostly concerned about upgrades from before 2.0.0; I flagged an item from UpgradeCatalog150 that could be problematic. If it is, then all of the upgrades before 2.0.0 could have issues since they assume relationships that don't exist yet.
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java
<https://reviews.apache.org/r/32557/#comment126773>
Just a thought; although this should never happen, auto boxing could cause NPEs around this. Should we return 0L instead?
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java
<https://reviews.apache.org/r/32557/#comment126774>
This might be a problem since the actual DDL hasn't changed yet (that happens in Upgrade210). You're using relationship classes that assume the database structure has already changed.
- Jonathan Hurley
On March 27, 2015, 8:45 p.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 27, 2015, 8:45 p.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/
-----------------------------------------------------------
(Updated March 28, 2015, 12:45 a.m.)
Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
Changes
-------
Addressed comments.
Bugs: AMBARI-10168
https://issues.apache.org/jira/browse/AMBARI-10168
Repository: ambari
Description
-------
See p;arent Jira AMBARI-10167
Deleting a host from the UI only removes it from the mapping to a cluster.
The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
This causes two problems.
1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
Expectation in Ambari 2.1.0
Deleting a host should be clean, i.e., it should delete all configuration and historic records.
Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
Tables:
10 tables have foreign keys to the host_name in the hosts table:
1. hostcomponentstate (fixed in this patch)
2. hostcomponentdesiredstate (fixed in this patch)
3. hoststate (fixed in this patch)
4. host_version (future patch)
5. host_role_command (future patch)
6. request_operation_level (future patch)
7. hostconfigmapping (future patch)
8. configgrouphostmapping (future patch)
9. kerberos_principal_host (future patch)
10. clusterhostmapping (already fixed)
Diffs (updated)
-----
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
ambari-server/src/test/java/org/apache/ambari/server/upgrade/SectionDDL.java PRE-CREATION
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
Diff: https://reviews.apache.org/r/32557/diff/
Testing
-------
Unit tests passed,
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Ambari Main ....................................... SUCCESS [4.737s]
[INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
[INFO] Ambari Web ........................................ SUCCESS [38.862s]
[INFO] Ambari Views ...................................... SUCCESS [2.078s]
[INFO] Ambari Admin View ................................. SUCCESS [9.289s]
[INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
[INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
[INFO] Ambari Agent ...................................... SUCCESS [21.818s]
[INFO] Ambari Client ..................................... SUCCESS [0.033s]
[INFO] Ambari Python Client .............................. SUCCESS [0.254s]
[INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
[INFO] Ambari Shell ...................................... SUCCESS [0.032s]
[INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
[INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 35:45.473s
[INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
[INFO] Final Memory: 66M/500M
[INFO] ------------------------------------------------------------------------
I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
-- For Postgres
--- Fix hoststate
SELECT COUNT(*) FROM hoststate;
ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hoststate DROP COLUMN host_name;
ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
-- Fix hostcomponentstate
SELECT COUNT(*) FROM hostcomponentstate;
ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentstate DROP COLUMN host_name;
ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
-- Fix hostcomponentdesiredstate
SELECT COUNT(*) FROM hostcomponentdesiredstate;
ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
Thanks,
Alejandro Fernandez
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78138
-----------------------------------------------------------
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java
<https://reviews.apache.org/r/32557/#comment126619>
I removed the setter, but had to keep the column because it has to coincide with the columns in HostComponentDesiredStateEntityPK.java.
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
<https://reviews.apache.org/r/32557/#comment126620>
Actually, I already added a UNIQUE constraint on the host_name, so it's already indexed.
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
<https://reviews.apache.org/r/32557/#comment126621>
Here's the UNIQUE constraint btw. However, it's missing from the *.sql scripts, so I'll add those too.
- Alejandro Fernandez
On March 27, 2015, 1:23 a.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 27, 2015, 1:23 a.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Nate Cole <nc...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78067
-----------------------------------------------------------
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java
<https://reviews.apache.org/r/32557/#comment126468>
How can this be? We shouldn't be touching these UpgradeCatalog* classes at all. The logic should work just as it did before, with the 210 class doing the fix-up work.
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
<https://reviews.apache.org/r/32557/#comment126469>
Nit: does order really matter here?
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
<https://reviews.apache.org/r/32557/#comment126470>
Yay!
ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java
<https://reviews.apache.org/r/32557/#comment126466>
Although I applaud you using the I-notation for an interface, it's not standard.
- Nate Cole
On March 26, 2015, 9:23 p.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 26, 2015, 9:23 p.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Jonathan Hurley <jh...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78062
-----------------------------------------------------------
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java
<https://reviews.apache.org/r/32557/#comment126454>
Any reason this is still needed if the entity is being returned?
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java
<https://reviews.apache.org/r/32557/#comment126455>
Feels like we should be setting the entity instead of the ID?
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
<https://reviews.apache.org/r/32557/#comment126459>
Now that ID is the PK, why not use that to lookup the host. It will be faster unless you add an index to the hosts table for name.
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
<https://reviews.apache.org/r/32557/#comment126460>
Maybe should consider indexing this unless we want to switch all references over to findById()
- Jonathan Hurley
On March 26, 2015, 9:23 p.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 26, 2015, 9:23 p.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/#review78079
-----------------------------------------------------------
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java
<https://reviews.apache.org/r/32557/#comment126481>
I was preserving as much of the original as possible. I can certainly remove the getter/setter and only leave the entity.
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
<https://reviews.apache.org/r/32557/#comment126482>
The argument to this function is the hostName, and the function constructs the HostComponentState and HostComponentState entities, so I believe using hostDao.findByName() is ok.
Good idea on adding an index on the host_name.
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java
<https://reviews.apache.org/r/32557/#comment126484>
I removed the setHostName() function from the HostComponentState entity, so to keep things compatible, I'm doing a host look-up and setting the HostId. I probably only need to set the HostEntity.
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
<https://reviews.apache.org/r/32557/#comment126487>
Technically no, but I thought it'd be nicer and probably on a max of 1000 rows.
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
<https://reviews.apache.org/r/32557/#comment126489>
Will certainly add an index.
- Alejandro Fernandez
On March 27, 2015, 1:23 a.m., Alejandro Fernandez wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/32557/
> -----------------------------------------------------------
>
> (Updated March 27, 2015, 1:23 a.m.)
>
>
> Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
>
>
> Bugs: AMBARI-10168
> https://issues.apache.org/jira/browse/AMBARI-10168
>
>
> Repository: ambari
>
>
> Description
> -------
>
> See p;arent Jira AMBARI-10167
>
> Deleting a host from the UI only removes it from the mapping to a cluster.
> The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
>
> This causes two problems.
> 1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
> 2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
> Expectation in Ambari 2.1.0
> Deleting a host should be clean, i.e., it should delete all configuration and historic records.
> Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
> Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
>
>
> Tables:
> 10 tables have foreign keys to the host_name in the hosts table:
> 1. hostcomponentstate (fixed in this patch)
> 2. hostcomponentdesiredstate (fixed in this patch)
> 3. hoststate (fixed in this patch)
> 4. host_version (future patch)
> 5. host_role_command (future patch)
> 6. request_operation_level (future patch)
> 7. hostconfigmapping (future patch)
> 8. configgrouphostmapping (future patch)
> 9. kerberos_principal_host (future patch)
> 10. clusterhostmapping (already fixed)
>
>
> Diffs
> -----
>
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
> ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
> ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
> ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
> ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
> ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
> ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
> ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
> ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
> ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
> ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
> ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
> ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
> ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
> ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
> ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
>
> Diff: https://reviews.apache.org/r/32557/diff/
>
>
> Testing
> -------
>
> Unit tests passed,
> [INFO] ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Ambari Main ....................................... SUCCESS [4.737s]
> [INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
> [INFO] Ambari Web ........................................ SUCCESS [38.862s]
> [INFO] Ambari Views ...................................... SUCCESS [2.078s]
> [INFO] Ambari Admin View ................................. SUCCESS [9.289s]
> [INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
> [INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
> [INFO] Ambari Agent ...................................... SUCCESS [21.818s]
> [INFO] Ambari Client ..................................... SUCCESS [0.033s]
> [INFO] Ambari Python Client .............................. SUCCESS [0.254s]
> [INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
> [INFO] Ambari Shell ...................................... SUCCESS [0.032s]
> [INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
> [INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
> [INFO] ------------------------------------------------------------------------
> [INFO] BUILD SUCCESS
> [INFO] ------------------------------------------------------------------------
> [INFO] Total time: 35:45.473s
> [INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
> [INFO] Final Memory: 66M/500M
> [INFO] ------------------------------------------------------------------------
>
> I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
>
>
> -- For Postgres
> --- Fix hoststate
> SELECT COUNT(*) FROM hoststate;
> ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
> UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hoststate DROP COLUMN host_name;
> ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
>
> -- Fix hostcomponentstate
> SELECT COUNT(*) FROM hostcomponentstate;
> ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
> -- Fix hostcomponentdesiredstate
> SELECT COUNT(*) FROM hostcomponentdesiredstate;
> ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
> UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
> ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
> ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
>
>
> Thanks,
>
> Alejandro Fernandez
>
>
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/
-----------------------------------------------------------
(Updated March 27, 2015, 1:23 a.m.)
Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
Changes
-------
Unit tests passed.
Bugs: AMBARI-10168
https://issues.apache.org/jira/browse/AMBARI-10168
Repository: ambari
Description
-------
See p;arent Jira AMBARI-10167
Deleting a host from the UI only removes it from the mapping to a cluster.
The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
This causes two problems.
1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
Expectation in Ambari 2.1.0
Deleting a host should be clean, i.e., it should delete all configuration and historic records.
Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
Tables:
10 tables have foreign keys to the host_name in the hosts table:
1. hostcomponentstate (fixed in this patch)
2. hostcomponentdesiredstate (fixed in this patch)
3. hoststate (fixed in this patch)
4. host_version (future patch)
5. host_role_command (future patch)
6. request_operation_level (future patch)
7. hostconfigmapping (future patch)
8. configgrouphostmapping (future patch)
9. kerberos_principal_host (future patch)
10. clusterhostmapping (already fixed)
Diffs
-----
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
Diff: https://reviews.apache.org/r/32557/diff/
Testing (updated)
-------
Unit tests passed,
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Ambari Main ....................................... SUCCESS [4.737s]
[INFO] Apache Ambari Project POM ......................... SUCCESS [0.031s]
[INFO] Ambari Web ........................................ SUCCESS [38.862s]
[INFO] Ambari Views ...................................... SUCCESS [2.078s]
[INFO] Ambari Admin View ................................. SUCCESS [9.289s]
[INFO] Ambari Metrics Common ............................. SUCCESS [1.487s]
[INFO] Ambari Server ..................................... SUCCESS [34:09.719s]
[INFO] Ambari Agent ...................................... SUCCESS [21.818s]
[INFO] Ambari Client ..................................... SUCCESS [0.033s]
[INFO] Ambari Python Client .............................. SUCCESS [0.254s]
[INFO] Ambari Groovy Client .............................. SUCCESS [9.834s]
[INFO] Ambari Shell ...................................... SUCCESS [0.032s]
[INFO] Ambari Python Shell ............................... SUCCESS [0.044s]
[INFO] Ambari Groovy Shell ............................... SUCCESS [6.661s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 35:45.473s
[INFO] Finished at: Thu Mar 26 17:39:24 PDT 2015
[INFO] Final Memory: 66M/500M
[INFO] ------------------------------------------------------------------------
I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
-- For Postgres
--- Fix hoststate
SELECT COUNT(*) FROM hoststate;
ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hoststate DROP COLUMN host_name;
ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
-- Fix hostcomponentstate
SELECT COUNT(*) FROM hostcomponentstate;
ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentstate DROP COLUMN host_name;
ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
-- Fix hostcomponentdesiredstate
SELECT COUNT(*) FROM hostcomponentdesiredstate;
ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
Thanks,
Alejandro Fernandez
Re: Review Request 32557: Full Delete of Host : Switch host-related
tables
to use host_id instead of host_name column for hoststate, hostcomponentstate,
hostcomponentdesiredstate
Posted by Alejandro Fernandez <af...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/32557/
-----------------------------------------------------------
(Updated March 27, 2015, 12:10 a.m.)
Review request for Ambari, Jonathan Hurley, Nate Cole, and Sid Wagle.
Bugs: AMBARI-10168
https://issues.apache.org/jira/browse/AMBARI-10168
Repository: ambari
Description
-------
See p;arent Jira AMBARI-10167
Deleting a host from the UI only removes it from the mapping to a cluster.
The host still exists at the hosts endpoint, and all of the historic data maintains intact; config overrides, config groups, host state, host version, host role command logs, etc.
This causes two problems.
1. If a host is removed from a cluster and its underlying environment changes (OS, IP, network card, etc), and is re-added to the cluster, then it still maintains the old information that is incorrect.
2. If host x dies, and is removed from the cluster, there is no way for a new host to take over its hostname.
Expectation in Ambari 2.1.0
Deleting a host should be clean, i.e., it should delete all configuration and historic records.
Because renaming a host requires a more complicated workflow, so it will have to be done manually by the user (see instructions in A1).
Lastly, Ambari will require ambari-agent to be stopped before the host can be deleted.
Tables:
10 tables have foreign keys to the host_name in the hosts table:
1. hostcomponentstate (fixed in this patch)
2. hostcomponentdesiredstate (fixed in this patch)
3. hoststate (fixed in this patch)
4. host_version (future patch)
5. host_role_command (future patch)
6. request_operation_level (future patch)
7. hostconfigmapping (future patch)
8. configgrouphostmapping (future patch)
9. kerberos_principal_host (future patch)
10. clusterhostmapping (already fixed)
Diffs
-----
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java 8e92877
ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java 0fb9c59
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java e0f1e9e
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntityPK.java fc92858
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java 0a31b90
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntityPK.java bc103a1
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java 4df5f39
ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostStateEntity.java 5d5a1e8
ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java 1bd2814
ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java 7975f4e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java dd8e33e
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java 98ac89f
ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ba4dd2d
ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql a9cf3e8
ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql 13e8939
ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql cd4427f
ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql 9fe9cd7
ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql a722d2a
ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAOTest.java 7b34827
ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java c27e600
ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java ebf742e
ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java 8ebf2ce
ambari-server/src/test/java/org/apache/ambari/server/upgrade/ISectionDDL.java PRE-CREATION
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog150Test.java af6721c
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java 6fa3e0a
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java 9c39749
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java 07b8410
ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b5db360
Diff: https://reviews.apache.org/r/32557/diff/
Testing (updated)
-------
Unit tests are in-progress.
I tested this by making the DB changes, copying my jar, then hitting the dashboard, API, and restarting services on a cluster.
-- For Postgres
--- Fix hoststate
SELECT COUNT(*) FROM hoststate;
ALTER TABLE hoststate ADD COLUMN host_id BIGINT NULL;
UPDATE hoststate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hoststate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hoststate ADD CONSTRAINT FK_hoststate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hoststate DROP COLUMN host_name;
ALTER TABLE hoststate ADD CONSTRAINT hoststate_pkey PRIMARY KEY (host_id);
-- Fix hostcomponentstate
SELECT COUNT(*) FROM hostcomponentstate;
ALTER TABLE hostcomponentstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentstate ADD CONSTRAINT FK_hostcomponentstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentstate DROP COLUMN host_name;
ALTER TABLE hostcomponentstate ADD CONSTRAINT hostcomponentstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
-- Fix hostcomponentdesiredstate
SELECT COUNT(*) FROM hostcomponentdesiredstate;
ALTER TABLE hostcomponentdesiredstate ADD COLUMN host_id BIGINT NULL;
UPDATE hostcomponentdesiredstate t SET host_id = (SELECT host_id FROM hosts h WHERE h.host_name = t.host_name) WHERE t.host_id IS NULL AND t.host_name IS NOT NULL;
ALTER TABLE hostcomponentdesiredstate ALTER COLUMN host_id SET NOT NULL;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT FK_hostcomponentdesiredstate_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id);
ALTER TABLE hostcomponentdesiredstate DROP COLUMN host_name;
ALTER TABLE hostcomponentdesiredstate ADD CONSTRAINT hostcomponentdesiredstate_pkey PRIMARY KEY (cluster_id, component_name, host_id, service_name);
Thanks,
Alejandro Fernandez