You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2015/12/29 18:45:46 UTC
ambari git commit: AMBARI-14519. GET calls to /requests become very
slow after installing packages on 900 node cluster. (mpapirkovskyy)
Repository: ambari
Updated Branches:
refs/heads/trunk 9022bfb11 -> 8a748ffd9
AMBARI-14519. GET calls to /requests become very slow after installing packages on 900 node cluster. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a748ffd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a748ffd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a748ffd
Branch: refs/heads/trunk
Commit: 8a748ffd9829cc25350ea79ed9f44abe0a98b012
Parents: 9022bfb
Author: Myroslav Papirkovskyy <mp...@hortonworks.com>
Authored: Tue Dec 29 16:45:15 2015 +0200
Committer: Myroslav Papirkovskyy <mp...@hortonworks.com>
Committed: Tue Dec 29 19:45:27 2015 +0200
----------------------------------------------------------------------
.../ambari/server/topology/AmbariContext.java | 4 +++
.../ambari/server/topology/HostRequest.java | 4 +++
.../ambari/server/topology/LogicalRequest.java | 35 +++++++++++++++++---
.../server/upgrade/UpgradeCatalog221.java | 6 +++-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 5 +++
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 5 +++
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 5 +++
.../resources/Ambari-DDL-Postgres-CREATE.sql | 5 +++
.../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql | 5 +++
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 5 +++
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 5 +++
.../server/upgrade/UpgradeCatalog221Test.java | 31 +++++++++++++++++
12 files changed, 110 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 0a2bd18..6bfee93 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -145,6 +145,10 @@ public class AmbariContext {
return getController().getActionManager().getTaskById(id);
}
+ public Collection<HostRoleCommand> getPhysicalTasks(Collection<Long> ids) {
+ return getController().getActionManager().getTasks(ids);
+ }
+
public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType) {
Stack stack = topology.getBlueprint().getStack();
createAmbariClusterResource(clusterName, stack.getName(), stack.getVersion(), securityType);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
index 7f178f4..440638c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
@@ -387,6 +387,10 @@ public class HostRequest implements Comparable<HostRequest> {
return physicalTasks.get(logicalTaskId);
}
+ public Map<Long, Long> getPhysicalTaskMapping() {
+ return new HashMap<>(physicalTasks);
+ }
+
//todo: since this is used to determine equality, using hashCode() isn't safe as it can return the same
//todo: value for 2 unequal requests
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
index fb6b774..bd9f2e0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
@@ -195,6 +195,7 @@ public class LogicalRequest extends Request {
}
// currently we are just returning all stages for all requests
+ //TODO technically StageEntity is simply a container for HostRequest info with additional redundant transformations
public Collection<StageEntity> getStageEntities() {
Collection<StageEntity> stages = new ArrayList<StageEntity>();
for (HostRequest hostRequest : allHostRequests) {
@@ -232,7 +233,32 @@ public class LogicalRequest extends Request {
public Map<Long, HostRoleCommandStatusSummaryDTO> getStageSummaries() {
Map<Long, HostRoleCommandStatusSummaryDTO> summaryMap = new HashMap<Long, HostRoleCommandStatusSummaryDTO>();
- for (StageEntity stage : getStageEntities()) {
+ Map<Long, Collection<HostRoleCommand>> stageTasksMap = new HashMap<>();
+
+ Map<Long, Long> taskToStageMap = new HashMap<>();
+
+
+ for (HostRequest hostRequest : getHostRequests()) {
+ Map<Long, Long> physicalTaskMapping = hostRequest.getPhysicalTaskMapping();
+ Collection<Long> stageTasks = physicalTaskMapping.values();
+ for (Long stageTask : stageTasks) {
+ taskToStageMap.put(stageTask, hostRequest.getStageId());
+ }
+ }
+
+ Collection<HostRoleCommand> physicalTasks = topology.getAmbariContext().getPhysicalTasks(taskToStageMap.keySet());
+
+ for (HostRoleCommand physicalTask : physicalTasks) {
+ Long stageId = taskToStageMap.get(physicalTask.getTaskId());
+ Collection<HostRoleCommand> stageTasks = stageTasksMap.get(stageId);
+ if (stageTasks == null) {
+ stageTasks = new ArrayList<>();
+ stageTasksMap.put(stageId, stageTasks);
+ }
+ stageTasks.add(physicalTask);
+ }
+
+ for (Long stageId : stageTasksMap.keySet()) {
//Number minStartTime = 0;
//Number maxEndTime = 0;
int aborted = 0;
@@ -248,7 +274,7 @@ public class LogicalRequest extends Request {
int skippedFailed = 0;
//todo: where does this logic belong?
- for (HostRoleCommandEntity task : stage.getHostRoleCommands()) {
+ for (HostRoleCommand task : stageTasksMap.get(stageId)) {
HostRoleStatus taskStatus = task.getStatus();
switch (taskStatus) {
@@ -291,12 +317,13 @@ public class LogicalRequest extends Request {
}
HostRoleCommandStatusSummaryDTO stageSummary = new HostRoleCommandStatusSummaryDTO(
- stage.isSkippable() ? 1 : 0, 0, 0, stage.getStageId(), aborted, completed, failed,
+ 0, 0, 0, stageId, aborted, completed, failed,
holding, holdingFailed, holdingTimedout, inProgress, pending, queued, timedout,
skippedFailed);
- summaryMap.put(stage.getStageId(), stageSummary);
+ summaryMap.put(stageId, stageSummary);
}
+
return summaryMap;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
index b33d44e..c4216bd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
@@ -109,7 +109,11 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
@Override
protected void executeDDLUpdates() throws AmbariException, SQLException {
- //To change body of implemented methods use File | Settings | File Templates.
+ // indices to improve request status calc performance
+ dbAccessor.createIndex("idx_stage_request_id", "stage", "request_id");
+ dbAccessor.createIndex("idx_hrc_request_id", "host_role_command", "request_id");
+ dbAccessor.createIndex("idx_rsc_request_id", "role_success_criteria", "request_id");
+
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
index 0e534b7..d93a3c2 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
@@ -653,6 +653,11 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
--------altering tables by creating unique constraints----------
ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (version_tag, type_name, cluster_id);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
index fcfcb93..aa8ced1 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
@@ -660,6 +660,11 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
-- altering tables by creating unique constraints----------
ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
index 4305a4c..b534344 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
@@ -649,6 +649,11 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
--------altering tables by creating unique constraints----------
ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
index 32fd2a6..941fc6e 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
@@ -653,6 +653,11 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
--------altering tables by creating unique constraints----------
ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name, version_tag);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
index 7894c43..dd517f8 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
@@ -734,6 +734,11 @@ CREATE TABLE ambari.topology_logical_task (
);
GRANT ALL PRIVILEGES ON TABLE ambari.topology_logical_task TO :username;
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON ambari.stage (request_id);
+CREATE INDEX idx_hrc_request_id ON ambari.host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON ambari.role_success_criteria (request_id);
+
--------altering tables by creating unique constraints----------
ALTER TABLE ambari.users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
ALTER TABLE ambari.clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name, version_tag);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
index 6df1569..f837f9e 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
@@ -650,6 +650,11 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
-- altering tables by creating unique constraints----------
ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type);
ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
index 83b2a7e..239d27e 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
@@ -758,6 +758,11 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY CLUSTERED (id)
);
+-- tasks indices --
+CREATE INDEX idx_stage_request_id ON stage (request_id);
+CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
+CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id);
+
-- altering tables by creating unique constraints----------
--------altering tables to add constraints----------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
index bbae7a7..5bd8f2a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
@@ -22,8 +22,10 @@ package org.apache.ambari.server.upgrade;
import com.google.inject.AbstractModule;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
+import com.google.inject.Binder;
import com.google.inject.Guice;
import com.google.inject.Injector;
+import com.google.inject.Module;
import com.google.inject.Provider;
import com.google.inject.persist.PersistService;
import junit.framework.Assert;
@@ -60,6 +62,7 @@ import java.util.Map;
import static org.easymock.EasyMock.createMockBuilder;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
@@ -101,6 +104,34 @@ public class UpgradeCatalog221Test {
}
@Test
+ public void testExecuteDDLUpdates() throws Exception{
+ final DBAccessor dbAccessor = createNiceMock(DBAccessor.class);
+
+ dbAccessor.createIndex(eq("idx_stage_request_id"), eq("stage"), eq("request_id"));
+ expectLastCall().once();
+ dbAccessor.createIndex(eq("idx_hrc_request_id"), eq("host_role_command"), eq("request_id"));
+ expectLastCall().once();
+ dbAccessor.createIndex(eq("idx_rsc_request_id"), eq("role_success_criteria"), eq("request_id"));
+ expectLastCall().once();
+
+
+ replay(dbAccessor);
+ Module module = new Module() {
+ @Override
+ public void configure(Binder binder) {
+ binder.bind(DBAccessor.class).toInstance(dbAccessor);
+ binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+ binder.bind(EntityManager.class).toInstance(entityManager);
+ }
+ };
+
+ Injector injector = Guice.createInjector(module);
+ UpgradeCatalog221 upgradeCatalog221 = injector.getInstance(UpgradeCatalog221.class);
+ upgradeCatalog221.executeDDLUpdates();
+ verify(dbAccessor);
+ }
+
+ @Test
public void testExecuteDMLUpdates() throws Exception {
Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
Method updateAlerts = UpgradeCatalog221.class.getDeclaredMethod("updateAlerts");