You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2015/03/13 22:23:37 UTC
ambari git commit: AMBARI-10065. Performance: ActionScheduler loads
too many objects per Stage iteration (ncole)
Repository: ambari
Updated Branches:
refs/heads/trunk 78be98069 -> 88aed0b8e
AMBARI-10065. Performance: ActionScheduler loads too many objects per Stage iteration (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/88aed0b8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/88aed0b8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/88aed0b8
Branch: refs/heads/trunk
Commit: 88aed0b8e3f0db2e2726358a7622c4d5519ea709
Parents: 78be980
Author: Nate Cole <nc...@hortonworks.com>
Authored: Fri Mar 13 15:31:41 2015 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Fri Mar 13 17:23:10 2015 -0400
----------------------------------------------------------------------
.../server/KdcServerConnectionVerification.java | 18 ++--
.../server/actionmanager/ActionDBAccessor.java | 18 +++-
.../actionmanager/ActionDBAccessorImpl.java | 11 ++-
.../server/actionmanager/ActionScheduler.java | 6 +-
.../ambari/server/agent/HeartbeatMonitor.java | 2 +-
.../actionmanager/TestActionScheduler.java | 96 ++++++++++++--------
6 files changed, 93 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/88aed0b8/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java b/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
index b7bfef9..e464800 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
@@ -45,11 +45,11 @@ import com.google.inject.Singleton;
* <p>
* It has two potential clients.
* <ul>
- * <li>Ambari Agent:
+ * <li>Ambari Agent:
* Uses it to make sure host can talk to specified KDC Server
* </li>
- *
- * <li>Ambari Server:
+ *
+ * <li>Ambari Server:
* Uses it for connection check, like agent, and also validates
* the credentials provided on Server side.
* </li>
@@ -67,18 +67,18 @@ public class KdcServerConnectionVerification {
* UDP connection timeout in seconds.
*/
private int udpTimeout = 10;
-
+
@Inject
public KdcServerConnectionVerification(Configuration config) {
this.config = config;
}
-
+
/**
* Given server IP or hostname, checks if server is reachable i.e.
* we can make a socket connection to it. Hostname may contain port
- * number separated by a colon.
- *
+ * number separated by a colon.
+ *
* @param kdcHost KDC server IP or hostname (with optional port number)
* @return true, if server is accepting connection given port; false otherwise.
*/
@@ -116,7 +116,7 @@ public class KdcServerConnectionVerification {
/**
* Attempt to connect to KDC server over TCP.
- *
+ *
* @param server KDC server IP or hostname
* @param port KDC server port
* @return true, if server is accepting connection given port; false otherwise.
@@ -181,7 +181,7 @@ public class KdcServerConnectionVerification {
}
});
- new Thread(future).start();
+ new Thread(future, "ambari-kdc-verify").start();
Boolean result;
try {
// timeout after specified timeout
http://git-wip-us.apache.org/repos/asf/ambari/blob/88aed0b8/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
index 8d85d19..e047f6d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.orm.entities.RequestEntity;
import com.google.inject.persist.Transactional;
@@ -40,9 +41,18 @@ public interface ActionDBAccessor {
public List<Stage> getAllStages(long requestId);
/**
- * Get request object by id
- * @param requestId
- * @return
+ * Gets the request entity by id. Will not load the entire
+ * Request/Stage/HostRoleCommand object hierarchy.
+ */
+ RequestEntity getRequestEntity(long requestId);
+
+ /**
+ * Get request object by id. USE WITH CAUTION! This method will
+ * result in loading the full object hierarchy, and can be expensive to
+ * construct.
+ * @param requestId the request id
+ * @return the Request instance, with all Stages and HostRoleCommands constructed
+ * from the database
*/
Request getRequest(long requestId);
@@ -205,4 +215,6 @@ public interface ActionDBAccessor {
* Gets request objects by ids
*/
public List<Request> getRequests(Collection<Long> requestIds);
+
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/88aed0b8/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
index df75c0c..7447a2d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
@@ -141,8 +141,13 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
}
@Override
+ public RequestEntity getRequestEntity(long requestId) {
+ return requestDAO.findByPK(requestId);
+ }
+
+ @Override
public Request getRequest(long requestId) {
- RequestEntity requestEntity = requestDAO.findByPK(requestId);
+ RequestEntity requestEntity = getRequestEntity(requestId);
if (requestEntity != null) {
return requestFactory.createExisting(requestEntity);
} else {
@@ -309,7 +314,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
@Override
public void startRequest(long requestId) {
- RequestEntity requestEntity = requestDAO.findByPK(requestId);
+ RequestEntity requestEntity = getRequestEntity(requestId);
if (requestEntity != null && requestEntity.getStartTime() == -1L) {
requestEntity.setStartTime(System.currentTimeMillis());
requestDAO.merge(requestEntity);
@@ -318,7 +323,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
@Override
public void endRequest(long requestId) {
- RequestEntity requestEntity = requestDAO.findByPK(requestId);
+ RequestEntity requestEntity = getRequestEntity(requestId);
if (requestEntity != null && requestEntity.getEndTime() == -1L) {
requestEntity.setEndTime(System.currentTimeMillis());
requestDAO.merge(requestEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/88aed0b8/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
index ca169b8..1a49289 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
@@ -44,6 +44,7 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.HostsMap;
import org.apache.ambari.server.events.ActionFinalReportReceivedEvent;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
+import org.apache.ambari.server.orm.entities.RequestEntity;
import org.apache.ambari.server.serveraction.ServerActionExecutor;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
@@ -152,7 +153,7 @@ class ActionScheduler implements Runnable {
}
public void start() {
- schedulerThread = new Thread(this);
+ schedulerThread = new Thread(this, "ambari-action-scheduler");
schedulerThread.start();
// Start up the ServerActionExecutor. Since it is directly related to the ActionScheduler it
@@ -252,7 +253,8 @@ class ActionScheduler implements Runnable {
i_stage ++;
long requestId = stage.getRequestId();
LOG.debug("==> STAGE_i = " + i_stage + "(requestId=" + requestId + ",StageId=" + stage.getStageId() + ")");
- Request request = db.getRequest(requestId);
+
+ RequestEntity request = db.getRequestEntity(requestId);
if (request.isExclusive()) {
if (runningRequestIds.size() > 0 ) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/88aed0b8/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
index ce6a0b8..d44e0a8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
@@ -94,7 +94,7 @@ public class HeartbeatMonitor implements Runnable {
}
public void start() {
- monitorThread = new Thread(this);
+ monitorThread = new Thread(this, "ambari-hearbeat-monitor");
monitorThread.start();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/88aed0b8/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
index 890ad80..dd93176 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
@@ -62,6 +62,7 @@ import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.HostsMap;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
+import org.apache.ambari.server.orm.entities.RequestEntity;
import org.apache.ambari.server.serveraction.MockServerAction;
import org.apache.ambari.server.serveraction.ServerActionExecutor;
import org.apache.ambari.server.state.Cluster;
@@ -160,9 +161,9 @@ public class TestActionScheduler {
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
//Keep large number of attempts so that the task is not expired finally
//Small action timeout to test rescheduling
@@ -252,9 +253,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
- Request request = mock(Request.class);
+
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
doAnswer(new Answer() {
@Override
@@ -330,9 +332,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -416,9 +419,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
@@ -549,9 +553,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -642,9 +647,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -727,9 +733,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -858,9 +865,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
@@ -949,9 +957,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
@@ -1025,9 +1034,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
@@ -1087,9 +1097,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -1268,9 +1279,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -1445,9 +1457,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -1673,9 +1686,9 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessorImpl.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
Stage s1 = StageUtils.getATestStage(requestId1, stageId, hostname, CLUSTER_HOST_INFO,
"{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}");
@@ -1760,9 +1773,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
@@ -1841,9 +1855,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
doAnswer(new Answer() {
@@ -1936,9 +1951,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
- Request request = mock(Request.class);
+ RequestEntity request = mock(RequestEntity.class);
when(request.isExclusive()).thenReturn(false);
- when(db.getRequest(anyLong())).thenReturn(request);
+ when(db.getRequestEntity(anyLong())).thenReturn(request);
+
when(db.getCommandsInProgressCount()).thenReturn(stages.size());
when(db.getStagesInProgress()).thenReturn(stages);
@@ -2164,16 +2180,16 @@ public class TestActionScheduler {
}
}).when(db).startRequest(anyLong());
- Request request1 = mock(Request.class);
+ RequestEntity request1 = mock(RequestEntity.class);
when(request1.isExclusive()).thenReturn(false);
- Request request2 = mock(Request.class);
+ RequestEntity request2 = mock(RequestEntity.class);
when(request2.isExclusive()).thenReturn(true);
- Request request3 = mock(Request.class);
+ RequestEntity request3 = mock(RequestEntity.class);
when(request3.isExclusive()).thenReturn(false);
- when(db.getRequest(requestId1)).thenReturn(request1);
- when(db.getRequest(requestId2)).thenReturn(request2);
- when(db.getRequest(requestId3)).thenReturn(request3);
+ when(db.getRequestEntity(requestId1)).thenReturn(request1);
+ when(db.getRequestEntity(requestId2)).thenReturn(request2);
+ when(db.getRequestEntity(requestId3)).thenReturn(request3);
Properties properties = new Properties();
Configuration conf = new Configuration(properties);