You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/09/16 21:54:45 UTC
ambari git commit: AMBARI-13115. BE: Call to filter 100 hosts by
alerts takes 25s on 800 node cluster (srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 0dfe5769f -> 4f2079fa4
AMBARI-13115. BE: Call to filter 100 hosts by alerts takes 25s on 800 node cluster (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4f2079fa
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4f2079fa
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4f2079fa
Branch: refs/heads/branch-2.1
Commit: 4f2079fa420ba9cff002a2a93939e3a54eddc0ab
Parents: 0dfe576
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Wed Sep 16 07:23:45 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Wed Sep 16 12:54:29 2015 -0700
----------------------------------------------------------------------
.../ambari/server/api/query/QueryImpl.java | 28 +++++++-
.../ambari/server/api/query/QueryImplTest.java | 74 ++++++++++++++++++++
.../internal/ClusterControllerImplTest.java | 9 +++
3 files changed, 110 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2079fa/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
index 10d9bef..cb22ddd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
@@ -213,7 +213,6 @@ public class QueryImpl implements Query, ResourceInstance {
SystemException,
NoSuchResourceException,
NoSuchParentResourceException {
-
queryForResources();
return getResult(null);
}
@@ -407,6 +406,33 @@ public class QueryImpl implements Query, ResourceInstance {
clusterController.populateResources(resourceType, providerResourceSet, request, queryPredicate);
}
+ // Optimization:
+ // Currently the steps executed when sub-resources are requested are:
+ // (1) Get *all* top-level resources
+ // (2) Populate all top-level resources
+ // (3) Query for and populate sub-resources of *all* top-level resources
+ // (4) Apply pagination and predicate on resources from above
+ //
+ // Though this works, it is very inefficient when either:
+ // (a) Predicate does not apply to sub-resources
+ // (b) Page request is present
+ // It is inefficient because we needlessly populate sub-resources that might not get
+ // used due to their top-level resources being filtered out by the predicate and paging
+ //
+ // The optimization is to apply the predicate and paging request on the top-level resources
+ // directly if there are no sub-resources predicates.
+ if ((pageRequest != null || userPredicate != null) && !hasSubResourcePredicate() && populateResourceRequired(resourceType)) {
+ QueryResponse newResponse = new QueryResponseImpl(resourceSet, queryResponse.isSortedResponse(), queryResponse.isPagedResponse(),
+ queryResponse.getTotalResourceCount());
+ PageResponse pageResponse = clusterController.getPage(resourceType, newResponse, request, queryPredicate, pageRequest, sortRequest);
+ // build a new set
+ Set<Resource> newResourceSet = new LinkedHashSet<Resource>();
+ for (Resource r : pageResponse.getIterable()) {
+ newResourceSet.add(r);
+ }
+ populatedQueryResults.put(null, new QueryResult(request, queryPredicate, userPredicate, getKeyValueMap(), new QueryResponseImpl(newResourceSet)));
+ }
+
queryForSubResources();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2079fa/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
index 3bdbb87..f869ba3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
@@ -40,10 +40,12 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ambari.server.api.query.render.DefaultRenderer;
import org.apache.ambari.server.api.query.render.Renderer;
import org.apache.ambari.server.api.resources.ClusterResourceDefinition;
+import org.apache.ambari.server.api.resources.HostResourceDefinition;
import org.apache.ambari.server.api.resources.ResourceDefinition;
import org.apache.ambari.server.api.resources.ResourceInstance;
import org.apache.ambari.server.api.resources.StackResourceDefinition;
@@ -59,6 +61,7 @@ import org.apache.ambari.server.controller.spi.ClusterController;
import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
import org.apache.ambari.server.controller.spi.PageRequest;
+import org.apache.ambari.server.controller.spi.PageResponse;
import org.apache.ambari.server.controller.spi.Predicate;
import org.apache.ambari.server.controller.spi.QueryResponse;
import org.apache.ambari.server.controller.spi.Request;
@@ -69,6 +72,7 @@ import org.apache.ambari.server.controller.spi.SortRequest;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.easymock.Capture;
import org.easymock.EasyMockSupport;
import org.junit.Assert;
@@ -643,6 +647,76 @@ public class QueryImplTest {
}
@Test
+ public void testExecute__Host_collection_AlertsSummary() throws Exception {
+ ResourceDefinition resourceDefinition = new HostResourceDefinition();
+ Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>();
+ final AtomicInteger pageCallCount = new AtomicInteger(0);
+ ClusterControllerImpl clusterControllerImpl = new ClusterControllerImpl(new ClusterControllerImplTest.TestProviderModule()) {
+ public PageResponse getPage(Resource.Type type, QueryResponse queryResponse, Request request, Predicate predicate, PageRequest pageRequest,
+ SortRequest sortRequest) throws UnsupportedPropertyException, SystemException, NoSuchResourceException, NoSuchParentResourceException {
+ pageCallCount.incrementAndGet();
+ return super.getPage(type, queryResponse, request, predicate, pageRequest, sortRequest);
+ }
+ };
+ QueryImpl instance = new TestQuery(mapIds, resourceDefinition, clusterControllerImpl);
+
+ //test 1: No predicate or paging request
+ pageCallCount.set(0);
+ Result result = instance.execute();
+ TreeNode<Resource> tree = result.getResultTree();
+ Assert.assertEquals(4, tree.getChildren().size());
+ Assert.assertEquals(1, pageCallCount.get());
+
+ //test 2: Predicate = (alerts_summary/CRITICAL > 0)
+ pageCallCount.set(0);
+ PredicateBuilder pb = new PredicateBuilder();
+ Predicate predicate = pb.property("alerts_summary/CRITICAL").greaterThan(0).toPredicate();
+ instance.setUserPredicate(predicate);
+ result = instance.execute();
+ tree = result.getResultTree();
+ Assert.assertEquals(2, tree.getChildren().size());
+ Assert.assertEquals(2, pageCallCount.get());
+ TreeNode<Resource> hostNode = tree.getChild("Host:1");
+ Assert.assertEquals("Host:1", hostNode.getName());
+ Assert.assertEquals("host:0", hostNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("Hosts", "host_name")));
+ Assert.assertEquals(Resource.Type.Host, hostNode.getObject().getType());
+ Assert.assertEquals("1", hostNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("alerts_summary", "CRITICAL")));
+ hostNode = tree.getChild("Host:2");
+ Assert.assertEquals("Host:2", hostNode.getName());
+ Assert.assertEquals("host:2", hostNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("Hosts", "host_name")));
+ Assert.assertEquals(Resource.Type.Host, hostNode.getObject().getType());
+ Assert.assertEquals("1", hostNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("alerts_summary", "CRITICAL")));
+
+ // test 3: Predicate = (alerts_summary/WARNING > 0) AND Page = (size=1)
+ pageCallCount.set(0);
+ pb = new PredicateBuilder();
+ predicate = pb.property("alerts_summary/WARNING").greaterThan(0).toPredicate();
+ instance.setUserPredicate(predicate);
+ instance.setPageRequest(new PageRequestImpl(PageRequest.StartingPoint.Beginning, 1, 0, null, null));
+ result = instance.execute();
+ tree = result.getResultTree();
+ Assert.assertEquals(1, tree.getChildren().size());
+ Assert.assertEquals(2, pageCallCount.get());
+ hostNode = tree.getChild("Host:1");
+ Assert.assertEquals("Host:1", hostNode.getName());
+ Assert.assertEquals("host:1", hostNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("Hosts", "host_name")));
+ Assert.assertEquals(Resource.Type.Host, hostNode.getObject().getType());
+ Assert.assertEquals("1", hostNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("alerts_summary", "WARNING")));
+
+ // test 4: SubResource Predicate = (alerts_summary/WARNING > 0 | host_components/HostRoles/component_name = DATANODE)
+ pageCallCount.set(0);
+ pb = new PredicateBuilder();
+ predicate = pb.property("alerts_summary/WARNING").greaterThan(0).or().property("host_components/HostRoles/component_name").equals("DATANODE").toPredicate();
+ instance.setUserPredicate(predicate);
+ instance.setPageRequest(null);
+ result = instance.execute();
+ tree = result.getResultTree();
+ Assert.assertEquals(0, tree.getChildren().size());
+ Assert.assertEquals(6, pageCallCount.get());
+
+ }
+
+ @Test
public void testExecute__collection_nullInternalPredicate_nullUserPredicate() throws Exception {
ResourceDefinition resourceDefinition = createNiceMock(ResourceDefinition.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2079fa/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterControllerImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterControllerImplTest.java
index d126198..3f292ca 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterControllerImplTest.java
@@ -76,6 +76,8 @@ public class ClusterControllerImplTest {
propertyProviderProperties.add(PropertyHelper.getPropertyId("c3", "p6"));
propertyProviderProperties.add(PropertyHelper.getPropertyId("c4", "p7"));
propertyProviderProperties.add(PropertyHelper.getPropertyId("c4", "p8"));
+ propertyProviderProperties.add(PropertyHelper.getPropertyId("alerts_summary", "WARNING"));
+ propertyProviderProperties.add(PropertyHelper.getPropertyId("alerts_summary", "CRITICAL"));
}
private static final PropertyProvider propertyProvider = new PropertyProvider() {
@@ -1067,6 +1069,13 @@ public class ClusterControllerImplTest {
resource.setProperty(PropertyHelper.getPropertyId("c1", "p2"), cnt % 2);
resource.setProperty(PropertyHelper.getPropertyId("c1", "p3"), "foo");
resource.setProperty(PropertyHelper.getPropertyId("c2", "p4"), "bar");
+
+ if (cnt % 2 == 0) {
+ resource.setProperty(PropertyHelper.getPropertyId("alerts_summary", "CRITICAL"), "1");
+ } else {
+ resource.setProperty(PropertyHelper.getPropertyId("alerts_summary", "WARNING"), "1");
+ }
+
resources.add(resource);
}