You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2015/06/11 15:20:56 UTC
ambari git commit: AMBARI-11850. Blueprint export process is very
slow in multi-node clusters. (rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk 3dd9ae471 -> 98405ef9b
AMBARI-11850. Blueprint export process is very slow in multi-node clusters. (rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/98405ef9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/98405ef9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/98405ef9
Branch: refs/heads/trunk
Commit: 98405ef9bee07b974baeaa13cb143aeea6aeadfd
Parents: 3dd9ae4
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Thu Jun 11 09:19:52 2015 -0400
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Thu Jun 11 09:20:30 2015 -0400
----------------------------------------------------------------------
.../ambari/server/api/query/QueryImpl.java | 11 +-
.../server/api/query/render/BaseRenderer.java | 6 +
.../query/render/ClusterBlueprintRenderer.java | 9 +
.../server/api/query/render/Renderer.java | 10 ++
.../ambari/server/api/query/QueryImplTest.java | 163 +++++++++++++++++++
.../render/ClusterBlueprintRendererTest.java | 10 ++
.../api/query/render/DefaultRendererTest.java | 9 +
7 files changed, 216 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
index 2319683..10d9bef 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
@@ -403,7 +403,10 @@ public class QueryImpl implements Query, ResourceInstance {
queryResults.put(null, new QueryResult(
request, queryPredicate, userPredicate, getKeyValueMap(), queryResponse));
- clusterController.populateResources(resourceType, providerResourceSet, request, queryPredicate);
+ if (renderer.requiresPropertyProviderInput()) {
+ clusterController.populateResources(resourceType, providerResourceSet, request, queryPredicate);
+ }
+
queryForSubResources();
}
@@ -445,7 +448,11 @@ public class QueryImpl implements Query, ResourceInstance {
new QueryResult(request, queryPredicate, subResourcePredicate, map, new QueryResponseImpl(resourceSet)));
}
}
- clusterController.populateResources(resourceType, providerResourceSet, request, subResourcePredicate);
+
+ if (renderer.requiresPropertyProviderInput()) {
+ clusterController.populateResources(resourceType, providerResourceSet, request, subResourcePredicate);
+ }
+
subResource.queryForSubResources();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java
index 7866aa4..03bce14 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java
@@ -46,6 +46,12 @@ public abstract class BaseRenderer implements Renderer {
m_schemaFactory = schemaFactory;
}
+ @Override
+ public boolean requiresPropertyProviderInput() {
+ // most renderers require the property provider input support
+ return true;
+ }
+
/**
* Obtain a schema instance based on resource type.
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index cfc9bc0..29c3040 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -134,6 +134,15 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
return new BlueprintPostProcessor(request);
}
+ @Override
+ public boolean requiresPropertyProviderInput() {
+ // the Blueprint-based renderer does not require property provider input
+ // this method will help to filter out the un-necessary calls to the AMS
+ // and Alerts Property providers, since they are not included in the
+ // exported Blueprint
+ return false;
+ }
+
// ----- private instance methods ------------------------------------------
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java
index f353d53..2ba245c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java
@@ -80,4 +80,14 @@ public interface Renderer {
* @return associated post processor
*/
public ResultPostProcessor getResultPostProcessor(Request request);
+
+
+ /**
+ * Obtains the property provider requirements of the given
+ * renderer implementation.
+ *
+ * @return true if property provider support is required
+ * false if property provider support is not required
+ */
+ public boolean requiresPropertyProviderInput();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
index 01361d2..3bdbb87 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java
@@ -27,6 +27,7 @@ import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.isA;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.junit.Assert.assertFalse;
@@ -60,13 +61,16 @@ import org.apache.ambari.server.controller.spi.NoSuchResourceException;
import org.apache.ambari.server.controller.spi.PageRequest;
import org.apache.ambari.server.controller.spi.Predicate;
import org.apache.ambari.server.controller.spi.QueryResponse;
+import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.Schema;
+import org.apache.ambari.server.controller.spi.SchemaFactory;
import org.apache.ambari.server.controller.spi.SortRequest;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
import org.easymock.Capture;
+import org.easymock.EasyMockSupport;
import org.junit.Assert;
import org.junit.Test;
@@ -842,6 +846,165 @@ public class QueryImplTest {
Assert.assertNotNull(hostNode.getObject().getPropertyValue("c1/p3"));
}
+ @Test
+ public void testExecute_RendererDoesNotRequirePropertyProviderInput() throws Exception {
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ ResourceDefinition mockResourceDefinition =
+ mockSupport.createMock(ResourceDefinition.class);
+
+ SubResourceDefinition mockSubResourceDefinition =
+ mockSupport.createMock(SubResourceDefinition.class);
+
+ // need to use a default mock here, to verify
+ // that certain methods are called or not called
+ ClusterController mockClusterController =
+ mockSupport.createMock(ClusterController.class);
+
+ Renderer mockRenderer =
+ mockSupport.createMock(Renderer.class);
+
+ QueryResponse mockQueryResponse =
+ mockSupport.createMock(QueryResponse.class);
+
+ QueryResponse mockSubQueryResponse =
+ mockSupport.createMock(QueryResponse.class);
+
+ Resource mockResource =
+ mockSupport.createMock(Resource.class);
+
+ Schema mockSchema =
+ mockSupport.createMock(Schema.class);
+
+ expect(mockResourceDefinition.getType()).andReturn(Resource.Type.Host).atLeastOnce();
+ expect(mockResourceDefinition.getSubResourceDefinitions()).andReturn(Collections.singleton(mockSubResourceDefinition)).atLeastOnce();
+
+ expect(mockSubResourceDefinition.getType()).andReturn(Resource.Type.Configuration).atLeastOnce();
+ expect(mockSubResourceDefinition.isCollection()).andReturn(false).atLeastOnce();
+
+ expect(mockSchema.getKeyPropertyId(isA(Resource.Type.class))).andReturn("test-value").anyTimes();
+ expect(mockSchema.getKeyTypes()).andReturn(Collections.<Resource.Type>emptySet()).anyTimes();
+
+ mockRenderer.init(isA(SchemaFactory.class));
+ // the mock renderer should return false for requiresPropertyProviderInput, to
+ // simulate the case of a renderer that does not need the property providers to execute
+ // this method should be called twice: once each resource (1 resource, 1 sub-resource in this test)
+ expect(mockRenderer.requiresPropertyProviderInput()).andReturn(false).times(2);
+ expect(mockRenderer.finalizeProperties(isA(TreeNode.class), eq(true))).andReturn(new TreeNodeImpl<Set<String>>(null, Collections.<String>emptySet(), "test-node"));
+ expect(mockRenderer.finalizeResult(isA(Result.class))).andReturn(null);
+
+ // note: the expectations for the ClusterController mock are significant to this test.
+ // the ClusterController.populateResources() method should not be called, and so
+ // is not expected here.
+ expect(mockClusterController.getSchema(Resource.Type.Host)).andReturn(mockSchema).anyTimes();
+ expect(mockClusterController.getSchema(Resource.Type.Configuration)).andReturn(mockSchema).anyTimes();
+ expect(mockClusterController.getResources(eq(Resource.Type.Host), isA(Request.class), (Predicate)eq(null))).andReturn(mockQueryResponse).atLeastOnce();
+ expect(mockClusterController.getResources(eq(Resource.Type.Configuration), isA(Request.class), (Predicate)eq(null))).andReturn(mockSubQueryResponse).atLeastOnce();
+ expect(mockClusterController.getIterable(eq(Resource.Type.Host), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+ expect(mockClusterController.getIterable(eq(Resource.Type.Configuration), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+
+ expect(mockQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+ expect(mockSubQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+
+ expect(mockResource.getType()).andReturn(Resource.Type.Host).atLeastOnce();
+
+ Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>();
+
+ mockSupport.replayAll();
+
+ QueryImpl instance = new QueryImpl(mapIds, mockResourceDefinition, mockClusterController);
+ instance.setRenderer(mockRenderer);
+
+ // call these methods to setup sub resources
+ instance.ensureSubResources();
+ instance.addProperty("*", null);
+
+ instance.execute();
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testExecute_RendererRequiresPropertyProviderInput() throws Exception {
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ ResourceDefinition mockResourceDefinition =
+ mockSupport.createMock(ResourceDefinition.class);
+
+ SubResourceDefinition mockSubResourceDefinition =
+ mockSupport.createMock(SubResourceDefinition.class);
+
+ // need to use a default mock here, to verify
+ // that certain methods are called or not called
+ ClusterController mockClusterController =
+ mockSupport.createMock(ClusterController.class);
+
+ Renderer mockRenderer =
+ mockSupport.createMock(Renderer.class);
+
+ QueryResponse mockQueryResponse =
+ mockSupport.createMock(QueryResponse.class);
+
+ QueryResponse mockSubQueryResponse =
+ mockSupport.createMock(QueryResponse.class);
+
+ Resource mockResource =
+ mockSupport.createMock(Resource.class);
+
+ Schema mockSchema =
+ mockSupport.createMock(Schema.class);
+
+ expect(mockResourceDefinition.getType()).andReturn(Resource.Type.Host).atLeastOnce();
+ expect(mockResourceDefinition.getSubResourceDefinitions()).andReturn(Collections.singleton(mockSubResourceDefinition)).atLeastOnce();
+
+ expect(mockSubResourceDefinition.getType()).andReturn(Resource.Type.Configuration).atLeastOnce();
+ expect(mockSubResourceDefinition.isCollection()).andReturn(false).atLeastOnce();
+
+ expect(mockSchema.getKeyPropertyId(isA(Resource.Type.class))).andReturn("test-value").anyTimes();
+ expect(mockSchema.getKeyTypes()).andReturn(Collections.<Resource.Type>emptySet()).anyTimes();
+
+ mockRenderer.init(isA(SchemaFactory.class));
+ // simulate the case of a renderer that requires the property providers to execute
+ // this method should be called twice: once for each resource (1 resource, 1 sub-resource in this test)
+ expect(mockRenderer.requiresPropertyProviderInput()).andReturn(true).times(2);
+ expect(mockRenderer.finalizeProperties(isA(TreeNode.class), eq(true))).andReturn(new TreeNodeImpl<Set<String>>(null, Collections.<String>emptySet(), "test-node"));
+ expect(mockRenderer.finalizeResult(isA(Result.class))).andReturn(null);
+
+ // note: the expectations for the ClusterController mock are significant to this test.
+ expect(mockClusterController.getSchema(Resource.Type.Host)).andReturn(mockSchema).anyTimes();
+ expect(mockClusterController.getSchema(Resource.Type.Configuration)).andReturn(mockSchema).anyTimes();
+ expect(mockClusterController.getResources(eq(Resource.Type.Host), isA(Request.class), (Predicate)eq(null))).andReturn(mockQueryResponse).atLeastOnce();
+ expect(mockClusterController.getResources(eq(Resource.Type.Configuration), isA(Request.class), (Predicate)eq(null))).andReturn(mockSubQueryResponse).atLeastOnce();
+ expect(mockClusterController.getIterable(eq(Resource.Type.Host), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+ expect(mockClusterController.getIterable(eq(Resource.Type.Configuration), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+ // expect call to activate property providers for Host resource
+ expect(mockClusterController.populateResources(eq(Resource.Type.Host), eq(Collections.singleton(mockResource)), isA(Request.class), (Predicate)eq(null))).andReturn(Collections.<Resource>emptySet()).times(1);
+ // expect call to activate property providers for Configuration sub-resource
+ expect(mockClusterController.populateResources(eq(Resource.Type.Configuration), eq(Collections.singleton(mockResource)), isA(Request.class), (Predicate)eq(null))).andReturn(Collections.<Resource>emptySet()).times(1);
+
+ expect(mockQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+ expect(mockSubQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce();
+
+ expect(mockResource.getType()).andReturn(Resource.Type.Host).atLeastOnce();
+
+ Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>();
+
+ mockSupport.replayAll();
+
+ QueryImpl instance = new QueryImpl(mapIds, mockResourceDefinition, mockClusterController);
+ instance.setRenderer(mockRenderer);
+
+ // call these methods to setup sub resources
+ instance.ensureSubResources();
+ instance.addProperty("*", null);
+
+ instance.execute();
+
+ mockSupport.verifyAll();
+
+ }
+
public static class TestQuery extends QueryImpl {
public TestQuery(Map<Resource.Type, String> mapIds, ResourceDefinition resourceDefinition) {
super(mapIds, resourceDefinition, new ClusterControllerImpl(new ClusterControllerImplTest.TestProviderModule()));
http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
index 96abb8c..c0fe07e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
@@ -58,6 +58,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
/**
* ClusterBlueprintRenderer unit tests.
@@ -375,6 +376,15 @@ public class ClusterBlueprintRendererTest {
}
+ @Test
+ public void testClusterRendererDefaults() throws Exception {
+ Renderer clusterBlueprintRenderer =
+ new ClusterBlueprintRenderer();
+
+ assertFalse("ClusterBlueprintRenderer should not require property provider input",
+ clusterBlueprintRenderer.requiresPropertyProviderInput());
+ }
+
//todo: collection resource
private void createClusterResultTree(TreeNode<Resource> resultTree) throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
index 4981a67..78efa1c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
@@ -341,4 +341,13 @@ public class DefaultRendererTest {
assertSame(result, renderer.finalizeResult(result));
}
+
+ @Test
+ public void testRequiresInputDefault() throws Exception {
+ Renderer defaultRenderer =
+ new DefaultRenderer();
+
+ assertTrue("Default renderer for cluster resources must require property provider input",
+ defaultRenderer.requiresPropertyProviderInput());
+ }
}