You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/10/27 15:20:32 UTC
[04/16] ambari git commit: AMBARI-18698. Filter by roles in Users
List page takes upto 20 secs to load with 1000+ users. (mpapirkovskyy)
AMBARI-18698. Filter by roles in Users List page takes upto 20 secs to load with 1000+ users. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/03f61bd6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/03f61bd6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/03f61bd6
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 03f61bd6a10be892d67237c45f1db34830d4c5d8
Parents: aa588ca
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Wed Oct 26 14:40:13 2016 +0300
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Wed Oct 26 20:57:46 2016 +0300
----------------------------------------------------------------------
.../internal/UserPrivilegeResourceProvider.java | 116 +++++++++++++++++--
.../UserPrivilegeResourceProviderTest.java | 2 +
2 files changed, 109 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/03f61bd6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
index 009c38b..ba32a5f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
@@ -17,6 +17,9 @@
*/
package org.apache.ambari.server.controller.internal;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
import org.apache.ambari.server.controller.spi.Predicate;
@@ -30,6 +33,7 @@ import org.apache.ambari.server.orm.dao.UserDAO;
import org.apache.ambari.server.orm.dao.ViewInstanceDAO;
import org.apache.ambari.server.orm.entities.ClusterEntity;
import org.apache.ambari.server.orm.entities.GroupEntity;
+import org.apache.ambari.server.orm.entities.PrincipalEntity;
import org.apache.ambari.server.orm.entities.PrincipalTypeEntity;
import org.apache.ambari.server.orm.entities.PrivilegeEntity;
import org.apache.ambari.server.orm.entities.UserEntity;
@@ -48,6 +52,8 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
/**
* Resource provider for user privilege resources.
@@ -142,6 +148,87 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
keyPropertyIds.put(Resource.Type.UserPrivilege, PRIVILEGE_PRIVILEGE_ID_PROPERTY_ID);
}
+ private ThreadLocal<LoadingCache<Long, ClusterEntity>> clusterCache =
+ new ThreadLocal<LoadingCache<Long, ClusterEntity>>(){
+ @Override
+ protected LoadingCache<Long, ClusterEntity> initialValue() {
+ CacheLoader<Long, ClusterEntity> loader = new CacheLoader<Long, ClusterEntity>() {
+ @Override
+ public ClusterEntity load(Long key) throws Exception {
+ return clusterDAO.findByResourceId(key);
+ }
+ };
+ return CacheBuilder.newBuilder().expireAfterWrite(20, TimeUnit.SECONDS).build(loader);
+ }
+ };
+
+ private ThreadLocal<LoadingCache<Long, ViewInstanceEntity>> viewInstanceCache =
+ new ThreadLocal<LoadingCache<Long, ViewInstanceEntity>>(){
+ @Override
+ protected LoadingCache<Long, ViewInstanceEntity> initialValue() {
+ CacheLoader<Long, ViewInstanceEntity> loader = new CacheLoader<Long, ViewInstanceEntity>() {
+ @Override
+ public ViewInstanceEntity load(Long key) throws Exception {
+ return viewInstanceDAO.findByResourceId(key);
+ }
+ };
+ return CacheBuilder.newBuilder().expireAfterWrite(20, TimeUnit.SECONDS).build(loader);
+ }
+ };
+
+ private ThreadLocal<LoadingCache<String, UserEntity>> usersCache =
+ new ThreadLocal<LoadingCache<String, UserEntity>>(){
+ @Override
+ protected LoadingCache<String, UserEntity> initialValue() {
+ CacheLoader<String, UserEntity> loader = new CacheLoader<String, UserEntity>() {
+ @Override
+ public UserEntity load(String key) throws Exception {
+ //fallback mechanism, mostly for unit tests
+ UserEntity userEntity = userDAO.findLocalUserByName(key);
+ if (userEntity == null) {
+ userEntity = userDAO.findLdapUserByName(key);
+ }
+ if (userEntity == null) {
+ userEntity = userDAO.findUserByNameAndType(key, UserType.JWT);
+ }
+ return userEntity;
+ }
+ };
+
+ return CacheBuilder.newBuilder()
+ .expireAfterWrite(20, TimeUnit.SECONDS)
+ .build(loader);
+ }
+ };
+
+ private ThreadLocal<LoadingCache<PrincipalEntity, GroupEntity>> groupsCache =
+ new ThreadLocal<LoadingCache<PrincipalEntity, GroupEntity>>(){
+ @Override
+ protected LoadingCache<PrincipalEntity, GroupEntity> initialValue() {
+ CacheLoader<PrincipalEntity, GroupEntity> loader = new CacheLoader<PrincipalEntity, GroupEntity>() {
+ @Override
+ public GroupEntity load(PrincipalEntity key) throws Exception {
+ return groupDAO.findGroupByPrincipal(key);
+ }
+ };
+
+ return CacheBuilder.newBuilder()
+ .expireAfterWrite(20, TimeUnit.SECONDS)
+ .build(loader);
+ }
+ };
+
+ private GroupEntity getCachedGroupByPrincipal(PrincipalEntity principalEntity) {
+ GroupEntity entity = groupsCache.get().getIfPresent(principalEntity);
+ if (entity == null) {
+ for (GroupEntity groupEntity : groupDAO.findAll()) {
+ groupsCache.get().put(groupEntity.getPrincipal(), groupEntity);
+ }
+ entity = groupsCache.get().getUnchecked(principalEntity);
+ }
+ return entity;
+ }
+
/**
* Constructor.
@@ -183,13 +270,24 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
}
if (userName != null) {
- UserEntity userEntity = userDAO.findLocalUserByName(userName);
- if (userEntity == null) {
- userEntity = userDAO.findLdapUserByName(userName);
- }
+
+ UserEntity userEntity = usersCache.get().getIfPresent(userName);
if (userEntity == null) {
- userEntity = userDAO.findUserByNameAndType(userName, UserType.JWT);
+ //temporary tradeoff, add ~200ms for single user call, but start saving time for 100+ subsequent calls
+ //usual case for management page is to populate subresources for all users
+ Map<String, UserEntity> userNames = new TreeMap<>();
+ for (UserEntity entity : userDAO.findAll()) {
+ UserEntity existing = userNames.get(entity.getUserName());
+ if (existing == null ||
+ entity.getUserType() == UserType.LOCAL ||
+ existing.getUserType() == UserType.JWT) {
+ userNames.put(entity.getUserName(), entity);
+ }
+ }
+ usersCache.get().putAll(userNames);
+ userEntity = usersCache.get().getUnchecked(userName);
}
+
if (userEntity == null) {
throw new SystemException("User " + userName + " was not found");
}
@@ -213,7 +311,7 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
* @param requestedIds the relevant request ids
* @return a resource
*/
- protected Resource toResource(PrivilegeEntity privilegeEntity, Object userName, Set<String> requestedIds) {
+ protected Resource toResource(PrivilegeEntity privilegeEntity, Object userName, Set<String> requestedIds){
final ResourceImpl resource = new ResourceImpl(Resource.Type.UserPrivilege);
setResourceProperty(resource, PRIVILEGE_USER_NAME_PROPERTY_ID, userName, requestedIds);
@@ -227,7 +325,7 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
final UserEntity user = userDAO.findUserByPrincipal(privilegeEntity.getPrincipal());
setResourceProperty(resource, PRIVILEGE_PRINCIPAL_NAME_PROPERTY_ID, user.getUserName(), requestedIds);
} else if (principalTypeName.equals(PrincipalTypeEntity.GROUP_PRINCIPAL_TYPE_NAME)) {
- final GroupEntity groupEntity = groupDAO.findGroupByPrincipal(privilegeEntity.getPrincipal());
+ final GroupEntity groupEntity = getCachedGroupByPrincipal(privilegeEntity.getPrincipal());
setResourceProperty(resource, PRIVILEGE_PRINCIPAL_NAME_PROPERTY_ID, groupEntity.getGroupName(), requestedIds);
}
@@ -239,11 +337,11 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
// there is nothing special to add for this case
break;
case CLUSTER:
- final ClusterEntity clusterEntity = clusterDAO.findByResourceId(privilegeEntity.getResource().getId());
+ final ClusterEntity clusterEntity = clusterCache.get().getUnchecked(privilegeEntity.getResource().getId());
setResourceProperty(resource, PRIVILEGE_CLUSTER_NAME_PROPERTY_ID, clusterEntity.getClusterName(), requestedIds);
break;
case VIEW:
- final ViewInstanceEntity viewInstanceEntity = viewInstanceDAO.findByResourceId(privilegeEntity.getResource().getId());
+ final ViewInstanceEntity viewInstanceEntity = viewInstanceCache.get().getUnchecked(privilegeEntity.getResource().getId());
final ViewEntity viewEntity = viewInstanceEntity.getViewEntity();
setResourceProperty(resource, PRIVILEGE_VIEW_NAME_PROPERTY_ID, viewEntity.getCommonName(), requestedIds);
http://git-wip-us.apache.org/repos/asf/ambari/blob/03f61bd6/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
index ddb510d..ce2d8e1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
@@ -385,6 +385,7 @@ public class UserPrivilegeResourceProviderTest extends AbstractPrivilegeResource
final UserDAO userDAO = createNiceMock(UserDAO.class);
expect(userDAO.findLocalUserByName("jdoe")).andReturn(userEntity).anyTimes();
expect(userDAO.findUserByPrincipal(anyObject(PrincipalEntity.class))).andReturn(userEntity).anyTimes();
+ expect(userDAO.findAll()).andReturn(Collections.<UserEntity>emptyList()).anyTimes();
final PrivilegeDAO privilegeDAO = createMock(PrivilegeDAO.class);
final MemberDAO memberDAO = createMock(MemberDAO.class);
@@ -465,6 +466,7 @@ public class UserPrivilegeResourceProviderTest extends AbstractPrivilegeResource
andReturn(Collections.<MemberEntity>emptyList())
.atLeastOnce();
expect(userDAO.findLocalUserByName(requestedUsername)).andReturn(userEntity).anyTimes();
+ expect(userDAO.findAll()).andReturn(Collections.<UserEntity>emptyList()).anyTimes();
expect(userEntity.getPrincipal()).andReturn(principalEntity).anyTimes();
expect(userEntity.getMemberEntities()).andReturn(Collections.<MemberEntity>emptySet()).anyTimes();
expect(privilegeEntity.getPermission()).andReturn(permissionEntity).anyTimes();