You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@yunikorn.apache.org by ma...@apache.org on 2023/07/28 07:23:47 UTC

[yunikorn-core] branch master updated: [YUNIKORN-1835] User belong to zero or multiple groups (#587)

This is an automated email from the ASF dual-hosted git repository.

mani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git


The following commit(s) were added to refs/heads/master by this push:
     new b4a9c7f7 [YUNIKORN-1835] User belong to zero or multiple groups (#587)
b4a9c7f7 is described below

commit b4a9c7f789359402e7ec694de5f40201635d2cef
Author: Manikandan R <ma...@gmail.com>
AuthorDate: Fri Jul 28 12:53:26 2023 +0530

    [YUNIKORN-1835] User belong to zero or multiple groups (#587)
    
    Closes: #587
    
    Signed-off-by: Manikandan R <ma...@gmail.com>
---
 pkg/scheduler/objects/application_test.go |  22 +--
 pkg/scheduler/objects/utilities_test.go   |   2 +-
 pkg/scheduler/ugm/group_tracker.go        |  19 ++
 pkg/scheduler/ugm/manager.go              | 268 ++++++++++++++++------------
 pkg/scheduler/ugm/manager_test.go         | 279 ++++++++++++++++++++++++------
 pkg/scheduler/ugm/queue_tracker.go        |   1 +
 pkg/scheduler/ugm/user_tracker.go         |  44 ++++-
 pkg/scheduler/ugm/utilities.go            |  13 ++
 pkg/scheduler/ugm/utilities_test.go       |  14 ++
 pkg/webservice/handlers_test.go           |  10 +-
 10 files changed, 487 insertions(+), 185 deletions(-)

diff --git a/pkg/scheduler/objects/application_test.go b/pkg/scheduler/objects/application_test.go
index 6e3f838d..f63bfcbd 100644
--- a/pkg/scheduler/objects/application_test.go
+++ b/pkg/scheduler/objects/application_test.go
@@ -1604,34 +1604,34 @@ func TestIncAndDecUserResourceUsageInSameGroup(t *testing.T) {
 
 	// Increase testuser
 	app.incUserResourceUsage(res)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), res, res, 0)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), nil, res, 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), res, nil, 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), nil, nil, 0)
 
 	// Increase both testuser and testuser2 with the same group testgroup
 	app.incUserResourceUsage(res)
 	app2.incUserResourceUsage(resources.Multiply(res, 2))
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), resources.Multiply(res, 2), resources.Multiply(res, 4), 0)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), resources.Multiply(res, 2), resources.Multiply(res, 4), 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), resources.Multiply(res, 2), nil, 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), resources.Multiply(res, 2), nil, 0)
 
 	// Increase nil
 	app.decUserResourceUsage(nil, false)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), resources.Multiply(res, 2), resources.Multiply(res, 4), 0)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), resources.Multiply(res, 2), resources.Multiply(res, 4), 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), resources.Multiply(res, 2), nil, 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), resources.Multiply(res, 2), nil, 0)
 
 	// Decrease testuser
 	app.decUserResourceUsage(res, false)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), res, resources.Multiply(res, 3), 0)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), resources.Multiply(res, 2), resources.Multiply(res, 3), 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), res, nil, 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), resources.Multiply(res, 2), nil, 0)
 
 	// Decrease testuser2
 	app2.decUserResourceUsage(res, false)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), res, resources.Multiply(res, 2), 0)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), res, resources.Multiply(res, 2), 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), res, nil, 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser2", testgroups), res, nil, 0)
 
 	// Decrease testuser and testuser2 to 0
 	app.decUserResourceUsage(res, false)
 	app2.decUserResourceUsage(res, false)
-	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), resources.NewResourceFromMap(map[string]resources.Quantity{"first": 0}), resources.NewResourceFromMap(map[string]resources.Quantity{"first": 0}), 0)
+	assertUserResourcesAndGroupResources(t, getUserGroup("testuser", testgroups), resources.NewResourceFromMap(map[string]resources.Quantity{"first": 0}), nil, 0)
 }
 
 func TestGetAllRequests(t *testing.T) {
diff --git a/pkg/scheduler/objects/utilities_test.go b/pkg/scheduler/objects/utilities_test.go
index a2765cdb..8b5dac9c 100644
--- a/pkg/scheduler/objects/utilities_test.go
+++ b/pkg/scheduler/objects/utilities_test.go
@@ -264,7 +264,7 @@ func assertUserGroupResource(t *testing.T, userGroup security.UserGroup, expecte
 	userResource := ugm.GetUserResources(userGroup)
 	groupResource := ugm.GetGroupResources(userGroup.Groups[0])
 	assert.Equal(t, resources.Equals(userResource, expected), true)
-	assert.Equal(t, resources.Equals(groupResource, expected), true)
+	assert.Equal(t, resources.Equals(groupResource, nil), true)
 }
 
 func assertUserResourcesAndGroupResources(t *testing.T, userGroup security.UserGroup, expectedUserResources *resources.Resource, expectedGroupResources *resources.Resource, i int) {
diff --git a/pkg/scheduler/ugm/group_tracker.go b/pkg/scheduler/ugm/group_tracker.go
index 5a7d3ffa..a48e9b22 100644
--- a/pkg/scheduler/ugm/group_tracker.go
+++ b/pkg/scheduler/ugm/group_tracker.go
@@ -44,6 +44,9 @@ func newGroupTracker(group string) *GroupTracker {
 }
 
 func (gt *GroupTracker) increaseTrackedResource(queuePath, applicationID string, usage *resources.Resource) bool {
+	if gt == nil {
+		return true
+	}
 	gt.Lock()
 	defer gt.Unlock()
 	gt.applications[applicationID] = true
@@ -51,6 +54,9 @@ func (gt *GroupTracker) increaseTrackedResource(queuePath, applicationID string,
 }
 
 func (gt *GroupTracker) decreaseTrackedResource(queuePath, applicationID string, usage *resources.Resource, removeApp bool) (bool, bool) {
+	if gt == nil {
+		return false, true
+	}
 	gt.Lock()
 	defer gt.Unlock()
 	if removeApp {
@@ -115,3 +121,16 @@ func (gt *GroupTracker) canBeRemoved() bool {
 	defer gt.RUnlock()
 	return len(gt.queueTracker.childQueueTrackers) == 0 && len(gt.queueTracker.runningApplications) == 0
 }
+
+func (gt *GroupTracker) removeApp(applicationID string) {
+	gt.Lock()
+	defer gt.Unlock()
+	delete(gt.applications, applicationID)
+}
+
+func (gt *GroupTracker) getName() string {
+	if gt == nil {
+		return ""
+	}
+	return gt.groupName
+}
diff --git a/pkg/scheduler/ugm/manager.go b/pkg/scheduler/ugm/manager.go
index 5163c03e..859d33cb 100644
--- a/pkg/scheduler/ugm/manager.go
+++ b/pkg/scheduler/ugm/manager.go
@@ -41,6 +41,7 @@ type Manager struct {
 	groupTrackers             map[string]*GroupTracker
 	userWildCardLimitsConfig  map[string]*LimitConfig // Hold limits settings of user '*'
 	groupWildCardLimitsConfig map[string]*LimitConfig // Hold limits settings of group '*'
+	configuredGroups          map[string][]string     // Hold groups for all configured queue paths.
 	sync.RWMutex
 }
 
@@ -86,6 +87,7 @@ func (m *Manager) IncreaseTrackedResource(queuePath, applicationID string, usage
 		return false
 	}
 	userTracker := m.getUserTracker(user.User, true)
+	m.ensureGroupTrackerForApp(queuePath, applicationID, user)
 	log.Log(log.SchedUGM).Debug("Increasing resource usage for user",
 		zap.String("user", user.User),
 		zap.String("queue path", queuePath),
@@ -95,23 +97,6 @@ func (m *Manager) IncreaseTrackedResource(queuePath, applicationID string, usage
 	if !increased {
 		return increased
 	}
-	if err := m.ensureGroupTrackerForApp(queuePath, applicationID, user); err != nil {
-		return false
-	}
-	group, err := m.getGroup(user)
-	if err != nil {
-		return false
-	}
-	groupTracker := m.getGroupTracker(group, true)
-	log.Log(log.SchedUGM).Debug("Increasing resource usage for group",
-		zap.String("group", group),
-		zap.String("queue path", queuePath),
-		zap.String("application", applicationID),
-		zap.Stringer("resource", usage))
-	increased = groupTracker.increaseTrackedResource(queuePath, applicationID, usage)
-	if !increased {
-		return increased
-	}
 	return true
 }
 
@@ -160,33 +145,27 @@ func (m *Manager) DecreaseTrackedResource(queuePath, applicationID string, usage
 		delete(m.userTrackers, user.User)
 	}
 
-	group, err := m.getGroup(user)
-	if err != nil {
-		return false
-	}
+	group := m.ensureGroup(user, queuePath)
 	groupTracker := m.getGroupTracker(group, false)
-	if groupTracker == nil {
-		log.Log(log.SchedUGM).Error("appGroupTrackers tracker must be available in groupTrackers map",
-			zap.String("appGroupTrackers", group))
-		return false
-	}
-	log.Log(log.SchedUGM).Debug("Decreasing resource usage for group",
-		zap.String("group", group),
-		zap.String("queue path", queuePath),
-		zap.String("application", applicationID),
-		zap.Stringer("resource", usage),
-		zap.Bool("removeApp", removeApp))
-	removeQT, decreased = groupTracker.decreaseTrackedResource(queuePath, applicationID, usage, removeApp)
-	if !decreased {
-		return decreased
-	}
-	if removeApp && removeQT {
-		log.Log(log.SchedUGM).Debug("Removing group from manager",
+	if groupTracker != nil {
+		log.Log(log.SchedUGM).Debug("Decreasing resource usage for group",
 			zap.String("group", group),
 			zap.String("queue path", queuePath),
 			zap.String("application", applicationID),
+			zap.Stringer("resource", usage),
 			zap.Bool("removeApp", removeApp))
-		delete(m.groupTrackers, group)
+		removeQT, decreased = groupTracker.decreaseTrackedResource(queuePath, applicationID, usage, removeApp)
+		if !decreased {
+			return decreased
+		}
+		if removeApp && removeQT {
+			log.Log(log.SchedUGM).Debug("Removing group from manager",
+				zap.String("group", group),
+				zap.String("queue path", queuePath),
+				zap.String("application", applicationID),
+				zap.Bool("removeApp", removeApp))
+			delete(m.groupTrackers, group)
+		}
 	}
 	return true
 }
@@ -249,44 +228,71 @@ func (m *Manager) GetGroupTracker(group string) *GroupTracker {
 	return nil
 }
 
-func (m *Manager) ensureGroupTrackerForApp(queuePath string, applicationID string, user security.UserGroup) error {
+func (m *Manager) ensureGroupTrackerForApp(queuePath string, applicationID string, user security.UserGroup) string {
 	m.Lock()
 	defer m.Unlock()
 	userTracker := m.userTrackers[user.User]
 	if !userTracker.hasGroupForApp(applicationID) {
 		var groupTracker *GroupTracker
-		group, err := m.getGroup(user)
-		if err != nil {
-			return err
-		}
-		if m.groupTrackers[group] == nil {
-			log.Log(log.SchedUGM).Debug("Group tracker doesn't exists. Creating group tracker",
-				zap.String("application", applicationID),
-				zap.String("queue path", queuePath),
-				zap.String("user", user.User),
-				zap.String("group", group))
-			groupTracker = newGroupTracker(group)
-			m.groupTrackers[group] = groupTracker
-		} else {
-			log.Log(log.SchedUGM).Debug("Group tracker already exists and linking (reusing) the same with application",
-				zap.String("application", applicationID),
-				zap.String("queue path", queuePath),
-				zap.String("user", user.User),
-				zap.String("group", group))
-			groupTracker = m.groupTrackers[group]
+		group := m.internalEnsureGroup(user, queuePath)
+		if group != "" {
+			if m.groupTrackers[group] == nil {
+				log.Log(log.SchedUGM).Debug("Group tracker doesn't exists. Creating group tracker",
+					zap.String("application", applicationID),
+					zap.String("queue path", queuePath),
+					zap.String("user", user.User),
+					zap.String("group", group))
+				groupTracker = newGroupTracker(group)
+				m.groupTrackers[group] = groupTracker
+			} else {
+				log.Log(log.SchedUGM).Debug("Group tracker already exists and linking (reusing) the same with application",
+					zap.String("application", applicationID),
+					zap.String("queue path", queuePath),
+					zap.String("user", user.User),
+					zap.String("group", group))
+				groupTracker = m.groupTrackers[group]
+			}
 		}
 		userTracker.setGroupForApp(applicationID, groupTracker)
+		return group
+	} else {
+		return userTracker.getGroupForApp(applicationID)
 	}
-	return nil
 }
 
-// getGroup Based on the current limitations, username and group name is same. Groups[0] is always set and same as username.
-// It would be changed in future based on user group resolution, limit configuration processing etc
-func (m *Manager) getGroup(user security.UserGroup) (string, error) {
-	if len(user.Groups) > 0 {
-		return user.Groups[0], nil
+// ensureGroup User may belong to zero or multiple groups. Limits are configured for different groups at different queue hierarchy.
+// Among these multiple groups stored in security.UserGroup, matching against group for which limit has been configured happens from leaf to root and first
+// matching group would be picked and used as user's group for all activities in UGM module.
+func (m *Manager) ensureGroup(user security.UserGroup, queuePath string) string {
+	m.RLock()
+	defer m.RUnlock()
+	return m.internalEnsureGroup(user, queuePath)
+}
+
+// lock free version
+func (m *Manager) internalEnsureGroup(user security.UserGroup, queuePath string) string {
+	userTracker := m.userTrackers[user.User]
+	if userTracker != nil {
+		if configGroups, ok := m.configuredGroups[queuePath]; ok {
+			for _, configGroup := range configGroups {
+				for _, g := range user.Groups {
+					if configGroup == g {
+						log.Log(log.SchedUGM).Debug("Found matching group for user",
+							zap.String("user", user.User),
+							zap.String("queue path", queuePath),
+							zap.String("matched group", configGroup))
+						return configGroup
+					}
+				}
+			}
+		}
+		parentQueuePath, _ := getParentQueuePath(queuePath)
+		if parentQueuePath != "" {
+			qt := userTracker.queueTracker.getChildQueueTracker(parentQueuePath)
+			return m.internalEnsureGroup(user, qt.queuePath)
+		}
 	}
-	return "", fmt.Errorf("group is not available in usergroup for user %s", user.User)
+	return ""
 }
 
 func (m *Manager) isUserRemovable(ut *UserTracker) bool {
@@ -309,6 +315,7 @@ func (m *Manager) UpdateConfig(config configs.QueueConfig, queuePath string) err
 
 	m.userWildCardLimitsConfig = make(map[string]*LimitConfig)
 	m.groupWildCardLimitsConfig = make(map[string]*LimitConfig)
+	m.configuredGroups = make(map[string][]string)
 	return m.internalProcessConfig(config, queuePath)
 }
 
@@ -363,6 +370,7 @@ func (m *Manager) internalProcessConfig(cur configs.QueueConfig, queuePath strin
 			if err := m.processGroupConfig(group, limitConfig, queuePath, groupLimits); err != nil {
 				return err
 			}
+			m.configuredGroups[queuePath] = append(m.configuredGroups[queuePath], group)
 		}
 	}
 	if err := m.clearEarlierSetLimits(userLimits, groupLimits, queuePath); err != nil {
@@ -402,58 +410,100 @@ func (m *Manager) clearEarlierSetLimits(userLimits map[string]bool, groupLimits
 	for u, ut := range m.userTrackers {
 		// Is this user already tracked for the queue path?
 		if ut.IsQueuePathTrackedCompletely(queuePath) {
-			// Is there any limit config set for user in the current configuration? If not, then clear those old limit settings
-			if _, ok := userLimits[u]; !ok {
-				log.Log(log.SchedUGM).Debug("Need to clear earlier set configs for user",
-					zap.String("user", u),
-					zap.String("queue path", queuePath))
-				// Is there any running applications in end queue of this queue path? If not, then remove the linkage between end queue and its immediate parent
-				if ut.IsUnlinkRequired(queuePath) {
-					ut.UnlinkQT(queuePath)
-				} else {
-					ut.setLimits(queuePath, resources.NewResource(), 0)
-					log.Log(log.SchedUGM).Debug("Cleared earlier set limit configs for user",
-						zap.String("user", u),
-						zap.String("queue path", queuePath))
-				}
-				// Does "root" queue has any child queue trackers? At some point during this whole traversal, root might
-				// not have any child queue trackers. When the situation comes, remove the linkage between the user and
-				// its root queue tracker
-				if ut.canBeRemoved() {
-					delete(m.userTrackers, ut.userName)
+			// Traverse all the group trackers linked to user through different applications and remove the linkage
+			for appID, gt := range ut.appGroupTrackers {
+				if gt != nil {
+					g := gt.groupName
+					// Is there any limit config set for group in the current configuration? If not, then remove the linkage by setting it to nil
+					if ok := groupLimits[g]; !ok {
+						log.Log(log.SchedUGM).Debug("Removed the linkage between user and group through applications",
+							zap.String("user", u),
+							zap.String("group", gt.groupName),
+							zap.String("application id", appID),
+							zap.String("queue path", queuePath))
+						// removing the linkage only happens here by setting it to nil and deleting app id
+						// but group resource usage so far remains as it is because we don't have app id wise resource usage with in group as of now.
+						// YUNIKORN-1858 handles the group resource usage properly
+						// In case of only one (last) application, group tracker would be removed from the manager.
+						ut.setGroupForApp(appID, nil)
+						gt.removeApp(appID)
+						if len(gt.getTrackedApplications()) == 0 {
+							log.Log(log.SchedUGM).Debug("Is this app the only running application in group?",
+								zap.String("user", u),
+								zap.String("group", gt.groupName),
+								zap.Int("no. of applications", len(gt.getTrackedApplications())),
+								zap.String("application id", appID),
+								zap.String("queue path", queuePath))
+							delete(m.groupTrackers, g)
+						}
+					}
 				}
 			}
 		}
+		m.clearEarlierSetUserLimits(ut, queuePath, userLimits)
 	}
 
 	// Clear already configured limits of group for which limits have been configured before but not now
-	for g, gt := range m.groupTrackers {
-		// Is this group already tracked for the queue path?
-		if gt.IsQueuePathTrackedCompletely(queuePath) {
-			// Is there any limit config set for group in the current configuration? If not, then clear those old limit settings
-			if ok := groupLimits[g]; !ok {
-				log.Log(log.SchedUGM).Debug("Need to clear earlier set configs for group",
+	for _, gt := range m.groupTrackers {
+		m.clearEarlierSetGroupLimits(gt, queuePath, groupLimits)
+	}
+	return nil
+}
+
+func (m *Manager) clearEarlierSetUserLimits(ut *UserTracker, queuePath string, userLimits map[string]bool) {
+	// Is this user already tracked for the queue path?
+	if ut.IsQueuePathTrackedCompletely(queuePath) {
+		u := ut.userName
+		// Is there any limit config set for user in the current configuration? If not, then clear those old limit settings
+		if _, ok := userLimits[u]; !ok {
+			log.Log(log.SchedUGM).Debug("Need to clear earlier set configs for user",
+				zap.String("user", u),
+				zap.String("queue path", queuePath))
+			// Is there any running applications in end queue of this queue path? If not, then remove the linkage between end queue and its immediate parent
+			if ut.IsUnlinkRequired(queuePath) {
+				ut.UnlinkQT(queuePath)
+			} else {
+				ut.setLimits(queuePath, resources.NewResource(), 0)
+				log.Log(log.SchedUGM).Debug("Cleared earlier set limit configs for user",
+					zap.String("user", u),
+					zap.String("queue path", queuePath))
+			}
+			// Does "root" queue has any child queue trackers? At some point during this whole traversal, root might
+			// not have any child queue trackers. When the situation comes, remove the linkage between the user and
+			// its root queue tracker
+			if ut.canBeRemoved() {
+				delete(m.userTrackers, ut.userName)
+			}
+		}
+	}
+}
+
+func (m *Manager) clearEarlierSetGroupLimits(gt *GroupTracker, queuePath string, groupLimits map[string]bool) {
+	// Is this group already tracked for the queue path?
+	if gt.IsQueuePathTrackedCompletely(queuePath) {
+		g := gt.groupName
+		// Is there any limit config set for group in the current configuration? If not, then clear those old limit settings
+		if ok := groupLimits[g]; !ok {
+			log.Log(log.SchedUGM).Debug("Need to clear earlier set configs for group",
+				zap.String("group", g),
+				zap.String("queue path", queuePath))
+			// Is there any running applications in end queue of this queue path? If not, then remove the linkage between end queue and its immediate parent
+			if gt.IsUnlinkRequired(queuePath) {
+				gt.UnlinkQT(queuePath)
+			} else {
+				gt.setLimits(queuePath, resources.NewResource(), 0)
+				log.Log(log.SchedUGM).Debug("Cleared earlier set limit configs for group",
 					zap.String("group", g),
 					zap.String("queue path", queuePath))
-				// Is there any running applications in end queue of this queue path? If not, then remove the linkage between end queue and its immediate parent
-				if gt.IsUnlinkRequired(queuePath) {
-					gt.UnlinkQT(queuePath)
-				} else {
-					gt.setLimits(queuePath, resources.NewResource(), 0)
-					log.Log(log.SchedUGM).Debug("Cleared earlier set limit configs for group",
-						zap.String("group", g),
-						zap.String("queue path", queuePath))
-				}
-				// Does "root" queue has any child queue trackers? At some point during this whole traversal, root might
-				// not have any child queue trackers. When the situation comes, remove the linkage between the group and
-				// its root queue tracker
-				if gt.canBeRemoved() {
-					delete(m.groupTrackers, gt.groupName)
-				}
+			}
+			// Does "root" queue has any child queue trackers? At some point during this whole traversal, root might
+			// not have any child queue trackers. When the situation comes, remove the linkage between the group and
+			// its root queue tracker
+			if gt.canBeRemoved() {
+				delete(m.groupTrackers, gt.groupName)
 			}
 		}
 	}
-	return nil
 }
 
 func (m *Manager) setUserLimits(user string, limitConfig *LimitConfig, queuePath string) error {
@@ -554,8 +604,8 @@ func (m *Manager) Headroom(queuePath string, user security.UserGroup) *resources
 			zap.String("queue path", queuePath),
 			zap.String("user headroom", userHeadroom.String()))
 	}
-	group, err := m.getGroup(user)
-	if err == nil {
+	group := m.internalEnsureGroup(user, queuePath)
+	if group != "" {
 		if m.groupTrackers[group] != nil {
 			groupHeadroom = m.groupTrackers[group].headroom(queuePath)
 			log.Log(log.SchedUGM).Debug("Calculated headroom for group",
diff --git a/pkg/scheduler/ugm/manager_test.go b/pkg/scheduler/ugm/manager_test.go
index c8698fe6..ba283037 100644
--- a/pkg/scheduler/ugm/manager_test.go
+++ b/pkg/scheduler/ugm/manager_test.go
@@ -39,13 +39,117 @@ func TestUserManagerOnceInitialization(t *testing.T) {
 func TestGetGroup(t *testing.T) {
 	user := security.UserGroup{User: "test", Groups: []string{"test", "test1"}}
 	manager := GetUserManager()
-	group, err := manager.getGroup(user)
-	assert.NilError(t, err)
+
+	// create config with limits for "test" group and ensure picked up group is "test"
+	conf := createUpdateConfig(user.User, user.Groups[0])
+	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
+
+	group := manager.ensureGroup(user, "root.parent.leaf")
+	assert.Equal(t, group, "test")
+
+	// Even though ordering of groups has been changed in UserGroup obj, still "test" should be picked up
+	user1 := security.UserGroup{User: "test", Groups: []string{"test1", "test"}}
+	group = manager.ensureGroup(user1, "root.parent")
 	assert.Equal(t, group, "test")
-	user = security.UserGroup{User: "test", Groups: []string{}}
-	group, err = manager.getGroup(user)
-	assert.Equal(t, err.Error(), "group is not available in usergroup for user test")
+
+	// clear all configs and ensure there is no tracking for "test" group as there is no matching
+	conf = createConfigWithoutLimits()
+	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
+
+	usage1, err := resources.NewResourceFromConf(map[string]string{"memory": "5", "vcores": "5"})
+	assert.NilError(t, err)
+
+	increased := manager.IncreaseTrackedResource("root.parent.leaf", TestApp1, usage1, user1)
+	assert.Equal(t, increased, true)
+	assert.Equal(t, len(manager.userTrackers), 1)
+	assert.Equal(t, len(manager.groupTrackers), 0)
+
+	group = manager.ensureGroup(user1, "root.parent.leaf")
+	assert.Equal(t, group, "")
+	group = manager.ensureGroup(user1, "root.parent")
+	assert.Equal(t, group, "")
+	group = manager.ensureGroup(user1, "root")
 	assert.Equal(t, group, "")
+
+	// create config with groups as "test_root" for root queue path and "test_parent" for root.parent queue path but no group defined for
+	// "root.parent.leaf" queue path. ensure immediate parent queue path "root.parent" has been used for group matching and picked.
+	// Same group would be used even for all queues coming under "root.parent.*" hierarchy.
+	// But for queues under "root.parent1.*" hierarchy (including root.parent1 queue) , "test_root" would be used as there is no group defined for that hierarchical queue path
+	user = security.UserGroup{User: "test1", Groups: []string{"test", "test_root", "test_parent"}}
+	conf = createConfigWithDifferentGroups(user.User, user.Groups[0], "memory", "10", 50, 5)
+	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
+	group = manager.ensureGroup(user, "root.parent.leaf")
+	assert.Equal(t, group, "test_parent")
+	group = manager.ensureGroup(user, "root.parent")
+	assert.Equal(t, group, "test_parent")
+	group = manager.ensureGroup(user, "root")
+	assert.Equal(t, group, "test_root")
+	group = manager.ensureGroup(user, "root.parent1.leaf1")
+	assert.Equal(t, group, "test_root")
+
+	// create config with groups as "test_root" for root queue path and "test_parent" for root.parent queue path but no group defined for
+	// "root.parent.leaf" queue path. Passing just "root" as queue path returns "test_root" as searching starts from lowest level queue in entire queue path
+	conf = createConfigWithDifferentGroups(user.User, user.Groups[0], "memory", "10", 50, 5)
+	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
+	group = manager.ensureGroup(user, "root")
+	assert.Equal(t, group, "test_root")
+
+	// create config with groups as "test_root" for root queue path and "test_parent" for root.parent queue path but no group defined for
+	// "root.parent.leaf" queue path. since "test_parent" group is not defined in UserGroup obj, "test_root" would be matched and picked.
+	user = security.UserGroup{User: "test1", Groups: []string{"test", "test_root"}}
+	conf = createConfigWithDifferentGroups(user.User, user.Groups[0], "memory", "10", 50, 5)
+	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
+	group = manager.ensureGroup(user, "root.parent")
+	assert.Equal(t, group, "test_root")
+	group = manager.ensureGroup(user, "root")
+	assert.Equal(t, group, "test_root")
+
+	// do some activities with different users - user "test1" and "test2" belongs to "test_root" group
+	// ensure user "test1" and "test2" and its group linkage is intact through appGroupTrackers map
+	increased = manager.IncreaseTrackedResource("root.parent.leaf", TestApp1, usage1, user)
+	assert.Equal(t, increased, true)
+	assert.Equal(t, len(manager.userTrackers), 2)
+
+	user2 := security.UserGroup{User: "test2", Groups: []string{"test", "test_root"}}
+	increased = manager.IncreaseTrackedResource("root.parent.leaf", TestApp2, usage1, user2)
+	assert.Equal(t, increased, true)
+
+	userGroupTrackersAssertsMap := map[string]string{}
+	userGroupTrackersAssertsMap[user.User] = TestApp1
+	userGroupTrackersAssertsMap[user2.User] = TestApp2
+	for u, a := range userGroupTrackersAssertsMap {
+		ut := manager.getUserTracker(u, false)
+		actualGT := ut.appGroupTrackers[a]
+		expectedGT := manager.getGroupTracker("test_root", false)
+		assert.Equal(t, manager.getGroupTracker("test_root", false) != nil, true)
+		assert.Equal(t, actualGT, expectedGT)
+		assert.Equal(t, actualGT.groupName, "test_root")
+	}
+
+	assert.Equal(t, len(manager.userTrackers), 3)
+	assert.Equal(t, len(manager.groupTrackers), 2)
+
+	user3 := security.UserGroup{User: "test3", Groups: []string{"test"}}
+	increased = manager.IncreaseTrackedResource("root.parent.leaf", TestApp2, usage1, user3)
+	assert.Equal(t, increased, true)
+	assert.Equal(t, len(manager.userTrackers), 4)
+	ut := manager.getUserTracker(user3.User, false)
+	actualGT := ut.appGroupTrackers[TestApp2]
+	if actualGT != nil {
+		t.Errorf("group tracker should be nil as there is no group found for user %s", user3.User)
+	}
+
+	// create config without any limit settings for groups and
+	// since earlier group (test_root) limit settings has been cleared, ensure earlier user and group linkage through appGroupTrackers map has been removed
+	// and cleared as necessary
+	conf = createConfigWithDifferentGroups(user2.User, "", "memory", "10", 50, 5)
+	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
+	assert.Equal(t, len(manager.userTrackers), 4)
+	assert.Equal(t, len(manager.groupTrackers), 0)
+
+	ut = manager.getUserTracker(user.User, false)
+	assert.Equal(t, ut.appGroupTrackers[TestApp1] == nil, true)
+	assert.Equal(t, manager.getGroupTracker("test_root", false) == nil, true)
 }
 
 func TestAddRemoveUserAndGroups(t *testing.T) {
@@ -58,6 +162,8 @@ func TestAddRemoveUserAndGroups(t *testing.T) {
 		t.Errorf("new resource create returned error or wrong resource: error %t, res %v", err, usage1)
 	}
 	manager := GetUserManager()
+	manager.ClearUserTrackers()
+	manager.ClearGroupTrackers()
 
 	increased := manager.IncreaseTrackedResource("", "", usage1, user)
 	if increased {
@@ -72,12 +178,11 @@ func TestAddRemoveUserAndGroups(t *testing.T) {
 	userTrackers := manager.GetUsersResources()
 	userTracker := userTrackers[0]
 	groupTrackers := manager.GetGroupsResources()
-	groupTracker := groupTrackers[0]
+	assert.Equal(t, len(groupTrackers), 0)
 	assert.Equal(t, false, manager.isUserRemovable(userTracker))
-	assert.Equal(t, false, manager.isGroupRemovable(groupTracker))
 	assertUGM(t, user, usage1, 1)
 	assert.Equal(t, user.User, manager.GetUserTracker(user.User).userName)
-	assert.Equal(t, user.Groups[0], manager.GetGroupTracker(user.Groups[0]).groupName)
+	assert.Equal(t, manager.GetGroupTracker(user.Groups[0]) == nil, true)
 
 	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage1, user)
 	if !increased {
@@ -96,9 +201,7 @@ func TestAddRemoveUserAndGroups(t *testing.T) {
 	}
 	assertUGM(t, user1, usage2, 2)
 	assert.Equal(t, user.User, manager.GetUserTracker(user.User).userName)
-	assert.Equal(t, user.Groups[0], manager.GetGroupTracker(user.Groups[0]).groupName)
 	assert.Equal(t, user1.User, manager.GetUserTracker(user1.User).userName)
-	assert.Equal(t, user1.Groups[0], manager.GetGroupTracker(user1.Groups[0]).groupName)
 
 	assert.Equal(t, true, manager.GetUserTracker(user.User).queueTracker.IsQueuePathTrackedCompletely(queuePath1))
 	assert.Equal(t, true, manager.GetUserTracker(user1.User).queueTracker.IsQueuePathTrackedCompletely(queuePath2))
@@ -133,7 +236,7 @@ func TestAddRemoveUserAndGroups(t *testing.T) {
 		t.Fatalf("unable to decrease tracked resource: queuepath %s, app %s, res %v, error %t", queuePath1, TestApp1, usage3, err)
 	}
 	assert.Equal(t, 1, len(manager.GetUsersResources()), "userTrackers count should be 1")
-	assert.Equal(t, 1, len(manager.GetGroupsResources()), "groupTrackers count should be 1")
+	assert.Equal(t, 0, len(manager.GetGroupsResources()), "groupTrackers count should be 1")
 
 	decreased = manager.DecreaseTrackedResource(queuePath2, TestApp2, usage2, user1, true)
 	if !decreased {
@@ -173,6 +276,14 @@ func TestUpdateConfig(t *testing.T) {
 			t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v, error %t", queuePath1, TestApp1, usage, err)
 		}
 	}
+
+	userTrackers := manager.GetUsersResources()
+	userTracker := userTrackers[0]
+	groupTrackers := manager.GetGroupsResources()
+	groupTracker := groupTrackers[0]
+	assert.Equal(t, false, manager.isUserRemovable(userTracker))
+	assert.Equal(t, false, manager.isGroupRemovable(groupTracker))
+
 	// configure max resource for root.parent lesser than current resource usage. should be allowed to set but user cannot be allowed to do any activity further
 	conf = createConfig(user.User, user.Groups[0], "memory", "50", 40, 4)
 	err = manager.UpdateConfig(conf.Queues[0], "root")
@@ -228,80 +339,75 @@ func TestUpdateConfigWithWildCardUsersAndGroups(t *testing.T) {
 		}
 	}
 
-	// configure max resource for root.parent to allow one more application to run
+	// configure max resource for root.parent as map[memory:60 vcores:60] to allow one more application to run
 	conf = createConfig(user.User, user.Groups[0], "memory", "50", 60, 6)
 	err = manager.UpdateConfig(conf.Queues[0], "root")
 	assert.NilError(t, err, "unable to set the limit for user user1 because current resource usage is greater than config max resource for root.parent")
 
 	// should run as user 'user' setting is map[memory:60 vcores:60] and total usage of "root.parent" is map[memory:50 vcores:50]
-	increased := manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user)
+	increased := manager.IncreaseTrackedResource(queuePath1, TestApp2, usage, user)
 	if !increased {
 		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
 	}
 
 	// should not run as user 'user' setting is map[memory:60 vcores:60] and total usage of "root.parent" is map[memory:60 vcores:60]
-	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user)
+	increased = manager.IncreaseTrackedResource(queuePath1, TestApp3, usage, user)
 	if increased {
 		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
 	}
 
 	// configure max resource for root.parent to allow one more application to run through wild card user settings (not through specific user)
-	user1 := security.UserGroup{User: "user2", Groups: []string{"group2"}}
+	// configure limits for user2 only. However, user1 should not be cleared as it has running applications
+	user1 := security.UserGroup{User: "user2", Groups: []string{"group1"}}
 	conf = createUpdateConfigWithWildCardUsersAndGroups(user1.User, user1.Groups[0], "*", "*", "70", "70")
 	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
+	// ensure both user1 & group1 has not been removed from local maps
+	assert.Equal(t, manager.GetUserTracker(user.User) != nil, true)
+	assert.Equal(t, manager.GetGroupTracker(user.Groups[0]) != nil, true)
 
-	// should run as wild card user '*' setting is map[memory:70 vcores:70] and total usage of "root.parent" is map[memory:60 vcores:60]
-	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user)
+	// user1 still should be able to run app as wild card user '*' setting is map[memory:70 vcores:70] for "root.parent" and
+	// total usage of "root.parent" is map[memory:60 vcores:60]
+	increased = manager.IncreaseTrackedResource(queuePath1, TestApp2, usage, user)
 	if !increased {
-		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
+		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp2, user)
 	}
 
-	// should not run as wild card user '*' setting is map[memory:70 vcores:70] and total usage of "root.parent" is map[memory:70 vcores:70]
-	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user)
+	// user1 should not be able to run app as wild card user '*' setting is map[memory:70 vcores:70] for "root.parent"
+	// and total usage of "root.parent" is map[memory:70 vcores:70]
+	increased = manager.IncreaseTrackedResource(queuePath1, TestApp3, usage, user)
 	if increased {
-		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
+		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp3, user)
 	}
 
-	// configure max resource for root.parent to allow one more application to run through wild card group settings (not through specific group)
-	// also wild card user limit settings not set
-	conf = createUpdateConfigWithWildCardUsersAndGroups(user1.User, user1.Groups[0], "", "*", "80", "80")
+	// configure max resource for group1 * root.parent (map[memory:70 vcores:70]) higher than wild card group * root.parent settings (map[memory:10 vcores:10])
+	// ensure group's specific settings has been used for enforcement checks as specific limits always has higher precedence when compared to wild card group limit settings
+	conf = createUpdateConfigWithWildCardUsersAndGroups(user1.User, user1.Groups[0], "*", "*", "10", "10")
 	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
 
-	// should run as wild card group '*' setting is map[memory:80 vcores:80] and total usage of "root.parent" is map[memory:70 vcores:70]
-	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user)
-	if !increased {
-		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
-	}
-
-	// should not run as wild card group '*' setting is map[memory:80 vcores:80] and total usage of "root.parent" is map[memory:80 vcores:80]
-	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user)
-	if increased {
-		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
-	}
-
-	// should run as wild card group '*' setting is map[memory:80 vcores:80] and total usage of "root.parent" is map[memory:70 vcores:70]
+	// since group1 quota (map[memory:70 vcores:70]) has reached before, further group increase is not allowed.
+	// Hence, user's tracked usage would be reverted
 	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user1)
-	if !increased {
+	if increased {
 		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user1)
 	}
 
 	// configure max resource for user2 * root.parent (map[memory:70 vcores:70]) higher than wild card user * root.parent settings (map[memory:10 vcores:10])
-	// ensure user's specific settings overrides the wild card user limit settings
-	conf = createUpdateConfigWithWildCardUsersAndGroups(user1.User, user1.Groups[0], "", "*", "10", "10")
+	// ensure user's specific settings has been used for enforcement checks as specific limits always has higher precedence when compared to wild card user limit settings
+	conf = createUpdateConfigWithWildCardUsersAndGroups(user1.User, "", "*", "*", "10", "10")
 	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
 
 	// can be allowed to run upto resource usage map[memory:70 vcores:70]
-	for i := 1; i <= 6; i++ {
+	for i := 1; i <= 7; i++ {
 		increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user1)
 		if !increased {
-			t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
+			t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user1)
 		}
 	}
 
-	// should not run as user2 max limit is map[memory:70 vcores:70] and usage so far is map[memory:70 vcores:70]
+	// user2 should not be able to run app as user2 max limit is map[memory:70 vcores:70] and usage so far is map[memory:70 vcores:70]
 	increased = manager.IncreaseTrackedResource(queuePath1, TestApp1, usage, user1)
 	if increased {
-		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user)
+		t.Fatalf("unable to increase tracked resource: queuepath %s, app %s, res %v", queuePath1, TestApp1, user1)
 	}
 }
 
@@ -321,7 +427,8 @@ func TestUpdateConfigClearEarlierSetLimits(t *testing.T) {
 	}
 	assertMaxLimits(t, user, expectedResource, 5)
 
-	// create config user2 * root.parent with [50, 50] and maxapps as 5 (twice for root), but not user1. so user1 should not be there as it doesn't have any running applications
+	// create config user2 * root.parent with [50, 50] and maxapps as 5 (twice for root), but not user1.
+	// so user1 should not be there as it doesn't have any running applications
 	user1 := security.UserGroup{User: "user2", Groups: []string{"group2"}}
 	conf = createUpdateConfig(user1.User, user1.Groups[0])
 	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
@@ -344,8 +451,8 @@ func TestUpdateConfigClearEarlierSetLimits(t *testing.T) {
 	if err != nil {
 		t.Errorf("new resource create returned error or wrong resource: error %t, res %v", err, expectedResource)
 	}
-	assert.Equal(t, manager.GetUserTracker(user.User) == nil, true)
-	assert.Equal(t, manager.GetGroupTracker(user.Groups[0]) == nil, true)
+	assert.Equal(t, manager.GetUserTracker(user1.User) != nil, true)
+	assert.Equal(t, manager.GetGroupTracker(user1.Groups[0]) != nil, true)
 	assertMaxLimits(t, user1, expectedResource, 6)
 
 	expectedResource, err = resources.NewResourceFromConf(map[string]string{"memory": "70", "vcores": "70"})
@@ -357,13 +464,12 @@ func TestUpdateConfigClearEarlierSetLimits(t *testing.T) {
 	// and wild card settings
 	conf = createUpdateConfigWithWildCardUsersAndGroups(user1.User, user1.Groups[0], "*", "*", "10", "10")
 	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
-	assert.Equal(t, manager.GetUserTracker(user.User) == nil, true)
-	assert.Equal(t, manager.GetGroupTracker(user.Groups[0]) == nil, true)
+	assert.Equal(t, manager.GetUserTracker(user1.User) != nil, true)
+	assert.Equal(t, manager.GetGroupTracker(user1.Groups[0]) != nil, true)
 	assertMaxLimits(t, user1, expectedResource, 10)
 	assertWildCardLimits(t, m.userWildCardLimitsConfig, expectedResource)
 	assertWildCardLimits(t, m.groupWildCardLimitsConfig, expectedResource)
 
-	print("ggg")
 	// config without limits - should clear all earlier set configs
 	conf = createConfigWithoutLimits()
 	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
@@ -421,6 +527,13 @@ func TestSetMaxLimitsForRemovedUsers(t *testing.T) {
 	decreased = manager.DecreaseTrackedResource("root.parent.leaf", TestApp1, usage, user, true)
 	assert.Equal(t, decreased, true, "unable to decrease tracked resource: queuepath root.parent.leaf, app "+TestApp1+", res "+usage.String())
 
+	userTrackers := manager.GetUsersResources()
+	userTracker := userTrackers[0]
+	groupTrackers := manager.GetGroupsResources()
+	groupTracker := groupTrackers[0]
+	assert.Equal(t, true, manager.isUserRemovable(userTracker))
+	assert.Equal(t, true, manager.isGroupRemovable(groupTracker))
+
 	conf = createConfigWithoutLimits()
 	assert.NilError(t, manager.UpdateConfig(conf.Queues[0], "root"))
 
@@ -429,7 +542,7 @@ func TestSetMaxLimitsForRemovedUsers(t *testing.T) {
 		assert.Equal(t, increased, true, "unable to increase tracked resource: queuepath "+queuePath1+", app "+TestApp1+", res "+usage.String())
 	}
 	assert.Equal(t, manager.GetUserTracker(user.User) != nil, true)
-	assert.Equal(t, manager.GetGroupTracker(user.Groups[0]) != nil, true)
+	assert.Equal(t, manager.GetGroupTracker(user.Groups[0]) == nil, true)
 
 	decreased = manager.DecreaseTrackedResource(queuePath1, TestApp1, usage, user, false)
 	assert.Equal(t, decreased, true)
@@ -634,6 +747,66 @@ func createConfig(user string, group string, resourceKey string, resourceValue s
 	return conf
 }
 
+func createConfigWithDifferentGroups(user string, group string, resourceKey string, resourceValue string, mem int, maxApps uint64) configs.PartitionConfig {
+	var rootGroups []string
+	var parentGroups []string
+	if group != "" {
+		parentGroups = []string{
+			group + "_parent",
+		}
+		rootGroups = []string{
+			group + "_root",
+		}
+	}
+	conf := configs.PartitionConfig{
+		Name: "test",
+		Queues: []configs.QueueConfig{
+			{
+				Name:      "root",
+				Parent:    true,
+				SubmitACL: "*",
+				Queues: []configs.QueueConfig{
+					{
+						Name:      "parent",
+						Parent:    true,
+						SubmitACL: "*",
+						Queues:    nil,
+						Limits: []configs.Limit{
+							{
+								Limit: "parent queue limit",
+								Users: []string{
+									user,
+								},
+								Groups: parentGroups,
+								MaxResources: map[string]string{
+									"memory": strconv.Itoa(mem),
+									"vcores": strconv.Itoa(mem),
+								},
+								MaxApplications: maxApps,
+							},
+						},
+					},
+				},
+				Limits: []configs.Limit{
+					{
+						Limit: "root queue limit",
+						Users: []string{
+							user,
+						},
+						Groups: rootGroups,
+						MaxResources: map[string]string{
+							"memory": strconv.Itoa(mem * 2),
+							"vcores": strconv.Itoa(mem * 2),
+						},
+						MaxApplications: maxApps * 2,
+					},
+				},
+			},
+		},
+	}
+	return conf
+}
+
 func createConfigWithoutLimits() configs.PartitionConfig {
 	conf := configs.PartitionConfig{
 		Name: "test",
@@ -665,11 +838,11 @@ func setupUGM() {
 func assertUGM(t *testing.T, userGroup security.UserGroup, expected *resources.Resource, usersCount int) {
 	manager := GetUserManager()
 	assert.Equal(t, usersCount, len(manager.GetUsersResources()), "userTrackers count should be "+strconv.Itoa(usersCount))
-	assert.Equal(t, usersCount, len(manager.GetGroupsResources()), "groupTrackers count should be "+strconv.Itoa(usersCount))
+	assert.Equal(t, 0, len(manager.GetGroupsResources()), "groupTrackers count should be "+strconv.Itoa(0))
 	userRes := manager.GetUserResources(userGroup)
 	assert.Equal(t, resources.Equals(userRes, expected), true)
 	groupRes := manager.GetGroupResources(userGroup.Groups[0])
-	assert.Equal(t, resources.Equals(groupRes, expected), true)
+	assert.Equal(t, resources.Equals(groupRes, nil), true)
 }
 
 func assertMaxLimits(t *testing.T, userGroup security.UserGroup, expectedResource *resources.Resource, expectedMaxApps int) {
diff --git a/pkg/scheduler/ugm/queue_tracker.go b/pkg/scheduler/ugm/queue_tracker.go
index 0dd001bd..acc94ba1 100644
--- a/pkg/scheduler/ugm/queue_tracker.go
+++ b/pkg/scheduler/ugm/queue_tracker.go
@@ -152,6 +152,7 @@ func (qt *QueueTracker) increaseTrackedResource(queuePath string, applicationID
 	log.Log(log.SchedUGM).Debug("Successfully increased resource usage",
 		zap.Int("tracking type", int(trackType)),
 		zap.String("queue path", queuePath),
+		zap.String("qt queue path", qt.queuePath),
 		zap.String("application", applicationID),
 		zap.Bool("existing app", existingApp),
 		zap.Stringer("resource", usage),
diff --git a/pkg/scheduler/ugm/user_tracker.go b/pkg/scheduler/ugm/user_tracker.go
index 022f1881..17353424 100644
--- a/pkg/scheduler/ugm/user_tracker.go
+++ b/pkg/scheduler/ugm/user_tracker.go
@@ -21,7 +21,10 @@ package ugm
 import (
 	"sync"
 
+	"go.uber.org/zap"
+
 	"github.com/apache/yunikorn-core/pkg/common/resources"
+	"github.com/apache/yunikorn-core/pkg/log"
 	"github.com/apache/yunikorn-core/pkg/webservice/dao"
 )
 
@@ -51,7 +54,28 @@ func newUserTracker(user string) *UserTracker {
 func (ut *UserTracker) increaseTrackedResource(queuePath, applicationID string, usage *resources.Resource) bool {
 	ut.Lock()
 	defer ut.Unlock()
-	return ut.queueTracker.increaseTrackedResource(queuePath, applicationID, user, usage)
+	increased := ut.queueTracker.increaseTrackedResource(queuePath, applicationID, user, usage)
+	if increased {
+		gt := ut.appGroupTrackers[applicationID]
+		log.Log(log.SchedUGM).Debug("Increasing resource usage for group",
+			zap.String("group", gt.getName()),
+			zap.String("queue path", queuePath),
+			zap.String("application", applicationID),
+			zap.Stringer("resource", usage))
+		increasedGroupUsage := gt.increaseTrackedResource(queuePath, applicationID, usage)
+		if !increasedGroupUsage {
+			_, decreased := ut.queueTracker.decreaseTrackedResource(queuePath, applicationID, usage, false)
+			if !decreased {
+				log.Log(log.SchedUGM).Error("Problem in increasing group resource usage. Hence tried to rollback user tracked usage, but failed to do.",
+					zap.String("queue path", queuePath),
+					zap.String("application", applicationID),
+					zap.String("user", ut.userName),
+					zap.Stringer("usage", usage))
+			}
+		}
+		return increasedGroupUsage
+	}
+	return increased
 }
 
 func (ut *UserTracker) decreaseTrackedResource(queuePath, applicationID string, usage *resources.Resource, removeApp bool) (bool, bool) {
@@ -66,15 +90,23 @@ func (ut *UserTracker) decreaseTrackedResource(queuePath, applicationID string,
 func (ut *UserTracker) hasGroupForApp(applicationID string) bool {
 	ut.RLock()
 	defer ut.RUnlock()
-	return ut.appGroupTrackers[applicationID] != nil
+	_, ok := ut.appGroupTrackers[applicationID]
+	return ok
 }
 
 func (ut *UserTracker) setGroupForApp(applicationID string, groupTrack *GroupTracker) {
 	ut.Lock()
 	defer ut.Unlock()
-	if ut.appGroupTrackers[applicationID] == nil {
-		ut.appGroupTrackers[applicationID] = groupTrack
+	ut.appGroupTrackers[applicationID] = groupTrack
+}
+
+func (ut *UserTracker) getGroupForApp(applicationID string) string {
+	ut.Lock()
+	defer ut.Unlock()
+	if ut.appGroupTrackers[applicationID] != nil {
+		return ut.appGroupTrackers[applicationID].groupName
 	}
+	return ""
 }
 
 func (ut *UserTracker) getTrackedApplications() map[string]*GroupTracker {
@@ -103,7 +135,9 @@ func (ut *UserTracker) GetUserResourceUsageDAOInfo() *dao.UserResourceUsageDAOIn
 	}
 	userResourceUsage.UserName = ut.userName
 	for app, gt := range ut.appGroupTrackers {
-		userResourceUsage.Groups[app] = gt.groupName
+		if gt != nil {
+			userResourceUsage.Groups[app] = gt.groupName
+		}
 	}
 	userResourceUsage.Queues = ut.queueTracker.getResourceUsageDAOInfo("")
 	return userResourceUsage
diff --git a/pkg/scheduler/ugm/utilities.go b/pkg/scheduler/ugm/utilities.go
index 3b7795e4..3a2b2761 100644
--- a/pkg/scheduler/ugm/utilities.go
+++ b/pkg/scheduler/ugm/utilities.go
@@ -36,3 +36,16 @@ func getChildQueuePath(queuePath string) (string, string) {
 	}
 	return childQueuePath, childQueuePath[:idx]
 }
+
+func getParentQueuePath(queuePath string) (string, string) {
+	idx := strings.LastIndex(queuePath, configs.DOT)
+	if idx == -1 {
+		return "", ""
+	}
+	parentQueuePath := queuePath[:idx]
+	idx = strings.LastIndex(parentQueuePath, configs.DOT)
+	if idx == -1 {
+		return parentQueuePath, parentQueuePath
+	}
+	return parentQueuePath, parentQueuePath[idx+1:]
+}
diff --git a/pkg/scheduler/ugm/utilities_test.go b/pkg/scheduler/ugm/utilities_test.go
index 8e38ae1f..f31ea501 100644
--- a/pkg/scheduler/ugm/utilities_test.go
+++ b/pkg/scheduler/ugm/utilities_test.go
@@ -50,3 +50,17 @@ func TestGetChildQueuePath(t *testing.T) {
 	assert.Equal(t, childPath, "")
 	assert.Equal(t, immediateChildName, "")
 }
+
+func TestGetParentQueuePath(t *testing.T) {
+	parentPath, immediateParentName := getParentQueuePath("root.parent.leaf")
+	assert.Equal(t, parentPath, "root.parent")
+	assert.Equal(t, immediateParentName, "parent")
+
+	parentPath, immediateParentName = getParentQueuePath("parent.leaf")
+	assert.Equal(t, parentPath, "parent")
+	assert.Equal(t, immediateParentName, "parent")
+
+	parentPath, immediateParentName = getParentQueuePath("leaf")
+	assert.Equal(t, parentPath, "")
+	assert.Equal(t, immediateParentName, "")
+}
diff --git a/pkg/webservice/handlers_test.go b/pkg/webservice/handlers_test.go
index 0b1fba98..9db0d721 100644
--- a/pkg/webservice/handlers_test.go
+++ b/pkg/webservice/handlers_test.go
@@ -1281,8 +1281,7 @@ func TestSpecificUserAndGroupResourceUsage(t *testing.T) {
 	getGroupResourceUsage(resp, req)
 	err = json.Unmarshal(resp.outputBytes, &groupResourceUsageDao)
 	assert.NilError(t, err, "failed to unmarshal group resource usage dao response from response body: %s", string(resp.outputBytes))
-	assert.Equal(t, groupResourceUsageDao.Queues.ResourceUsage.String(),
-		resources.NewResourceFromMap(map[string]resources.Quantity{siCommon.CPU: 1}).String())
+	assert.DeepEqual(t, groupResourceUsageDao, &dao.GroupResourceUsageDAOInfo{GroupName: "", Applications: nil, Queues: nil})
 
 	// Test non-existing group query
 	req, err = http.NewRequest("GET", "/ws/v1/partition/default/usage/group/", strings.NewReader(""))
@@ -1323,15 +1322,14 @@ func TestUsersAndGroupsResourceUsage(t *testing.T) {
 	assert.NilError(t, err, "Get Groups Resource Usage Handler request failed")
 
 	var groupsResourceUsageDao []*dao.GroupResourceUsageDAOInfo
+	var expGroupsResourceUsageDao []*dao.GroupResourceUsageDAOInfo
 	getGroupsResourceUsage(resp, req)
 	err = json.Unmarshal(resp.outputBytes, &groupsResourceUsageDao)
 	assert.NilError(t, err, "failed to unmarshal groups resource usage dao response from response body: %s", string(resp.outputBytes))
-	assert.Equal(t, groupsResourceUsageDao[0].Queues.ResourceUsage.String(),
-		resources.NewResourceFromMap(map[string]resources.Quantity{siCommon.CPU: 1}).String())
+	assert.DeepEqual(t, groupsResourceUsageDao, expGroupsResourceUsageDao)
 
 	// Assert existing groups
-	assert.Equal(t, len(groupsResourceUsageDao), 1)
-	assert.Equal(t, groupsResourceUsageDao[0].GroupName, "testgroup")
+	assert.Equal(t, len(groupsResourceUsageDao), 0)
 }
 
 func TestGetEvents(t *testing.T) {


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@yunikorn.apache.org
For additional commands, e-mail: issues-help@yunikorn.apache.org