RANGER-3899: performance improvement for policies with many users/groups/roles (ranger-2.9)

dev-support/checks/coverage.sh

@@ -49,6 +49,10 @@ find . -type d -name 'target' -prune -exec find {} -type f \( -name 'ranger-*.ja
 -or -name '*shim*' -prune \
 | xargs -n1 unzip -o -q -d target/coverage-classes
 
+# Multi-release JARs (e.g. BouncyCastle) ship the same classes under
+# META-INF/versions/* and at the root; JaCoCo fails with duplicate class names.
+rm -rf target/coverage-classes/META-INF/versions || true
+
 # get all source file paths
 src=$(find . -path '*/src/main/java' -o -path './target' -prune | sed 's/^/--sourcefiles /g' | xargs echo)
 

security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java

@@ -2132,7 +2132,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, boolean createPrincipalsIf
 		}
 
 		XXPolicy xCreatedPolicy = daoMgr.getXXPolicy().getById(policy.getId());
-		policyRefUpdater.createNewPolMappingForRefTable(policy, xCreatedPolicy, xServiceDef, createPrincipalsIfAbsent);
+		policyRefUpdater.createNewPolMappingForRefTable(policy, xCreatedPolicy, xServiceDef, createPrincipalsIfAbsent, false);
 		createOrMapLabels(xCreatedPolicy, uniquePolicyLabels);
 		RangerPolicy createdPolicy = policyService.getPopulatedViewObject(xCreatedPolicy);
 
@@ -2315,10 +2315,9 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception {
 		policy = policyService.update(policy);
 		XXPolicy newUpdPolicy = daoMgr.getXXPolicy().getById(policy.getId());
 
-		policyRefUpdater.cleanupRefTables(policy);
 		deleteExistingPolicyLabel(policy);
 
-		policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef, bizUtil.getCreatePrincipalsIfAbsent());
+		policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef, bizUtil.getCreatePrincipalsIfAbsent(), true);
 		createOrMapLabels(newUpdPolicy, uniquePolicyLabels);
 		RangerPolicy updPolicy = policyService.getPopulatedViewObject(newUpdPolicy);
 

security-admin/src/main/java/org/apache/ranger/db/XXAccessTypeDefDao.java

@@ -18,16 +18,25 @@
 package org.apache.ranger.db;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.persistence.NoResultException;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.ranger.common.db.BaseDao;
 import org.apache.ranger.entity.XXAccessTypeDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 @Service
 public class XXAccessTypeDefDao extends BaseDao<XXAccessTypeDef> {
+	private static final Logger logger = LoggerFactory.getLogger(XXAccessTypeDefDao.class);
 
 	public XXAccessTypeDefDao(RangerDaoManagerBase daoManager) {
 		super(daoManager);
@@ -60,4 +69,22 @@ public XXAccessTypeDef findByNameAndServiceId(String name, Long serviceId) {
 			return null;
 		}
 	}
+
+	public Map<String, Long> findAccessTypeDefIdsByNamesAndServiceId(Set<String> names, Long serviceId) {
+		if (serviceId != null && CollectionUtils.isNotEmpty(names)) {
+			try {
+				Collection<Object[]> result = getEntityManager()
+						.createNamedQuery("XXAccessTypeDef.findAccessTypeDefIdsByNamesAndServiceId", Object[].class)
+						.setParameter("names", names)
+						.setParameter("serviceId", serviceId)
+						.getResultList();
+
+				return result.stream().collect(Collectors.toMap(object -> (String) object[1], object -> (Long) object[0], (a, b) -> a));
+			} catch (NoResultException e) {
+				logger.debug(e.getMessage());
+			}
+		}
+
+		return Collections.emptyMap();
+	}
 }

security-admin/src/main/java/org/apache/ranger/db/XXDataMaskTypeDefDao.java

@@ -18,16 +18,25 @@
 package org.apache.ranger.db;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.persistence.NoResultException;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.ranger.common.db.BaseDao;
 import org.apache.ranger.entity.XXDataMaskTypeDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 @Service
 public class XXDataMaskTypeDefDao extends BaseDao<XXDataMaskTypeDef> {
+	private static final Logger logger = LoggerFactory.getLogger(XXDataMaskTypeDefDao.class);
 
 	public XXDataMaskTypeDefDao(RangerDaoManagerBase daoManager) {
 		super(daoManager);
@@ -60,4 +69,22 @@ public XXDataMaskTypeDef findByNameAndServiceId(String name, Long serviceId) {
 			return null;
 		}
 	}
+
+	public Map<String, Long> findDataMaskTypeDefIdsByNamesAndServiceId(Set<String> names, Long serviceId) {
+		if (serviceId != null && CollectionUtils.isNotEmpty(names)) {
+			try {
+				Collection<Object[]> result = getEntityManager()
+						.createNamedQuery("XXDataMaskTypeDef.findDataMaskTypeDefIdsByNamesAndServiceId", Object[].class)
+						.setParameter("names", names)
+						.setParameter("serviceId", serviceId)
+						.getResultList();
+
+				return result.stream().collect(Collectors.toMap(object -> (String) object[1], object -> (Long) object[0], (a, b) -> a));
+			} catch (NoResultException e) {
+				logger.debug(e.getMessage());
+			}
+		}
+
+		return Collections.emptyMap();
+	}
 }

security-admin/src/main/java/org/apache/ranger/db/XXGroupDao.java

@@ -21,10 +21,14 @@
 
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.ranger.authorization.utils.JsonUtils;
 import org.apache.ranger.common.RangerCommonEnums;
@@ -117,6 +121,23 @@ public List<GroupInfo> getAllGroupsInfo() {
 		return ret;
 	}
 
+	public Map<String, Long> getIdsByGroupNames(Collection<String> groupNames) {
+		if (CollectionUtils.isNotEmpty(groupNames)) {
+			try {
+				Collection<Object[]> result = getEntityManager()
+						.createNamedQuery("XXGroup.getIdsByGroupNames", Object[].class)
+						.setParameter("names", groupNames)
+						.getResultList();
+
+				return result.stream().collect(Collectors.toMap(object -> (String) (object[1]), object -> (Long) (object[0])));
+			} catch (NoResultException excp) {
+				logger.debug(excp.getMessage());
+			}
+		}
+
+		return Collections.emptyMap();
+	}
+
 	private GroupInfo toGroupInfo(Object[] row) {
 		String              name        = (String) row[0];
 		String              description = (String) row[1];

security-admin/src/main/java/org/apache/ranger/db/XXPolicyConditionDefDao.java

@@ -18,16 +18,25 @@
 package org.apache.ranger.db;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.persistence.NoResultException;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.ranger.common.db.BaseDao;
 import org.apache.ranger.entity.XXPolicyConditionDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 @Service
 public class XXPolicyConditionDefDao extends BaseDao<XXPolicyConditionDef> {
+	private static final Logger logger = LoggerFactory.getLogger(XXPolicyConditionDefDao.class);
 
 	public XXPolicyConditionDefDao(RangerDaoManagerBase daoManager) {
 		super(daoManager);
@@ -61,5 +70,22 @@ public XXPolicyConditionDef findByServiceDefIdAndName(Long serviceDefId, String
 			return null;
 		}
 	}
-
+
+	public Map<String, Long> findConditionDefIdsByServiceDefIdAndNames(Long serviceDefId, Set<String> names) {
+		if (serviceDefId != null && CollectionUtils.isNotEmpty(names)) {
+			try {
+				Collection<Object[]> result = getEntityManager()
+						.createNamedQuery("XXPolicyConditionDef.findConditionDefIdsByServiceDefIdAndNames", Object[].class)
+						.setParameter("serviceDefId", serviceDefId)
+						.setParameter("names", names)
+						.getResultList();
+
+				return result.stream().collect(Collectors.toMap(object -> (String) object[1], object -> (Long) object[0], (a, b) -> a));
+			} catch (NoResultException e) {
+				logger.debug(e.getMessage());
+			}
+		}
+
+		return Collections.emptyMap();
+	}
 }

security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java

@@ -20,8 +20,11 @@
 package org.apache.ranger.db;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import javax.persistence.NoResultException;
 
@@ -115,4 +118,30 @@ public void deleteByPolicyId(Long policyId) {
 
 		batchDeleteByIds("XXPolicyRefGroup.deleteByIds", ids, "ids");
 	}
+
+	public Map<String, Long> findGroupNameByPolicyId(Long policyId) {
+		Map<String, Long> ret = Collections.emptyMap();
+		if (policyId != null) {
+			try {
+				Collection<Object[]> results = getEntityManager()
+						.createNamedQuery("XXPolicyRefGroup.findGroupNameByPolicyId", Object[].class)
+						.setParameter("policyId", policyId)
+						.getResultList();
+				ret = results.stream().collect(
+						Collectors.toMap(
+								object -> (String) object[0],
+								object -> (Long) object[1]));
+			} catch (NoResultException e) {
+				// ignore
+			}
+		}
+		return ret;
+	}
+
+	public void deletePolicyRefGroupByIds(List<Long> ids) {
+		if (CollectionUtils.isEmpty(ids)) {
+			return;
+		}
+		batchDeleteByIds("XXPolicyRefGroup.deletePolicyRefGroupByIds", ids, "ids");
+	}
 }

security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java

@@ -20,8 +20,11 @@
 package org.apache.ranger.db;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import javax.persistence.NoResultException;
 
@@ -127,5 +130,31 @@ public void deleteByPolicyId(Long policyId) {
 
         batchDeleteByIds("XXPolicyRefRole.deleteByIds", ids, "ids");
 	}
+
+	public Map<String, Long> findRoleNameIdByPolicyId(Long policyId) {
+		Map<String, Long> ret = Collections.emptyMap();
+		if (policyId != null) {
+			try {
+				Collection<Object[]> results = getEntityManager()
+						.createNamedQuery("XXPolicyRefRole.findRoleNameIdByPolicyId", Object[].class)
+						.setParameter("policyId", policyId)
+						.getResultList();
+				ret = results.stream().collect(
+						Collectors.toMap(
+								object -> (String) object[0],
+								object -> (Long) object[1]));
+			} catch (NoResultException e) {
+				// ignore
+			}
+		}
+		return ret;
+	}
+
+	public void deletePolicyRefRoleByIds(List<Long> ids) {
+		if (CollectionUtils.isEmpty(ids)) {
+			return;
+		}
+		batchDeleteByIds("XXPolicyRefRole.deletePolicyRefRoleByIds", ids, "ids");
+	}
 }
 

security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java

@@ -20,8 +20,11 @@
 package org.apache.ranger.db;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import javax.persistence.NoResultException;
 
@@ -127,4 +130,30 @@ public void deleteByPolicyId(Long policyId) {
 
 		batchDeleteByIds("XXPolicyRefUser.deleteByIds", ids, "ids");
 	}
+
+	public Map<String, Long> findUserNameIdByPolicyId(Long policyId) {
+		Map<String, Long> ret = Collections.emptyMap();
+		if (policyId != null) {
+			try {
+				Collection<Object[]> results = getEntityManager()
+						.createNamedQuery("XXPolicyRefUser.findUserNameIdByPolicyId", Object[].class)
+						.setParameter("policyId", policyId)
+						.getResultList();
+				ret = results.stream().collect(
+						Collectors.toMap(
+								object -> (String) object[0],
+								object -> (Long) object[1]));
+			} catch (NoResultException e) {
+				// ignore
+			}
+		}
+		return ret;
+	}
+
+	public void deletePolicyRefUserByIds(List<Long> ids) {
+		if (CollectionUtils.isEmpty(ids)) {
+			return;
+		}
+		batchDeleteByIds("XXPolicyRefUser.deletePolicyRefUserByIds", ids, "ids");
+	}
 }

security-admin/src/main/java/org/apache/ranger/db/XXResourceDefDao.java

@@ -18,16 +18,25 @@
 package org.apache.ranger.db;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.persistence.NoResultException;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.ranger.common.db.BaseDao;
 import org.apache.ranger.entity.XXResourceDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 @Service
 public class XXResourceDefDao extends BaseDao<XXResourceDef> {
+	private static final Logger logger = LoggerFactory.getLogger(XXResourceDefDao.class);
 
 	public XXResourceDefDao(RangerDaoManagerBase daoMgr) {
 		super(daoMgr);
@@ -99,4 +108,22 @@ public List<XXResourceDef> findByParentResId(Long parentId) {
 			return new ArrayList<XXResourceDef>();
 		}
 	}
+
+	public Map<String, Long> findResourceDefIdsByNameAndPolicyId(Set<String> names, Long policyId) {
+		if (policyId != null && CollectionUtils.isNotEmpty(names)) {
+			try {
+				Collection<Object[]> result = getEntityManager()
+						.createNamedQuery("XXResourceDef.findResourceDefIdsByNameAndPolicyId", Object[].class)
+						.setParameter("policyId", policyId)
+						.setParameter("names", names)
+						.getResultList();
+
+				return result.stream().collect(Collectors.toMap(object -> (String) object[1], object -> (Long) object[0], (a, b) -> a));
+			} catch (NoResultException e) {
+				logger.debug(e.getMessage());
+			}
+		}
+
+		return Collections.emptyMap();
+	}
 }