Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cursor based pagination for SCIM resources effort 2 #863

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

Cursor based pagination for SCIM resources effort 2 #863

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9affd23
Adjusting pre and post listeners for get user with ID when using curs…
BojithaPiyathilake May 26, 2022
168b97a
Adding cursor pagination for Oracle, DB2, MSSQL when identity claims …
BojithaPiyathilake Jun 17, 2022
710b296
Changing the type of direction attribute to enum
BojithaPiyathilake Jul 8, 2022
ae50fb0
Resolve merge conflicts
KD23243 Jul 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions .../src/main/java/org/wso2/carbon/identity/governance/listener/IdentityMgtEventListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,33 @@ public boolean doPreGetUserListWithID(Condition condition, String domain, String
return true;
}

@Override
public boolean doPreGetUserListWithID(Condition condition, String domain, String profileName, int limit,
String cursor, UserCoreConstants.PaginationDirection direction, String sortBy, String sortOrder,
UserStoreManager userStoreManager)
throws UserStoreException {

if (!isEnable()) {
return true;
}
if (log.isDebugEnabled()) {
log.debug("pre get user user list condition with id is called in IdentityMgtEventListener");
}
String eventName = IdentityEventConstants.Event.PRE_GET_USER_LIST_CONDITION_WITH_ID;
HashMap<String, Object> properties = new HashMap<>();
properties.put(IdentityEventConstants.EventProperty.CONDITION, condition);
properties.put(IdentityEventConstants.EventProperty.USER_STORE_DOMAIN, domain);
properties.put(IdentityEventConstants.EventProperty.PROFILE_NAME, profileName);
properties.put(IdentityEventConstants.EventProperty.LIMIT, limit);
properties.put(IdentityEventConstants.EventProperty.CURSOR, cursor);
properties.put(IdentityEventConstants.EventProperty.DIRECTION, direction);
properties.put(IdentityEventConstants.EventProperty.SORT_BY, sortBy);
properties.put(IdentityEventConstants.EventProperty.SORT_ORDER, sortOrder);

handleEvent(eventName, properties, userStoreManager);
return true;
}

@Override
public boolean doPreGetUserListWithID(String claimUri, String claimValue, int limit, int offset,
final List<User> returnUsersList, UserStoreManager userStoreManager) throws UserStoreException {
Expand Down Expand Up @@ -932,6 +959,33 @@ public boolean doPostGetUserListWithID(Condition condition, String domain, Strin
return true;
}

@Override
public boolean doPostGetUserListWithID(Condition condition, String domain, String profileName, int limit,
String cursor, UserCoreConstants.PaginationDirection direction, String sortBy, String sortOrder,
List<User> users, UserStoreManager userStoreManager)
throws UserStoreException {

if (!isEnable()) {
return true;
}
if (log.isDebugEnabled()) {
log.debug("post get user user list condition with id is called in IdentityMgtEventListener");
}
String eventName = IdentityEventConstants.Event.POST_GET_USER_LIST_CONDITION_WITH_ID;
HashMap<String, Object> properties = new HashMap<>();
properties.put(IdentityEventConstants.EventProperty.CONDITION, condition);
properties.put(IdentityEventConstants.EventProperty.USER_STORE_DOMAIN, domain);
properties.put(IdentityEventConstants.EventProperty.PROFILE_NAME, profileName);
properties.put(IdentityEventConstants.EventProperty.LIMIT, limit);
properties.put(IdentityEventConstants.EventProperty.CURSOR, cursor);
properties.put(IdentityEventConstants.EventProperty.DIRECTION, direction);
properties.put(IdentityEventConstants.EventProperty.SORT_BY, sortBy);
properties.put(IdentityEventConstants.EventProperty.SORT_ORDER, sortOrder);

handleEvent(eventName, properties, userStoreManager);
return true;
}

@Override
public boolean doPreGetUserWithID(String userID, String[] requestedClaims, String profileName,
UserStoreManager userStoreManager) throws UserStoreException {
Expand Down
30 changes: 30 additions & 0 deletions ...rc/main/java/org/wso2/carbon/identity/governance/listener/IdentityStoreEventListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.wso2.carbon.identity.governance.model.UserIdentityClaim;
import org.wso2.carbon.identity.governance.service.IdentityDataStoreService;
import org.wso2.carbon.identity.governance.store.UserIdentityDataStore;
import org.wso2.carbon.identity.governance.store.UserStoreBasedIdentityDataStore;
import org.wso2.carbon.user.core.UserCoreConstants;
import org.wso2.carbon.user.core.UserRealm;
import org.wso2.carbon.user.core.UserStoreException;
Expand Down Expand Up @@ -561,6 +562,35 @@ public boolean doPreGetPaginatedUserList(Condition condition, List<String> ident
return true;
}

public boolean doPreGetPaginatedUserList(Condition condition, List<String> identityClaimFilteredUserNames,
String domain, UserStoreManager userStoreManager, int limit, String cursor,
UserCoreConstants.PaginationDirection direction)
throws UserStoreException {

if (!isEnable()) {
return true;
}

// No need to separately handle if identity data store is user store based.
if (identityDataStore instanceof UserStoreBasedIdentityDataStore) {
return true;
}

List<ExpressionCondition> identityClaimFilterConditions = new ArrayList<>();
try {
// Extract identity Claim filter-conditions from the given conditions.
extractIdentityClaimFilterConditions(condition, identityClaimFilterConditions);
if (!identityClaimFilterConditions.isEmpty()) {
identityDataStore.listPaginatedUsersNames(identityClaimFilterConditions, identityClaimFilteredUserNames,
domain, userStoreManager, limit, cursor, direction);
}
} catch (IdentityException e) {
throw new UserStoreException("Error while listing the users for identity claim filters with pagination " +
"parameters.", e);
}
return true;
}

private void extractIdentityClaimFilterConditions(Condition condition,
List<ExpressionCondition> expressionConditions) {

Expand Down
126 changes: 112 additions & 14 deletions ...rnance/src/main/java/org/wso2/carbon/identity/governance/store/JDBCIdentityDataStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,8 +351,8 @@ public List<String> listPaginatedUsersNames(List<ExpressionCondition> identityCl
offset = offset - 1;
}

SqlBuilder sqlBuilder = getQueryString(identityClaimFilterExpressionConditions, limit, offset, domain,
tenantId, dBType);
SqlBuilder sqlBuilder = getQueryString(identityClaimFilterExpressionConditions, limit, offset,
null, null, domain, tenantId, dBType);

String fullQuery = sqlBuilder.getQuery();
int startIndex = 0;
Expand Down Expand Up @@ -464,6 +464,51 @@ public List<String> getUserNamesBetweenProvidedClaimValues(String claimURI, Str
}


@Override
public List<String> listPaginatedUsersNames(List<ExpressionCondition> identityClaimFilterExpressionConditions,
List<String> identityClaimFilteredUserNames, String domain,
org.wso2.carbon.user.core.UserStoreManager userStoreManager,
int limit, String cursor, UserCoreConstants.PaginationDirection direction)
throws IdentityException {

try {
int tenantId = userStoreManager.getTenantId();
try (Connection connection = IdentityDatabaseUtil.getDBConnection()) {
// Based on the DB Type might need to extend support.
String dBType = DatabaseCreator.getDatabaseType(connection);
SqlBuilder sqlBuilder = getQueryString(identityClaimFilterExpressionConditions, limit, null,
cursor, direction, domain, tenantId, dBType);
String fullQuery = sqlBuilder.getQuery();
int startIndex = 0;
int endIndex = 0;
int occurrence = StringUtils.countMatches(fullQuery, QUERY_BINDING_SYMBOL);
endIndex = endIndex + occurrence;
try (PreparedStatement preparedStatement = connection.prepareStatement(fullQuery)) {
populatePrepareStatement(sqlBuilder, preparedStatement, startIndex, endIndex);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
identityClaimFilteredUserNames.add(resultSet.getString("USER_NAME"));
}
IdentityDatabaseUtil.commitTransaction(connection);
} catch (SQLException e) {
if (log.isDebugEnabled()) {
log.debug("Error occurred while retrieving users from Identity Store for " + domain +
"with limit " + limit + "and cursor " + cursor, e);
}
IdentityDatabaseUtil.rollbackTransaction(connection);
}
} catch (SQLException e) {
throw new IdentityException("Error occurred while retrieving users from Identity Store.", e);
}
return identityClaimFilteredUserNames;
} catch (Exception e) {
throw new IdentityException("Error occurred while retrieving users from Identity Store.", e);
}
} catch (org.wso2.carbon.user.core.UserStoreException e) {
throw new IdentityException("Error occurred while retrieving users.", e);
}
}

private void populatePrepareStatement(SqlBuilder sqlBuilder, PreparedStatement prepStmt, int startIndex,
int endIndex) throws SQLException {

Expand Down Expand Up @@ -506,7 +551,8 @@ private void populatePrepareStatement(SqlBuilder sqlBuilder, PreparedStatement p
}

private SqlBuilder getQueryString(List<ExpressionCondition> identityClaimFilterExpressionConditions,
int limit, int offset, String userStoreDomain, int tenantID, String dbType) {
int limit, Integer offset, String cursor, UserCoreConstants.PaginationDirection direction,
String userStoreDomain, int tenantID, String dbType) {

boolean hitClaimFilter = false;
String userNameWithDomain;
Expand All @@ -522,7 +568,25 @@ private SqlBuilder getQueryString(List<ExpressionCondition> identityClaimFilterE
userStoreDomain.toUpperCase() + UserCoreConstants.DOMAIN_SEPARATOR + SQL_FILTER_STRING_ANY;
sqlBuilder.where(" USER_NAME LIKE ? ", userNameWithDomain);
}

if (cursor != null) {
if (UserCoreConstants.PaginationDirection.PREVIOUS == direction) {
if (StringUtils.equalsIgnoreCase(userStoreDomain, UserCoreConstants.PRIMARY_DEFAULT_DOMAIN_NAME)) {
sqlBuilder.where("USER_NAME < ?", cursor);
} else {
//Identity datastores store users in a secondary user store as DOMAIN/USER_NAME
String userDomain = userStoreDomain.toUpperCase() + UserCoreConstants.DOMAIN_SEPARATOR;
sqlBuilder.where("USER_NAME < ?", userDomain + cursor);
}
} else {
if (StringUtils.equalsIgnoreCase(userStoreDomain, UserCoreConstants.PRIMARY_DEFAULT_DOMAIN_NAME)) {
sqlBuilder.where("USER_NAME > ?", cursor);
} else {
//Identity datastores store users in a secondary user store as DOMAIN/USER_NAME
String userDomain = userStoreDomain.toUpperCase() + UserCoreConstants.DOMAIN_SEPARATOR;
sqlBuilder.where("USER_NAME > ?", userDomain + cursor);
}
}
}
SqlBuilder header = new SqlBuilder(new StringBuilder(sqlBuilder.getSql()));
addingWheres(sqlBuilder, header);

Expand All @@ -540,18 +604,52 @@ private SqlBuilder getQueryString(List<ExpressionCondition> identityClaimFilterE
hitClaimFilter = true;
}

if (DB2.equals(dbType)) {
sqlBuilder.setTail(" ORDER BY USER_NAME LIMIT ? , ? ", limit, offset);
} else if (MSSQL.equals(dbType)) {
sqlBuilder.setTail(" ORDER BY USER_NAME OFFSET ? ROWS FETCH NEXT ? ROWS ONLY ", offset, limit);
} else if (ORACLE.equals(dbType)) {
sqlBuilder.setTail(" ORDER BY USER_NAME OFFSET ? ROWS FETCH NEXT ? ROWS ONLY ", offset, limit);
} else if (POSTGRE_SQL.equals(dbType)) {
sqlBuilder.setTail(" ORDER BY USER_NAME OFFSET ? ROWS FETCH NEXT ? ROWS ONLY ", offset, limit);
} else {
sqlBuilder.setTail(" ORDER BY USER_NAME ASC LIMIT ? OFFSET ?", limit, offset);
// SQL to cover the whole query and reverse the result when querying for the previous page which will be in
// DESC order.
if (UserCoreConstants.PaginationDirection.PREVIOUS == direction) {
sqlBuilder.prependSql("SELECT * FROM ( ");
}

if (cursor != null) {
if (DB2.equals(dbType)) {
if (UserCoreConstants.PaginationDirection.PREVIOUS == direction) {
sqlBuilder.setTail(" ORDER BY USER_NAME DESC LIMIT ? " +
") AS results ORDER BY results.USER_NAME ASC", limit);
} else {
sqlBuilder.setTail(" ORDER BY USER_NAME LIMIT ?", limit);
}
} else if (ORACLE.equals(dbType) || POSTGRE_SQL.equals(dbType)) {
if (UserCoreConstants.PaginationDirection.PREVIOUS == direction) {
sqlBuilder.setTail(" ORDER BY USER_NAME DESC FETCH NEXT ? ROWS ONLY" +
") results ORDER BY results.USER_NAME ASC", limit);
} else {
sqlBuilder.setTail(" ORDER BY USER_NAME FETCH NEXT ? ROWS ONLY", limit);
}
} else if (MSSQL.equals(dbType)) {
// Even with cursor pagination we must use the OFFSET keyword as it is compulsory to LIMIT rows in MSSQL
if (UserCoreConstants.PaginationDirection.PREVIOUS == direction) {
sqlBuilder.setTail(" ORDER BY USER_NAME DESC OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY" +
") AS results ORDER BY results.USER_NAME ASC", limit);
} else {
sqlBuilder.setTail(" ORDER BY USER_NAME OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY", limit);
}
} else {
if (UserCoreConstants.PaginationDirection.PREVIOUS == direction) {
sqlBuilder.setTail(" ORDER BY USER_NAME DESC LIMIT ? " +
") AS results ORDER BY results.USER_NAME ASC;", limit);
} else {
sqlBuilder.setTail(" ORDER BY USER_NAME ASC LIMIT ?", limit);
}
}
} else {
if (DB2.equals(dbType)) {
sqlBuilder.setTail(" ORDER BY USER_NAME LIMIT ? , ? ", offset, limit);
} else if (MSSQL.equals(dbType) || ORACLE.equals(dbType) || POSTGRE_SQL.equals(dbType)) {
sqlBuilder.setTail(" ORDER BY USER_NAME OFFSET ? ROWS FETCH NEXT ? ROWS ONLY ", offset, limit);
} else {
sqlBuilder.setTail(" ORDER BY USER_NAME ASC LIMIT ? OFFSET ?", limit, offset);
}
}
return sqlBuilder;
}

Expand Down
22 changes: 22 additions & 0 deletions ...rnance/src/main/java/org/wso2/carbon/identity/governance/store/UserIdentityDataStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,28 @@ public List<String> listPaginatedUsersNames(List<ExpressionCondition> expression
return Collections.emptyList();
}

/**
* List users according to the given claim URI and value and pagination parameters.
*
* @param expressionConditions List of expression conditions.
* @param identityClaimFilteredUserNames List to hold filtered usernames.
* @param domain Userstore domain
* @param userStoreManager UserStoreManager instance.
* @param limit Number of users to return
* @param cursor Cursor value for cursor-based pagination
* @param direction Direction of pagination
* @return List of usernames.
* @throws IdentityException Identity Exception.
*/
public List<String> listPaginatedUsersNames(List<ExpressionCondition> expressionConditions,
List<String> identityClaimFilteredUserNames, String domain,
org.wso2.carbon.user.core.UserStoreManager userStoreManager, int limit, String cursor,
UserCoreConstants.PaginationDirection direction) throws IdentityException {

// Return an immutable empty list if sub classes do not have any overrides.
return Collections.emptyList();
}

/**
* Get the list of usernames who have the claim value less than the provided claim value for a given claim URI.
*
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -688,14 +688,14 @@
<identity.data.publisher.authentication.version>5.3.27</identity.data.publisher.authentication.version>

<!--Carbon Kernel Version-->
<carbon.kernel.version>4.9.0</carbon.kernel.version>
<carbon.kernel.version>4.10.17-SNAPSHOT</carbon.kernel.version>
<carbon.kernel.feature.version>4.9.0</carbon.kernel.feature.version>
<carbon.kernel.package.import.version.range>[4.5.0, 5.0.0)</carbon.kernel.package.import.version.range>
<carbon.user.api.imp.pkg.version.range>[1.0.1, 2.0.0)</carbon.user.api.imp.pkg.version.range>
<carbon.kernel.registry.imp.pkg.version.range>[1.0.1, 2.0.0)</carbon.kernel.registry.imp.pkg.version.range>

<!--Carbon Identity Framework Version-->
<carbon.identity.framework.version>7.3.15</carbon.identity.framework.version>
<carbon.identity.framework.version>7.3.22-SNAPSHOT</carbon.identity.framework.version>
<carbon.identity.framework.imp.pkg.version.range>[7.3.6, 8.0.0)
</carbon.identity.framework.imp.pkg.version.range>

Expand Down
Loading