You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@syncope.apache.org by il...@apache.org on 2019/09/04 12:35:21 UTC
[syncope] branch 2_1_X updated: [SYNCOPE-1485] Reduce temporary
result cache
This is an automated email from the ASF dual-hosted git repository.
ilgrosso pushed a commit to branch 2_1_X
in repository https://gitbox.apache.org/repos/asf/syncope.git
The following commit(s) were added to refs/heads/2_1_X by this push:
new f7a620c [SYNCOPE-1485] Reduce temporary result cache
f7a620c is described below
commit f7a620cb295c6cc6af49f14d13e111e391de7063
Author: Francesco Chicchiriccò <il...@apache.org>
AuthorDate: Wed Sep 4 14:05:57 2019 +0200
[SYNCOPE-1485] Reduce temporary result cache
---
.../syncope/core/persistence/api/dao/AnyDAO.java | 9 +++++++++
.../core/persistence/jpa/dao/AbstractAnyDAO.java | 14 ++++++++++++++
.../core/persistence/jpa/dao/JPAAnyObjectDAO.java | 5 +++++
.../core/persistence/jpa/dao/JPAGroupDAO.java | 5 +++++
.../core/persistence/jpa/dao/JPAUserDAO.java | 4 ++++
.../core/persistence/jpa/inner/AnyObjectTest.java | 13 +++++++++++++
.../core/persistence/jpa/inner/GroupTest.java | 9 +++++++--
.../core/persistence/jpa/inner/UserTest.java | 9 +++++++--
.../java/job/ElasticsearchReindex.java | 22 +++++++++-------------
9 files changed, 73 insertions(+), 17 deletions(-)
diff --git a/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/AnyDAO.java b/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/AnyDAO.java
index a55abf5..2300314 100644
--- a/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/AnyDAO.java
+++ b/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/AnyDAO.java
@@ -79,6 +79,15 @@ public interface AnyDAO<A extends Any<?>> extends DAO<A> {
*/
List<A> findAll(int page, int itemsPerPage);
+ /**
+ * Find any objects' keys without any limitation, according to given page and items per page.
+ *
+ * @param page search result page
+ * @param itemsPerPage items per search result page
+ * @return any objects' keys matching the provided conditions
+ */
+ List<String> findAllKeys(int page, int itemsPerPage);
+
<S extends Schema> AllowedSchemas<S> findAllowedSchemas(A any, Class<S> reference);
A save(A any);
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/AbstractAnyDAO.java b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/AbstractAnyDAO.java
index f440124..ff4c43e 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/AbstractAnyDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/AbstractAnyDAO.java
@@ -114,6 +114,20 @@ public abstract class AbstractAnyDAO<A extends Any<?>> extends AbstractDAO<A> im
return key;
}
+ protected List<String> findAllKeys(final String table, final int page, final int itemsPerPage) {
+ Query query = entityManager().createNativeQuery(
+ "SELECT id FROM " + table + " ORDER BY id", String.class);
+ query.setFirstResult(itemsPerPage * (page <= 0 ? 0 : page - 1));
+ query.setMaxResults(itemsPerPage);
+
+ List<String> result = new ArrayList<>();
+ query.getResultList().stream().map(resultKey -> resultKey instanceof Object[]
+ ? (String) ((Object[]) resultKey)[0]
+ : ((String) resultKey)).
+ forEach(actualKey -> result.add(actualKey.toString()));
+ return result;
+ }
+
protected Date findLastChange(final String key, final String table) {
Query query = entityManager().createNativeQuery(
"SELECT creationDate, lastChangeDate FROM " + table + " WHERE id=?");
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java
index 18eab7a..26a2a12 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java
@@ -195,6 +195,11 @@ public class JPAAnyObjectDAO extends AbstractAnyDAO<AnyObject> implements AnyObj
return query.getResultList();
}
+ @Override
+ public List<String> findAllKeys(final int page, final int itemsPerPage) {
+ return findAllKeys(JPAAnyObject.TABLE, page, itemsPerPage);
+ }
+
protected Pair<AnyObject, Pair<Set<String>, Set<String>>> doSave(final AnyObject anyObject) {
AnyObject merged = super.save(anyObject);
publisher.publishEvent(new AnyCreatedUpdatedEvent<>(this, merged, AuthContextUtils.getDomain()));
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java
index 7909022..0d714c5 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java
@@ -229,6 +229,11 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements GroupDAO {
return query.getResultList();
}
+ @Override
+ public List<String> findAllKeys(final int page, final int itemsPerPage) {
+ return findAllKeys(JPAGroup.TABLE, page, itemsPerPage);
+ }
+
private SearchCond buildDynMembershipCond(final String baseCondFIQL, final Realm groupRealm) {
AssignableCond cond = new AssignableCond();
cond.setRealmFullPath(groupRealm.getFullPath());
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java
index 85201fa..66d0574 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java
@@ -255,6 +255,10 @@ public class JPAUserDAO extends AbstractAnyDAO<User> implements UserDAO {
return query.getResultList();
}
+ public List<String> findAllKeys(final int page, final int itemsPerPage) {
+ return findAllKeys(JPAUser.TABLE, page, itemsPerPage);
+ }
+
protected List<AccountPolicy> getAccountPolicies(final User user) {
List<AccountPolicy> policies = new ArrayList<>();
diff --git a/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/AnyObjectTest.java b/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/AnyObjectTest.java
index c0d71a7..5848ce8 100644
--- a/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/AnyObjectTest.java
+++ b/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/AnyObjectTest.java
@@ -18,10 +18,12 @@
*/
package org.apache.syncope.core.persistence.jpa.inner;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import java.util.List;
import org.apache.syncope.common.lib.SyncopeConstants;
import org.apache.syncope.core.persistence.api.dao.AnyObjectDAO;
import org.apache.syncope.core.persistence.api.dao.AnyTypeDAO;
@@ -53,6 +55,17 @@ public class AnyObjectTest extends AbstractTest {
}
@Test
+ public void findAll() {
+ List<AnyObject> anyObjects = anyObjectDAO.findAll(1, 100);
+ assertNotNull(anyObjects);
+
+ List<String> anyObjectKeys = anyObjectDAO.findAllKeys(1, 100);
+ assertNotNull(anyObjectKeys);
+
+ assertEquals(anyObjects.size(), anyObjectKeys.size());
+ }
+
+ @Test
public void save() {
AnyObject anyObject = entityFactory.newEntity(AnyObject.class);
anyObject.setName("a name");
diff --git a/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/GroupTest.java b/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/GroupTest.java
index 52766e5..496c618 100644
--- a/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/GroupTest.java
+++ b/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/GroupTest.java
@@ -47,8 +47,13 @@ public class GroupTest extends AbstractTest {
@Test
public void findAll() {
- List<Group> list = groupDAO.findAll(1, 100);
- assertEquals(16, list.size());
+ List<Group> groups = groupDAO.findAll(1, 100);
+ assertEquals(16, groups.size());
+
+ List<String> groupKeys = groupDAO.findAllKeys(1, 100);
+ assertNotNull(groupKeys);
+
+ assertEquals(groups.size(), groupKeys.size());
}
@Test
diff --git a/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/UserTest.java b/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/UserTest.java
index 51a85cf..41f7e54 100644
--- a/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/UserTest.java
+++ b/core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/inner/UserTest.java
@@ -85,8 +85,13 @@ public class UserTest extends AbstractTest {
@Test
public void findAll() {
- List<User> list = userDAO.findAll(1, 100);
- assertEquals(5, list.size());
+ List<User> users = userDAO.findAll(1, 100);
+ assertEquals(5, users.size());
+
+ List<String> userKeys = userDAO.findAllKeys(1, 100);
+ assertNotNull(userKeys);
+
+ assertEquals(users.size(), userKeys.size());
}
@Test
diff --git a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
index eea2a8f..691c959 100644
--- a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
+++ b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
@@ -25,10 +25,7 @@ import org.apache.syncope.core.persistence.api.dao.AnyDAO;
import org.apache.syncope.core.persistence.api.dao.AnyObjectDAO;
import org.apache.syncope.core.persistence.api.dao.GroupDAO;
import org.apache.syncope.core.persistence.api.dao.UserDAO;
-import org.apache.syncope.core.persistence.api.entity.anyobject.AnyObject;
-import org.apache.syncope.core.persistence.api.entity.group.Group;
import org.apache.syncope.core.persistence.api.entity.task.TaskExec;
-import org.apache.syncope.core.persistence.api.entity.user.User;
import org.apache.syncope.core.spring.security.AuthContextUtils;
import org.apache.syncope.ext.elasticsearch.client.ElasticsearchUtils;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@@ -81,13 +78,12 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
LOG.debug("Indexing users...");
for (int page = 1; page <= (userDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
-
- for (User user : userDAO.findAll(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
+ for (String user : userDAO.findAllKeys(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
IndexRequest request = new IndexRequest(
elasticsearchUtils.getContextDomainName(AnyTypeKind.USER),
AnyTypeKind.USER.name(),
- user.getKey()).
- source(elasticsearchUtils.builder(user));
+ user).
+ source(elasticsearchUtils.builder(userDAO.find(user)));
IndexResponse response = client.index(request, RequestOptions.DEFAULT);
LOG.debug("Index successfully created for {}: {}", user, response);
}
@@ -95,12 +91,12 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
LOG.debug("Indexing groups...");
for (int page = 1; page <= (groupDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
- for (Group group : groupDAO.findAll(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
+ for (String group : groupDAO.findAllKeys(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
IndexRequest request = new IndexRequest(
elasticsearchUtils.getContextDomainName(AnyTypeKind.GROUP),
AnyTypeKind.GROUP.name(),
- group.getKey()).
- source(elasticsearchUtils.builder(group));
+ group).
+ source(elasticsearchUtils.builder(groupDAO.find(group)));
IndexResponse response = client.index(request, RequestOptions.DEFAULT);
LOG.debug("Index successfully created for {}: {}", group, response);
}
@@ -108,12 +104,12 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
LOG.debug("Indexing any objects...");
for (int page = 1; page <= (anyObjectDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
- for (AnyObject anyObject : anyObjectDAO.findAll(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
+ for (String anyObject : anyObjectDAO.findAllKeys(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
IndexRequest request = new IndexRequest(
elasticsearchUtils.getContextDomainName(AnyTypeKind.ANY_OBJECT),
AnyTypeKind.ANY_OBJECT.name(),
- anyObject.getKey()).
- source(elasticsearchUtils.builder(anyObject));
+ anyObject).
+ source(elasticsearchUtils.builder(anyObjectDAO.find(anyObject)));
IndexResponse response = client.index(request, RequestOptions.DEFAULT);
LOG.debug("Index successfully created for {}: {}", anyObject, response);
}