You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@syncope.apache.org by il...@apache.org on 2018/11/21 08:07:42 UTC
[syncope] branch master updated: Upgrading Elasticsearch
This is an automated email from the ASF dual-hosted git repository.
ilgrosso pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/syncope.git
The following commit(s) were added to refs/heads/master by this push:
new 756c390 Upgrading Elasticsearch
756c390 is described below
commit 756c39099237924459ae7491d8b07c07a5886e8a
Author: Francesco Chicchiriccò <il...@apache.org>
AuthorDate: Wed Nov 21 08:50:59 2018 +0100
Upgrading Elasticsearch
---
.../elasticsearch/client/ElasticsearchUtils.java | 7 +-
.../java/job/ElasticsearchReindex.java | 136 +++++++++------------
.../syncope/core/logic/init/ElasticsearchInit.java | 5 +-
.../fit/core/reference/ITImplementationLookup.java | 1 -
pom.xml | 4 +-
.../workingwithapachesyncope/customization.adoc | 12 +-
6 files changed, 67 insertions(+), 98 deletions(-)
diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java
index f5b69be..464d94e 100644
--- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java
+++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java
@@ -186,11 +186,6 @@ public class ElasticsearchUtils {
}
public String getContextDomainName(final AnyTypeKind kind) {
- return AuthContextUtils.getDomain().toLowerCase()
- + (kind.equals(AnyTypeKind.USER)
- ? "_user"
- : (kind.equals(AnyTypeKind.GROUP)
- ? "_group"
- : "_anyobject"));
+ return AuthContextUtils.getDomain().toLowerCase() + "_" + kind.name().toLowerCase();
}
}
diff --git a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
index 1a3331f..b2e1a43 100644
--- a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
+++ b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
@@ -18,6 +18,7 @@
*/
package org.apache.syncope.core.provisioning.java.job;
+import java.io.IOException;
import java.util.concurrent.ExecutionException;
import org.apache.syncope.common.lib.types.AnyTypeKind;
import org.apache.syncope.core.persistence.api.dao.AnyDAO;
@@ -66,72 +67,16 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
@Override
protected String doExecute(final boolean dryRun) throws JobExecutionException {
if (!dryRun) {
+ LOG.debug("Start rebuilding indexes");
+
try {
- checkExistsIndexResponse(AnyTypeKind.USER);
- checkExistsIndexResponse(AnyTypeKind.GROUP);
- checkExistsIndexResponse(AnyTypeKind.ANY_OBJECT);
-
- XContentBuilder settings = XContentFactory.jsonBuilder().
- startObject().
- startObject("analysis").
- startObject("analyzer").
- startObject("string_lowercase").
- field("type", "custom").
- field("tokenizer", "standard").
- field("filter").
- startArray().
- value("lowercase").
- endArray().
- endObject().
- endObject().
- endObject().
- endObject();
- XContentBuilder mappingUser = XContentFactory.jsonBuilder().
- startObject().
- startArray("dynamic_templates").
- startObject().
- startObject("strings").
- field("match_mapping_type", "string").
- startObject("mapping").
- field("type", "keyword").
- field("analyzer", "string_lowercase").
- endObject().
- endObject().
- endObject().
- endArray().
- endObject();
- XContentBuilder mappingGroup = XContentFactory.jsonBuilder().
- startObject().
- startArray("dynamic_templates").
- startObject().
- startObject("strings").
- field("match_mapping_type", "string").
- startObject("mapping").
- field("type", "keyword").
- field("analyzer", "string_lowercase").
- endObject().
- endObject().
- endObject().
- endArray().
- endObject();
- XContentBuilder mappingAnyobject = XContentFactory.jsonBuilder().
- startObject().
- startArray("dynamic_templates").
- startObject().
- startObject("strings").
- field("match_mapping_type", "string").
- startObject("mapping").
- field("type", "keyword").
- field("analyzer", "string_lowercase").
- endObject().
- endObject().
- endObject().
- endArray().
- endObject();
-
- createIndexResponse(AnyTypeKind.USER, settings, mappingUser);
- createIndexResponse(AnyTypeKind.GROUP, settings, mappingGroup);
- createIndexResponse(AnyTypeKind.ANY_OBJECT, settings, mappingAnyobject);
+ removeIndexIfExists(AnyTypeKind.USER);
+ removeIndexIfExists(AnyTypeKind.GROUP);
+ removeIndexIfExists(AnyTypeKind.ANY_OBJECT);
+
+ createIndex(AnyTypeKind.USER);
+ createIndex(AnyTypeKind.GROUP);
+ createIndex(AnyTypeKind.ANY_OBJECT);
LOG.debug("Indexing users...");
for (int page = 1; page <= (userDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
@@ -145,6 +90,7 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
LOG.debug("Index successfully created for {}: {}", user, response);
}
}
+
LOG.debug("Indexing groups...");
for (int page = 1; page <= (groupDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
for (Group group : groupDAO.findAll(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
@@ -157,6 +103,7 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
LOG.debug("Index successfully created for {}: {}", group, response);
}
}
+
LOG.debug("Indexing any objects...");
for (int page = 1; page <= (anyObjectDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
for (AnyObject anyObject : anyObjectDAO.findAll(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
@@ -170,19 +117,18 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
}
}
- LOG.debug("Rebuild index {} successfully completed", AuthContextUtils.getDomain().toLowerCase());
+ LOG.debug("Rebuild indexes for domain {} successfully completed", AuthContextUtils.getDomain());
} catch (Exception e) {
- throw new JobExecutionException(
- "While rebuilding index " + AuthContextUtils.getDomain().toLowerCase(), e);
+ throw new JobExecutionException("While rebuilding index for domain " + AuthContextUtils.getDomain(), e);
}
}
return "SUCCESS";
}
- private void checkExistsIndexResponse(final AnyTypeKind kind) throws InterruptedException, ExecutionException {
- LOG.debug("Start rebuild index {}",
- elasticsearchUtils.getContextDomainName(kind));
+ private void removeIndexIfExists(final AnyTypeKind kind)
+ throws InterruptedException, ExecutionException {
+
IndicesExistsResponse existsIndexResponse = client.admin().indices().
exists(new IndicesExistsRequest(elasticsearchUtils.getContextDomainName(kind))).
get();
@@ -195,17 +141,45 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
}
}
- private void createIndexResponse(final AnyTypeKind kind,
- final XContentBuilder settings,
- final XContentBuilder mapping) throws InterruptedException, ExecutionException {
-
- CreateIndexResponse createIndexResponseUser = client.admin().indices().
- create(new CreateIndexRequest(elasticsearchUtils.getContextDomainName(kind)).
- settings(settings).
- mapping(kind.name(), mapping)).
- get();
+ private void createIndex(final AnyTypeKind kind)
+ throws InterruptedException, ExecutionException, IOException {
+
+ XContentBuilder settings = XContentFactory.jsonBuilder().
+ startObject().
+ startObject("analysis").
+ startObject("analyzer").
+ startObject("string_lowercase").
+ field("type", "custom").
+ field("tokenizer", "standard").
+ field("filter").
+ startArray().
+ value("lowercase").
+ endArray().
+ endObject().
+ endObject().
+ endObject().
+ endObject();
+
+ XContentBuilder mapping = XContentFactory.jsonBuilder().
+ startObject().
+ startArray("dynamic_templates").
+ startObject().
+ startObject("strings").
+ field("match_mapping_type", "string").
+ startObject("mapping").
+ field("type", "keyword").
+ field("analyzer", "string_lowercase").
+ endObject().
+ endObject().
+ endObject().
+ endArray().
+ endObject();
+
+ CreateIndexResponse response = client.admin().indices().
+ create(new CreateIndexRequest(elasticsearchUtils.getContextDomainName(kind)).settings(settings).
+ mapping(kind.name(), mapping)).get();
LOG.debug("Successfully created {} for {}: {}",
- elasticsearchUtils.getContextDomainName(kind), kind.name(), createIndexResponseUser);
+ elasticsearchUtils.getContextDomainName(kind), kind.name(), response);
}
@Override
diff --git a/fit/core-reference/src/main/java/org/apache/syncope/core/logic/init/ElasticsearchInit.java b/fit/core-reference/src/main/java/org/apache/syncope/core/logic/init/ElasticsearchInit.java
index 60d1893..404f789 100644
--- a/fit/core-reference/src/main/java/org/apache/syncope/core/logic/init/ElasticsearchInit.java
+++ b/fit/core-reference/src/main/java/org/apache/syncope/core/logic/init/ElasticsearchInit.java
@@ -33,6 +33,8 @@ import org.springframework.transaction.annotation.Transactional;
@Component
public class ElasticsearchInit {
+ private static final String ES_REINDEX = "org.apache.syncope.core.provisioning.java.job.ElasticsearchReindex";
+
@Autowired
private ImplementationDAO implementationDAO;
@@ -42,8 +44,6 @@ public class ElasticsearchInit {
@Autowired
private TaskLogic taskLogic;
- private static final String ES_REINDEX = "org.apache.syncope.core.provisioning.java.job.ElasticsearchReindex";
-
@Transactional
public void init() {
Implementation reindex = implementationDAO.find(ImplementationType.TASKJOB_DELEGATE).
@@ -67,5 +67,4 @@ public class ElasticsearchInit {
taskLogic.execute(task.getKey(), null, false);
}
-
}
diff --git a/fit/core-reference/src/main/java/org/apache/syncope/fit/core/reference/ITImplementationLookup.java b/fit/core-reference/src/main/java/org/apache/syncope/fit/core/reference/ITImplementationLookup.java
index c8c2a5e..3ab13c9 100644
--- a/fit/core-reference/src/main/java/org/apache/syncope/fit/core/reference/ITImplementationLookup.java
+++ b/fit/core-reference/src/main/java/org/apache/syncope/fit/core/reference/ITImplementationLookup.java
@@ -252,7 +252,6 @@ public class ITImplementationLookup implements ImplementationLookup {
for (Map.Entry<String, DataSource> entry : domainsHolder.getDomains().entrySet()) {
AuthContextUtils.execWithAuthContext(entry.getKey(), () -> {
elasticsearchInit.init();
-
return null;
});
}
diff --git a/pom.xml b/pom.xml
index 9f03673..afe4a54 100644
--- a/pom.xml
+++ b/pom.xml
@@ -415,7 +415,7 @@ under the License.
<slf4j.version>1.7.25</slf4j.version>
<opensaml.version>3.3.0</opensaml.version>
- <elasticsearch.version>6.5.0</elasticsearch.version>
+ <elasticsearch.version>6.5.1</elasticsearch.version>
<log4j.version>2.11.1</log4j.version>
<disruptor.version>3.4.2</disruptor.version>
@@ -2326,7 +2326,7 @@ under the License.
<link>https://docs.spring.io/spring-security/site/docs/5.0.x/api/</link>
<link>http://www.flowable.org/docs/javadocs/</link>
<link>https://build.shibboleth.net/nexus/content/sites/site/java-opensaml/3.3.0/apidocs/</link>
- <link>https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/6.5.0/index.html</link>
+ <link>https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/6.5.1/index.html</link>
<link>http://docs.swagger.io/swagger-core/v2.0.5/apidocs/</link>
</links>
</configuration>
diff --git a/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc b/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc
index 2324890..27738a3 100644
--- a/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc
+++ b/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc
@@ -487,12 +487,14 @@ classpath*:/provisioning*Context.xml
classpath*:/workflow*Context.xml
....
-It is also required to initialize the Elasticsearch indexes.
+It is also required to initialize the Elasticsearch indexes: add a new Java <<implementations,implementation>> for
+`TASKJOB_DELEGATE` and use `org.apache.syncope.core.provisioning.java.job.ElasticsearchReindex` as class. +
+Then, create a new <<tasks-custom, custom task>>, select the implementation just created as job delegate and execute it.
-Add a new Java <<implementations,implementation>> for `TASKJOB_DELEGATE` and use
-`org.apache.syncope.core.provisioning.java.job.ElasticsearchReindex` as class.
-
-Then, create a new <<tasks-custom, Custom task>>, select the implementation just created as job delegate and execute it.
+[TIP]
+The `org.apache.syncope.core.provisioning.java.job.ElasticsearchReindex` custom task created above is not meant for
+scheduled execution; rather, it can be run every time you want to blank and re-create the Elasticsearch indexes
+starting from Syncope's users, groups and any objects.
[discrete]
===== Enable the <<SCIM>> extension