You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@syncope.apache.org by il...@apache.org on 2022/11/04 10:23:16 UTC
[syncope] branch 2_1_X updated: [SYNCOPE-1696] Adding support to manage Audit entries via Elasticsearch (#388)
This is an automated email from the ASF dual-hosted git repository.
ilgrosso pushed a commit to branch 2_1_X
in repository https://gitbox.apache.org/repos/asf/syncope.git
The following commit(s) were added to refs/heads/2_1_X by this push:
new f91f5c00ab [SYNCOPE-1696] Adding support to manage Audit entries via Elasticsearch (#388)
f91f5c00ab is described below
commit f91f5c00ab3808234f1da8a31b13d4b78b76f209
Author: Francesco Chicchiriccò <il...@users.noreply.github.com>
AuthorDate: Fri Nov 4 11:23:10 2022 +0100
[SYNCOPE-1696] Adding support to manage Audit entries via Elasticsearch (#388)
---
.../core/logic/audit/JdbcAuditAppender.java | 7 +-
.../syncope/core/logic/init/LoggerLoader.java | 25 ++-
core/logic/src/main/resources/logic.properties | 2 +-
.../core/persistence/api/dao/LoggerDAO.java | 4 +-
.../core/persistence/jpa/dao/JPALoggerDAO.java | 8 +-
.../provisioning/api/serialization/POJOHelper.java | 12 ++
.../provisioning/java/DefaultAuditManager.java | 16 +-
.../client/ElasticsearchIndexManager.java | 139 +++++++++++++---
.../elasticsearch/client/ElasticsearchUtils.java | 51 +++++-
ext/elasticsearch/{ => logic}/pom.xml | 44 ++++--
.../core/logic/audit/ElasticsearchAppender.java | 97 ++++++++++++
.../logic/audit/ElasticsearchAuditAppender.java | 55 +++++++
.../jpa/dao/ElasticsearchAnySearchDAO.java | 4 +-
.../jpa/dao/ElasticsearchLoggerDAO.java | 176 +++++++++++++++++++++
.../src/main/resources/persistence.properties | 2 +-
.../jpa/dao/ElasticsearchAnySearchDAOTest.java | 4 +-
ext/elasticsearch/pom.xml | 1 +
.../java/job/ElasticsearchReindex.java | 29 ++--
fit/core-reference/pom.xml | 6 +
.../resources/{ => elasticsearch}/logic.properties | 2 +-
.../resources/elasticsearch/persistence.properties | 2 +-
.../src/main/resources/logic.properties | 2 +-
.../org/apache/syncope/fit/AbstractITCase.java | 8 +
.../asciidoc/reference-guide/concepts/audit.adoc | 5 +-
.../reference-guide/concepts/extensions.adoc | 4 +-
.../workingwithapachesyncope/customization.adoc | 21 ++-
26 files changed, 639 insertions(+), 87 deletions(-)
diff --git a/core/logic/src/main/java/org/apache/syncope/core/logic/audit/JdbcAuditAppender.java b/core/logic/src/main/java/org/apache/syncope/core/logic/audit/JdbcAuditAppender.java
index 15575d2b3d..a3a1f98403 100644
--- a/core/logic/src/main/java/org/apache/syncope/core/logic/audit/JdbcAuditAppender.java
+++ b/core/logic/src/main/java/org/apache/syncope/core/logic/audit/JdbcAuditAppender.java
@@ -56,10 +56,10 @@ public class JdbcAuditAppender extends DefaultAuditAppender {
setConfiguration(ctx.getConfiguration()).setName("THROWABLE").setPattern("%ex{full}").build()
};
- Appender appender = ctx.getConfiguration().getAppender("audit_for_" + domain);
+ Appender appender = ctx.getConfiguration().getAppender(getTargetAppenderName());
if (appender == null) {
appender = JdbcAppender.newBuilder().
- setName("audit_for_" + domain).
+ setName(getTargetAppenderName()).
setIgnoreExceptions(false).
setConnectionSource(new DataSourceConnectionSource(domain, domainsHolder.getDomains().get(domain))).
setBufferSize(0).
@@ -74,8 +74,7 @@ public class JdbcAuditAppender extends DefaultAuditAppender {
@Override
public String getTargetAppenderName() {
- // not used
- return null;
+ return "audit_for_" + domain;
}
protected static class DataSourceConnectionSource extends AbstractConnectionSource {
diff --git a/core/logic/src/main/java/org/apache/syncope/core/logic/init/LoggerLoader.java b/core/logic/src/main/java/org/apache/syncope/core/logic/init/LoggerLoader.java
index aff916d904..f985917c12 100644
--- a/core/logic/src/main/java/org/apache/syncope/core/logic/init/LoggerLoader.java
+++ b/core/logic/src/main/java/org/apache/syncope/core/logic/init/LoggerLoader.java
@@ -32,21 +32,26 @@ import org.apache.logging.log4j.core.config.LoggerConfig;
import org.apache.syncope.common.lib.types.AuditLoggerName;
import org.apache.syncope.core.logic.audit.AuditAppender;
import org.apache.syncope.core.logic.MemoryAppender;
-import org.apache.syncope.core.logic.audit.JdbcAuditAppender;
+import org.apache.syncope.core.logic.audit.DefaultAuditAppender;
import org.apache.syncope.core.spring.security.AuthContextUtils;
import org.apache.syncope.core.persistence.api.DomainsHolder;
import org.apache.syncope.core.persistence.api.ImplementationLookup;
import org.apache.syncope.core.persistence.api.SyncopeLoader;
import org.apache.syncope.core.spring.ApplicationContextProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.support.AbstractBeanDefinition;
import org.springframework.stereotype.Component;
+import org.springframework.util.ClassUtils;
@Component
public class LoggerLoader implements SyncopeLoader {
+ private static final Logger LOG = LoggerFactory.getLogger(LoggerLoader.class);
+
@Autowired
private DomainsHolder domainsHolder;
@@ -56,8 +61,8 @@ public class LoggerLoader implements SyncopeLoader {
@Autowired
private ImplementationLookup implementationLookup;
- @Value("${enable.jdbcAuditAppender:true}")
- private boolean enableJdbcAuditAppender;
+ @Value("${default.audit.appender:org.apache.syncope.core.logic.audit.JdbcAuditAppender}")
+ private String defaultAuditAppender;
private final Map<String, MemoryAppender> memoryAppenders = new HashMap<>();
@@ -75,15 +80,19 @@ public class LoggerLoader implements SyncopeLoader {
forEach(entry -> memoryAppenders.put(entry.getKey(), (MemoryAppender) entry.getValue()));
domainsHolder.getDomains().keySet().forEach(domain -> {
- if (enableJdbcAuditAppender) {
- JdbcAuditAppender jdbcAuditAppender = (JdbcAuditAppender) ApplicationContextProvider.getBeanFactory().
- createBean(JdbcAuditAppender.class, AbstractBeanDefinition.AUTOWIRE_BY_TYPE, true);
- jdbcAuditAppender.init(domain);
+ try {
+ DefaultAuditAppender dfaa = (DefaultAuditAppender) ApplicationContextProvider.getBeanFactory().
+ createBean(
+ ClassUtils.forName(defaultAuditAppender, ClassUtils.getDefaultClassLoader()),
+ AbstractBeanDefinition.AUTOWIRE_BY_TYPE, true);
+ dfaa.init(domain);
LoggerConfig logConf = new LoggerConfig(AuditLoggerName.getAuditLoggerName(domain), null, false);
- logConf.addAppender(jdbcAuditAppender.getTargetAppender(), Level.DEBUG, null);
+ logConf.addAppender(dfaa.getTargetAppender(), Level.DEBUG, null);
logConf.setLevel(Level.DEBUG);
ctx.getConfiguration().addLogger(logConf.getName(), logConf);
+ } catch (Exception e) {
+ LOG.error("While creating instance of DefaultAuditAppender {}", defaultAuditAppender, e);
}
// SYNCOPE-1144 For each custom audit appender class add related appenders to log4j logger
diff --git a/core/logic/src/main/resources/logic.properties b/core/logic/src/main/resources/logic.properties
index 60dcae60da..806b8b76b4 100644
--- a/core/logic/src/main/resources/logic.properties
+++ b/core/logic/src/main/resources/logic.properties
@@ -16,4 +16,4 @@
# under the License.
logicInvocationHandler=org.apache.syncope.core.logic.LogicInvocationHandler
classPathScanImplementationLookup=org.apache.syncope.core.logic.init.ClassPathScanImplementationLookup
-enable.jdbcAuditAppender=true
+default.audit.appender=org.apache.syncope.core.logic.audit.JdbcAuditAppender
diff --git a/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/LoggerDAO.java b/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/LoggerDAO.java
index 0539ef3bf3..a4c355d50f 100644
--- a/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/LoggerDAO.java
+++ b/core/persistence-api/src/main/java/org/apache/syncope/core/persistence/api/dao/LoggerDAO.java
@@ -52,11 +52,11 @@ public interface LoggerDAO extends DAO<Logger> {
List<AuditEntry> findAuditEntries(
String entityKey,
int page,
- int size,
+ int itemsPerPage,
AuditElements.EventCategoryType type,
String category,
String subcategory,
List<String> events,
AuditElements.Result result,
- List<OrderByClause> orderByClauses);
+ List<OrderByClause> orderBy);
}
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPALoggerDAO.java b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPALoggerDAO.java
index a2f44dd393..1b6f0c0ff6 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPALoggerDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPALoggerDAO.java
@@ -182,7 +182,7 @@ public class JPALoggerDAO extends AbstractDAO<Logger> implements LoggerDAO {
final String subcategory,
final List<String> events,
final AuditElements.Result result,
- final List<OrderByClause> orderByClauses) {
+ final List<OrderByClause> orderBy) {
String queryString = "SELECT " + select()
+ " FROM " + AUDIT_TABLE
@@ -193,9 +193,9 @@ public class JPALoggerDAO extends AbstractDAO<Logger> implements LoggerDAO {
result(result).
events(events).
build();
- if (!orderByClauses.isEmpty()) {
- queryString += " ORDER BY " + orderByClauses.stream().
- map(orderBy -> orderBy.getField() + ' ' + orderBy.getDirection().name()).
+ if (!orderBy.isEmpty()) {
+ queryString += " ORDER BY " + orderBy.stream().
+ map(clause -> clause.getField() + ' ' + clause.getDirection().name()).
collect(Collectors.joining(","));
}
diff --git a/core/provisioning-api/src/main/java/org/apache/syncope/core/provisioning/api/serialization/POJOHelper.java b/core/provisioning-api/src/main/java/org/apache/syncope/core/provisioning/api/serialization/POJOHelper.java
index af999062c1..5636795f4f 100644
--- a/core/provisioning-api/src/main/java/org/apache/syncope/core/provisioning/api/serialization/POJOHelper.java
+++ b/core/provisioning-api/src/main/java/org/apache/syncope/core/provisioning/api/serialization/POJOHelper.java
@@ -88,6 +88,18 @@ public final class POJOHelper {
return result;
}
+ public static <T extends Object> T convertValue(final Object value, final Class<T> reference) {
+ T result = null;
+
+ try {
+ result = MAPPER.convertValue(value, reference);
+ } catch (Exception e) {
+ LOG.error("During conversion", e);
+ }
+
+ return result;
+ }
+
private POJOHelper() {
}
}
diff --git a/core/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/DefaultAuditManager.java b/core/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/DefaultAuditManager.java
index 446d57ebcc..8436354559 100644
--- a/core/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/DefaultAuditManager.java
+++ b/core/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/DefaultAuditManager.java
@@ -20,6 +20,7 @@ package org.apache.syncope.core.provisioning.java;
import java.util.Arrays;
import java.util.Date;
+import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.syncope.common.lib.log.AuditEntry;
@@ -132,13 +133,12 @@ public class DefaultAuditManager implements AuditManager {
AuditLoggerName auditLoggerName = new AuditLoggerName.Builder().
type(type).category(category).subcategory(subcategory).event(event).result(condition).build();
- org.apache.syncope.core.persistence.api.entity.Logger syncopeLogger =
- loggerDAO.find(auditLoggerName.toLoggerName());
- if (syncopeLogger != null && syncopeLogger.getLevel() == LoggerLevel.DEBUG) {
- Throwable throwable = null;
- if (output instanceof Throwable) {
- throwable = (Throwable) output;
- }
+ Optional.ofNullable(loggerDAO.find(auditLoggerName.toLoggerName())).
+ filter(syncopeLogger -> syncopeLogger.getLevel() == LoggerLevel.DEBUG).ifPresent(syncopeLogger -> {
+
+ Throwable throwable = output instanceof Throwable
+ ? (Throwable) output
+ : null;
AuditEntry auditEntry = new AuditEntry();
auditEntry.setWho(who);
@@ -170,6 +170,6 @@ public class DefaultAuditManager implements AuditManager {
logger.debug(serializedAuditEntry, throwable);
eventLogger.debug(serializedAuditEntry, throwable);
}
- }
+ });
}
}
diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java
index 7dc8db64d8..ebf5ff7763 100644
--- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java
+++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java
@@ -19,9 +19,11 @@
package org.apache.syncope.ext.elasticsearch.client;
import java.io.IOException;
+import org.apache.syncope.common.lib.log.AuditEntry;
import org.apache.syncope.common.lib.types.AnyTypeKind;
import org.apache.syncope.core.persistence.api.entity.Any;
import org.apache.syncope.core.provisioning.api.event.AnyLifecycleEvent;
+import org.apache.syncope.core.spring.security.SecureRandomUtils;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.delete.DeleteRequest;
@@ -64,9 +66,14 @@ public class ElasticsearchIndexManager {
this.elasticsearchUtils = elasticsearchUtils;
}
- public boolean existsIndex(final String domain, final AnyTypeKind kind) throws IOException {
+ public boolean existsAnyIndex(final String domain, final AnyTypeKind kind) throws IOException {
return client.indices().exists(
- new GetIndexRequest(ElasticsearchUtils.getContextDomainName(domain, kind)), RequestOptions.DEFAULT);
+ new GetIndexRequest(ElasticsearchUtils.getAnyIndex(domain, kind)), RequestOptions.DEFAULT);
+ }
+
+ public boolean existsAuditIndex(final String domain) throws IOException {
+ return client.indices().exists(
+ new GetIndexRequest(ElasticsearchUtils.getAuditIndex(domain)), RequestOptions.DEFAULT);
}
public XContentBuilder defaultSettings() throws IOException {
@@ -91,7 +98,24 @@ public class ElasticsearchIndexManager {
endObject();
}
- public XContentBuilder defaultMapping() throws IOException {
+ public XContentBuilder defaultAnyMapping() throws IOException {
+ return XContentFactory.jsonBuilder().
+ startObject().
+ startArray("dynamic_templates").
+ startObject().
+ startObject("strings").
+ field("match_mapping_type", "string").
+ startObject("mapping").
+ field("type", "keyword").
+ field("normalizer", "string_lowercase").
+ endObject().
+ endObject().
+ endObject().
+ endArray().
+ endObject();
+ }
+
+ public XContentBuilder defaultAuditMapping() throws IOException {
return XContentFactory.jsonBuilder().
startObject().
startArray("dynamic_templates").
@@ -105,22 +129,44 @@ public class ElasticsearchIndexManager {
endObject().
endObject().
endArray().
+ startObject("properties").
+ startObject("message").
+ startObject("properties").
+ startObject("before").
+ field("type", "text").
+ field("analyzer", "standard").
+ endObject().
+ startObject("inputs").
+ field("type", "text").
+ field("analyzer", "standard").
+ endObject().
+ startObject("output").
+ field("type", "text").
+ field("analyzer", "standard").
+ endObject().
+ startObject("throwable").
+ field("type", "text").
+ field("analyzer", "standard").
+ endObject().
+ endObject().
+ endObject().
+ endObject().
endObject();
}
- protected CreateIndexResponse doCreateIndex(
+ protected CreateIndexResponse doCreateAnyIndex(
final String domain,
final AnyTypeKind kind,
final XContentBuilder settings,
final XContentBuilder mapping) throws IOException {
return client.indices().create(
- new CreateIndexRequest(ElasticsearchUtils.getContextDomainName(domain, kind)).
+ new CreateIndexRequest(ElasticsearchUtils.getAnyIndex(domain, kind)).
settings(settings).
mapping(mapping), RequestOptions.DEFAULT);
}
- public void createIndex(
+ public void createAnyIndex(
final String domain,
final AnyTypeKind kind,
final XContentBuilder settings,
@@ -128,49 +174,85 @@ public class ElasticsearchIndexManager {
throws IOException {
try {
- CreateIndexResponse response = doCreateIndex(domain, kind, settings, mapping);
+ CreateIndexResponse response = doCreateAnyIndex(domain, kind, settings, mapping);
LOG.debug("Successfully created {} for {}: {}",
- ElasticsearchUtils.getContextDomainName(domain, kind), kind.name(), response);
+ ElasticsearchUtils.getAnyIndex(domain, kind), kind.name(), response);
} catch (ElasticsearchStatusException e) {
LOG.debug("Could not create index {} because it already exists",
- ElasticsearchUtils.getContextDomainName(domain, kind), e);
+ ElasticsearchUtils.getAnyIndex(domain, kind), e);
- removeIndex(domain, kind);
- doCreateIndex(domain, kind, settings, mapping);
+ removeAnyIndex(domain, kind);
+ doCreateAnyIndex(domain, kind, settings, mapping);
}
}
- public void removeIndex(final String domain, final AnyTypeKind kind) throws IOException {
+ public void removeAnyIndex(final String domain, final AnyTypeKind kind) throws IOException {
AcknowledgedResponse acknowledgedResponse = client.indices().delete(
- new DeleteIndexRequest(ElasticsearchUtils.getContextDomainName(domain, kind)), RequestOptions.DEFAULT);
+ new DeleteIndexRequest(ElasticsearchUtils.getAnyIndex(domain, kind)), RequestOptions.DEFAULT);
LOG.debug("Successfully removed {}: {}",
- ElasticsearchUtils.getContextDomainName(domain, kind), acknowledgedResponse);
+ ElasticsearchUtils.getAnyIndex(domain, kind), acknowledgedResponse);
+ }
+
+ protected CreateIndexResponse doCreateAuditIndex(
+ final String domain,
+ final XContentBuilder settings,
+ final XContentBuilder mapping) throws IOException {
+
+ return client.indices().create(
+ new CreateIndexRequest(ElasticsearchUtils.getAuditIndex(domain)).
+ settings(settings).
+ mapping(mapping), RequestOptions.DEFAULT);
+ }
+
+ public void createAuditIndex(
+ final String domain,
+ final XContentBuilder settings,
+ final XContentBuilder mapping)
+ throws IOException {
+
+ try {
+ CreateIndexResponse response = doCreateAuditIndex(domain, settings, mapping);
+
+ LOG.debug("Successfully created {} for {}: {}",
+ ElasticsearchUtils.getAuditIndex(domain), response);
+ } catch (ElasticsearchStatusException e) {
+ LOG.debug("Could not create index {} because it already exists",
+ ElasticsearchUtils.getAuditIndex(domain), e);
+
+ removeAuditIndex(domain);
+ doCreateAuditIndex(domain, settings, mapping);
+ }
+ }
+
+ public void removeAuditIndex(final String domain) throws IOException {
+ AcknowledgedResponse acknowledgedResponse = client.indices().delete(
+ new DeleteIndexRequest(ElasticsearchUtils.getAuditIndex(domain)), RequestOptions.DEFAULT);
+ LOG.debug("Successfully removed {}: {}",
+ ElasticsearchUtils.getAuditIndex(domain), acknowledgedResponse);
}
@TransactionalEventListener
- public void after(final AnyLifecycleEvent<Any<?>> event) throws IOException {
+ public void any(final AnyLifecycleEvent<Any<?>> event) throws IOException {
LOG.debug("About to {} index for {}", event.getType().name(), event.getAny());
if (event.getType() == SyncDeltaType.DELETE) {
DeleteRequest request = new DeleteRequest(
- ElasticsearchUtils.getContextDomainName(event.getDomain(), event.getAny().getType().getKind()),
+ ElasticsearchUtils.getAnyIndex(event.getDomain(), event.getAny().getType().getKind()),
event.getAny().getKey());
DeleteResponse response = client.delete(request, RequestOptions.DEFAULT);
LOG.debug("Index successfully deleted for {}[{}]: {}",
event.getAny().getType().getKind(), event.getAny().getKey(), response);
} else {
GetRequest getRequest = new GetRequest(
- ElasticsearchUtils.getContextDomainName(
- event.getDomain(), event.getAny().getType().getKind()),
+ ElasticsearchUtils.getAnyIndex(event.getDomain(), event.getAny().getType().getKind()),
event.getAny().getKey());
GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT);
if (getResponse.isExists()) {
LOG.debug("About to update index for {}", event.getAny());
UpdateRequest request = new UpdateRequest(
- ElasticsearchUtils.getContextDomainName(
- event.getDomain(), event.getAny().getType().getKind()),
+ ElasticsearchUtils.getAnyIndex(event.getDomain(), event.getAny().getType().getKind()),
event.getAny().getKey()).
retryOnConflict(elasticsearchUtils.getRetryOnConflict()).
doc(elasticsearchUtils.builder(event.getAny(), event.getDomain()));
@@ -180,8 +262,7 @@ public class ElasticsearchIndexManager {
LOG.debug("About to create index for {}", event.getAny());
IndexRequest request = new IndexRequest(
- ElasticsearchUtils.getContextDomainName(
- event.getDomain(), event.getAny().getType().getKind())).
+ ElasticsearchUtils.getAnyIndex(event.getDomain(), event.getAny().getType().getKind())).
id(event.getAny().getKey()).
source(elasticsearchUtils.builder(event.getAny(), event.getDomain()));
IndexResponse response = client.index(request, RequestOptions.DEFAULT);
@@ -189,4 +270,18 @@ public class ElasticsearchIndexManager {
}
}
}
+
+ public void audit(final String domain, final long instant, final AuditEntry message)
+ throws IOException {
+
+ LOG.debug("About to audit");
+
+ IndexRequest request = new IndexRequest(
+ ElasticsearchUtils.getAuditIndex(domain)).
+ id(SecureRandomUtils.generateRandomUUID().toString()).
+ source(elasticsearchUtils.builder(instant, message, domain));
+ IndexResponse response = client.index(request, RequestOptions.DEFAULT);
+
+ LOG.debug("Audit successfully created: {}", response);
+ }
}
diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java
index 3a030cc096..85ecae129b 100644
--- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java
+++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.syncope.common.lib.log.AuditEntry;
import org.apache.syncope.common.lib.types.AnyTypeKind;
import org.apache.syncope.core.persistence.api.dao.AnyObjectDAO;
import org.apache.syncope.core.persistence.api.dao.GroupDAO;
@@ -47,10 +48,14 @@ import org.springframework.transaction.annotation.Transactional;
@SuppressWarnings("deprecation")
public class ElasticsearchUtils {
- public static String getContextDomainName(final String domain, final AnyTypeKind kind) {
+ public static String getAnyIndex(final String domain, final AnyTypeKind kind) {
return domain.toLowerCase() + '_' + kind.name().toLowerCase();
}
+ public static String getAuditIndex(final String domain) {
+ return domain.toLowerCase() + "_audit";
+ }
+
@Autowired
protected UserDAO userDAO;
@@ -251,7 +256,49 @@ public class ElasticsearchUtils {
return builder;
}
- protected XContentBuilder customizeBuilder(final XContentBuilder builder, final User user, final String domain)
+ protected XContentBuilder customizeBuilder(
+ final XContentBuilder builder, final User user, final String domain)
+ throws IOException {
+
+ return builder;
+ }
+
+ public XContentBuilder builder(
+ final long instant, final AuditEntry message, final String domain) throws IOException {
+
+ XContentBuilder builder = XContentFactory.jsonBuilder().
+ startObject().
+ field("instant", instant).
+ field("message").
+ startObject().
+ field("who", message.getWho()).
+ field("date", message.getDate()).
+ field("logger").
+ startObject().
+ field("type", message.getLogger().getType().name()).
+ field("category", message.getLogger().getCategory()).
+ field("subcategory", message.getLogger().getSubcategory()).
+ field("event", message.getLogger().getEvent()).
+ field("result", message.getLogger().getResult().name()).
+ endObject().
+ field("before", message.getBefore()).
+ field("output", message.getOutput()).
+ field("throwable", message.getThrowable()).
+ field("inputs").startArray();
+ for (String input : message.getInputs()) {
+ builder.value(input);
+ }
+ builder.endArray().
+ endObject().
+ endObject();
+
+ builder = customizeBuilder(builder, instant, message, domain);
+
+ return builder;
+ }
+
+ protected XContentBuilder customizeBuilder(
+ final XContentBuilder builder, final long instant, final AuditEntry message, final String domain)
throws IOException {
return builder;
diff --git a/ext/elasticsearch/pom.xml b/ext/elasticsearch/logic/pom.xml
similarity index 52%
copy from ext/elasticsearch/pom.xml
copy to ext/elasticsearch/logic/pom.xml
index 8e7c0fa4a5..84ed6ce545 100644
--- a/ext/elasticsearch/pom.xml
+++ b/ext/elasticsearch/logic/pom.xml
@@ -22,25 +22,41 @@ under the License.
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache.syncope</groupId>
- <artifactId>syncope-ext</artifactId>
+ <groupId>org.apache.syncope.ext</groupId>
+ <artifactId>syncope-ext-elasticsearch</artifactId>
<version>2.1.13-SNAPSHOT</version>
</parent>
- <name>Apache Syncope Ext: Elasticsearch</name>
- <description>Apache Syncope Ext: Elasticsearch</description>
- <groupId>org.apache.syncope.ext</groupId>
- <artifactId>syncope-ext-elasticsearch</artifactId>
- <packaging>pom</packaging>
+ <name>Apache Syncope Ext: Elasticsearch Logic</name>
+ <description>Apache Syncope Ext: Elasticsearch Logic</description>
+ <groupId>org.apache.syncope.ext.elasticsearch</groupId>
+ <artifactId>syncope-ext-elasticsearch-logic</artifactId>
+ <packaging>jar</packaging>
<properties>
- <rootpom.basedir>${basedir}/../..</rootpom.basedir>
+ <rootpom.basedir>${basedir}/../../..</rootpom.basedir>
</properties>
-
- <modules>
- <module>client-elasticsearch</module>
- <module>persistence-jpa</module>
- <module>provisioning-java</module>
- </modules>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.syncope.core</groupId>
+ <artifactId>syncope-core-logic</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.syncope.ext.elasticsearch</groupId>
+ <artifactId>syncope-ext-elasticsearch-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/ext/elasticsearch/logic/src/main/java/org/apache/syncope/core/logic/audit/ElasticsearchAppender.java b/ext/elasticsearch/logic/src/main/java/org/apache/syncope/core/logic/audit/ElasticsearchAppender.java
new file mode 100644
index 0000000000..a0b8a0cdb2
--- /dev/null
+++ b/ext/elasticsearch/logic/src/main/java/org/apache/syncope/core/logic/audit/ElasticsearchAppender.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.syncope.core.logic.audit;
+
+import java.io.Serializable;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Property;
+import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;
+import org.apache.syncope.common.lib.log.AuditEntry;
+import org.apache.syncope.core.provisioning.api.serialization.POJOHelper;
+import org.apache.syncope.ext.elasticsearch.client.ElasticsearchIndexManager;
+
+public class ElasticsearchAppender extends AbstractAppender {
+
+ public static class Builder extends AbstractAppender.Builder<Builder>
+ implements org.apache.logging.log4j.core.util.Builder<ElasticsearchAppender> {
+
+ private ElasticsearchIndexManager elasticsearchIndexManager;
+
+ private String domain;
+
+ public ElasticsearchAppender.Builder setDomain(final String domain) {
+ this.domain = domain;
+ return this;
+ }
+
+ public ElasticsearchAppender.Builder setIndexManager(
+ final ElasticsearchIndexManager elasticsearchIndexManager) {
+
+ this.elasticsearchIndexManager = elasticsearchIndexManager;
+ return this;
+ }
+
+ @Override
+ public ElasticsearchAppender build() {
+ if (domain == null || elasticsearchIndexManager == null) {
+ LOGGER.error("Cannot create ElasticsearchAppender without Domain or IndexManager.");
+ return null;
+ }
+ return new ElasticsearchAppender(
+ getName(), getFilter(), getLayout(), isIgnoreExceptions(), domain, elasticsearchIndexManager);
+ }
+ }
+
+ @PluginBuilderFactory
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ private final String domain;
+
+ protected final ElasticsearchIndexManager elasticsearchIndexManager;
+
+ protected ElasticsearchAppender(
+ final String name,
+ final Filter filter,
+ final Layout<? extends Serializable> layout,
+ final boolean ignoreExceptions,
+ final String domain,
+ final ElasticsearchIndexManager elasticsearchIndexManager) {
+
+ super(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY);
+ this.domain = domain;
+ this.elasticsearchIndexManager = elasticsearchIndexManager;
+ }
+
+ @Override
+ public void append(final LogEvent event) {
+ try {
+ elasticsearchIndexManager.audit(
+ domain,
+ event.getTimeMillis(),
+ POJOHelper.deserialize(event.getMessage().getFormattedMessage(), AuditEntry.class));
+ } catch (Exception e) {
+ LOGGER.error("While requesting to index event for appender [{}]", getName(), e);
+ }
+ }
+}
diff --git a/ext/elasticsearch/logic/src/main/java/org/apache/syncope/core/logic/audit/ElasticsearchAuditAppender.java b/ext/elasticsearch/logic/src/main/java/org/apache/syncope/core/logic/audit/ElasticsearchAuditAppender.java
new file mode 100644
index 0000000000..120093f595
--- /dev/null
+++ b/ext/elasticsearch/logic/src/main/java/org/apache/syncope/core/logic/audit/ElasticsearchAuditAppender.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.syncope.core.logic.audit;
+
+import java.util.Optional;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.syncope.ext.elasticsearch.client.ElasticsearchIndexManager;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class ElasticsearchAuditAppender extends DefaultAuditAppender {
+
+ @Autowired
+ protected ElasticsearchIndexManager elasticsearchIndexManager;
+
+ @Override
+ protected void initTargetAppender() {
+ LoggerContext logCtx = (LoggerContext) LogManager.getContext(false);
+
+ targetAppender = Optional.ofNullable(logCtx.getConfiguration().<Appender>getAppender(getTargetAppenderName())).
+ orElseGet(() -> {
+ ElasticsearchAppender a = ElasticsearchAppender.newBuilder().
+ setName(getTargetAppenderName()).
+ setIgnoreExceptions(false).
+ setDomain(domain).
+ setIndexManager(elasticsearchIndexManager).
+ build();
+ a.start();
+ logCtx.getConfiguration().addAppender(a);
+ return a;
+ });
+ }
+
+ @Override
+ public String getTargetAppenderName() {
+ return "audit_for_" + domain;
+ }
+}
diff --git a/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java
index 6843a02bd8..89b09ca3f6 100644
--- a/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java
+++ b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java
@@ -165,7 +165,7 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO {
@Override
protected int doCount(final Set<String> adminRealms, final SearchCond cond, final AnyTypeKind kind) {
CountRequest request = new CountRequest(
- ElasticsearchUtils.getContextDomainName(AuthContextUtils.getDomain(), kind)).
+ ElasticsearchUtils.getAnyIndex(AuthContextUtils.getDomain(), kind)).
query(getQueryBuilder(adminRealms, cond, kind));
try {
@@ -224,7 +224,7 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO {
sortBuilders(kind, orderBy).forEach(sourceBuilder::sort);
SearchRequest request = new SearchRequest(
- ElasticsearchUtils.getContextDomainName(AuthContextUtils.getDomain(), kind)).
+ ElasticsearchUtils.getAnyIndex(AuthContextUtils.getDomain(), kind)).
searchType(SearchType.QUERY_THEN_FETCH).
source(sourceBuilder);
diff --git a/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchLoggerDAO.java b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchLoggerDAO.java
new file mode 100644
index 0000000000..78be158a96
--- /dev/null
+++ b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchLoggerDAO.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.syncope.core.persistence.jpa.dao;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.syncope.common.lib.log.AuditEntry;
+import org.apache.syncope.common.lib.types.AuditElements;
+import org.apache.syncope.core.persistence.api.dao.search.OrderByClause;
+import org.apache.syncope.core.provisioning.api.serialization.POJOHelper;
+import org.apache.syncope.core.spring.security.AuthContextUtils;
+import org.apache.syncope.ext.elasticsearch.client.ElasticsearchUtils;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.DisMaxQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.springframework.beans.factory.annotation.Autowired;
+
+@SuppressWarnings("deprecation")
+public class ElasticsearchLoggerDAO extends JPALoggerDAO {
+
+ @Autowired
+ protected org.elasticsearch.client.RestHighLevelClient client;
+
+ @Autowired
+ protected ElasticsearchUtils elasticsearchUtils;
+
+ protected QueryBuilder getQueryBuilder(
+ final String entityKey,
+ final AuditElements.EventCategoryType type,
+ final String category,
+ final String subcategory,
+ final List<String> events,
+ final AuditElements.Result result) {
+
+ List<QueryBuilder> queryBuilders = new ArrayList<>();
+
+ if (entityKey != null) {
+ queryBuilders.add(QueryBuilders.multiMatchQuery(
+ entityKey, "message.before", "message.inputs", "message.output", "message.throwable"));
+ }
+
+ if (type != null) {
+ queryBuilders.add(QueryBuilders.termQuery("message.logger.type", type.name()));
+ }
+
+ if (StringUtils.isNotBlank(category)) {
+ queryBuilders.add(QueryBuilders.termQuery("message.logger.category", category));
+ }
+
+ if (StringUtils.isNotBlank(subcategory)) {
+ queryBuilders.add(QueryBuilders.termQuery("message.logger.subcategory", subcategory));
+ }
+
+ List<QueryBuilder> eventQueryBuilders = events.stream().
+ map(event -> QueryBuilders.termQuery("message.logger.event", event)).
+ collect(Collectors.toList());
+ if (!eventQueryBuilders.isEmpty()) {
+ if (eventQueryBuilders.size() == 1) {
+ queryBuilders.add(eventQueryBuilders.get(0));
+ } else {
+ DisMaxQueryBuilder disMax = QueryBuilders.disMaxQuery();
+ eventQueryBuilders.forEach(disMax::add);
+ queryBuilders.add(disMax);
+ }
+ }
+
+ if (result != null) {
+ queryBuilders.add(QueryBuilders.termQuery("message.logger.result", result.name()));
+ }
+
+ BoolQueryBuilder bool = QueryBuilders.boolQuery();
+ queryBuilders.forEach(bool::must);
+ return bool;
+ }
+
+ @Override
+ public int countAuditEntries(
+ final String entityKey,
+ final AuditElements.EventCategoryType type,
+ final String category,
+ final String subcategory,
+ final List<String> events,
+ final AuditElements.Result result) {
+
+ CountRequest request = new CountRequest(
+ ElasticsearchUtils.getAuditIndex(AuthContextUtils.getDomain())).
+ query(getQueryBuilder(entityKey, type, category, subcategory, events, result));
+ try {
+ return (int) client.count(request, RequestOptions.DEFAULT).getCount();
+ } catch (IOException e) {
+ LOG.error("Search error", e);
+ return 0;
+ }
+ }
+
+ protected List<SortBuilder<?>> sortBuilders(final List<OrderByClause> orderBy) {
+ return orderBy.stream().map(clause -> {
+ String sortField = clause.getField();
+ if ("EVENT_DATE".equalsIgnoreCase(sortField)) {
+ sortField = "message.date";
+ }
+
+ return new FieldSortBuilder(sortField).order(SortOrder.valueOf(clause.getDirection().name()));
+ }).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<AuditEntry> findAuditEntries(
+ final String entityKey,
+ final int page,
+ final int itemsPerPage,
+ final AuditElements.EventCategoryType type,
+ final String category,
+ final String subcategory,
+ final List<String> events,
+ final AuditElements.Result result,
+ final List<OrderByClause> orderBy) {
+
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().
+ query(getQueryBuilder(entityKey, type, category, subcategory, events, result)).
+ from(itemsPerPage * (page <= 0 ? 0 : page - 1)).
+ size(itemsPerPage < 0 ? elasticsearchUtils.getIndexMaxResultWindow() : itemsPerPage);
+ sortBuilders(orderBy).forEach(sourceBuilder::sort);
+
+ SearchRequest request = new SearchRequest(
+ ElasticsearchUtils.getAuditIndex(AuthContextUtils.getDomain())).
+ searchType(SearchType.QUERY_THEN_FETCH).
+ source(sourceBuilder);
+
+ SearchHit[] esResult = null;
+ try {
+ esResult = client.search(request, RequestOptions.DEFAULT).getHits().getHits();
+ } catch (Exception e) {
+ LOG.error("While searching in Elasticsearch", e);
+ }
+
+ return ArrayUtils.isEmpty(esResult)
+ ? Collections.emptyList()
+ : Arrays.stream(esResult).
+ map(hit -> POJOHelper.convertValue(hit.getSourceAsMap().get("message"), AuditEntry.class)).
+ filter(Objects::nonNull).collect(Collectors.toList());
+ }
+}
diff --git a/ext/elasticsearch/persistence-jpa/src/main/resources/persistence.properties b/ext/elasticsearch/persistence-jpa/src/main/resources/persistence.properties
index 6a968d85ba..c0d168303e 100644
--- a/ext/elasticsearch/persistence-jpa/src/main/resources/persistence.properties
+++ b/ext/elasticsearch/persistence-jpa/src/main/resources/persistence.properties
@@ -24,5 +24,5 @@ any.search.visitor=org.apache.syncope.core.persistence.api.search.SearchCondVisi
user.dao=org.apache.syncope.core.persistence.jpa.dao.JPAUserDAO
group.dao=org.apache.syncope.core.persistence.jpa.dao.JPAGroupDAO
anyObject.dao=org.apache.syncope.core.persistence.jpa.dao.JPAAnyObjectDAO
-logger.dao=org.apache.syncope.core.persistence.jpa.dao.JPALoggerDAO
+logger.dao=org.apache.syncope.core.persistence.jpa.dao.ElasticsearchLoggerDAO
openjpa.RemoteCommitProvider=sjvm
diff --git a/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java b/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java
index 51a7236727..59204bb83b 100644
--- a/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java
+++ b/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java
@@ -147,7 +147,7 @@ public class ElasticsearchAnySearchDAOTest {
when(groupDAO.findKey("groupKey")).thenReturn("groupKey");
try (MockedStatic<ElasticsearchUtils> utils = Mockito.mockStatic(ElasticsearchUtils.class)) {
- utils.when(() -> ElasticsearchUtils.getContextDomainName(
+ utils.when(() -> ElasticsearchUtils.getAnyIndex(
SyncopeConstants.MASTER_DOMAIN, AnyTypeKind.USER)).thenReturn("master_user");
// 2. test
@@ -163,7 +163,7 @@ public class ElasticsearchAnySearchDAOTest {
searchDAO.sortBuilders(AnyTypeKind.USER, Collections.emptyList()).forEach(sourceBuilder::sort);
SearchRequest searchRequest = new SearchRequest(
- ElasticsearchUtils.getContextDomainName(AuthContextUtils.getDomain(), AnyTypeKind.USER)).
+ ElasticsearchUtils.getAnyIndex(AuthContextUtils.getDomain(), AnyTypeKind.USER)).
searchType(SearchType.QUERY_THEN_FETCH).
source(sourceBuilder);
diff --git a/ext/elasticsearch/pom.xml b/ext/elasticsearch/pom.xml
index 8e7c0fa4a5..9c8d5324e0 100644
--- a/ext/elasticsearch/pom.xml
+++ b/ext/elasticsearch/pom.xml
@@ -41,6 +41,7 @@ under the License.
<module>client-elasticsearch</module>
<module>persistence-jpa</module>
<module>provisioning-java</module>
+ <module>logic</module>
</modules>
</project>
diff --git a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
index c2e691e101..df665cac9a 100644
--- a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
+++ b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java
@@ -72,16 +72,24 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
return indexManager.defaultSettings();
}
+ protected XContentBuilder auditSettings() throws IOException {
+ return indexManager.defaultSettings();
+ }
+
protected XContentBuilder userMapping() throws IOException {
- return indexManager.defaultMapping();
+ return indexManager.defaultAnyMapping();
}
protected XContentBuilder groupMapping() throws IOException {
- return indexManager.defaultMapping();
+ return indexManager.defaultAnyMapping();
}
protected XContentBuilder anyObjectMapping() throws IOException {
- return indexManager.defaultMapping();
+ return indexManager.defaultAnyMapping();
+ }
+
+ protected XContentBuilder auditMapping() throws IOException {
+ return indexManager.defaultAuditMapping();
}
@Override
@@ -90,20 +98,20 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
LOG.debug("Start rebuilding indexes");
try {
- indexManager.createIndex(
+ indexManager.createAnyIndex(
AuthContextUtils.getDomain(), AnyTypeKind.USER, userSettings(), userMapping());
- indexManager.createIndex(
+ indexManager.createAnyIndex(
AuthContextUtils.getDomain(), AnyTypeKind.GROUP, groupSettings(), groupMapping());
- indexManager.createIndex(
+ indexManager.createAnyIndex(
AuthContextUtils.getDomain(), AnyTypeKind.ANY_OBJECT, anyObjectSettings(), anyObjectMapping());
LOG.debug("Indexing users...");
for (int page = 1; page <= (userDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
for (String user : userDAO.findAllKeys(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
IndexRequest request = new IndexRequest(
- ElasticsearchUtils.getContextDomainName(
+ ElasticsearchUtils.getAnyIndex(
AuthContextUtils.getDomain(), AnyTypeKind.USER)).
id(user).
source(utils.builder(userDAO.find(user), AuthContextUtils.getDomain()));
@@ -120,7 +128,7 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
for (int page = 1; page <= (groupDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
for (String group : groupDAO.findAllKeys(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
IndexRequest request = new IndexRequest(
- ElasticsearchUtils.getContextDomainName(
+ ElasticsearchUtils.getAnyIndex(
AuthContextUtils.getDomain(), AnyTypeKind.GROUP)).
id(group).
source(utils.builder(groupDAO.find(group), AuthContextUtils.getDomain()));
@@ -137,7 +145,7 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
for (int page = 1; page <= (anyObjectDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) {
for (String anyObject : anyObjectDAO.findAllKeys(page, AnyDAO.DEFAULT_PAGE_SIZE)) {
IndexRequest request = new IndexRequest(
- ElasticsearchUtils.getContextDomainName(
+ ElasticsearchUtils.getAnyIndex(
AuthContextUtils.getDomain(), AnyTypeKind.ANY_OBJECT)).
id(anyObject).
source(utils.builder(anyObjectDAO.find(anyObject), AuthContextUtils.getDomain()));
@@ -150,6 +158,9 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate {
}
}
+ indexManager.createAuditIndex(
+ AuthContextUtils.getDomain(), auditSettings(), auditMapping());
+
LOG.debug("Rebuild indexes for domain {} successfully completed", AuthContextUtils.getDomain());
} catch (Exception e) {
throw new JobExecutionException("While rebuilding index for domain " + AuthContextUtils.getDomain(), e);
diff --git a/fit/core-reference/pom.xml b/fit/core-reference/pom.xml
index f245e41a18..5b7d665056 100644
--- a/fit/core-reference/pom.xml
+++ b/fit/core-reference/pom.xml
@@ -446,6 +446,11 @@ under the License.
<id>elasticsearch-it</id>
<dependencies>
+ <dependency>
+ <groupId>org.apache.syncope.ext.elasticsearch</groupId>
+ <artifactId>syncope-ext-elasticsearch-logic</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.syncope.ext.elasticsearch</groupId>
<artifactId>syncope-ext-elasticsearch-provisioning-java</artifactId>
@@ -499,6 +504,7 @@ under the License.
<discovery.type>single-node</discovery.type>
<cluster.name>elasticsearch</cluster.name>
<xpack.security.enabled>false</xpack.security.enabled>
+ <ES_JAVA_OPTS>-Xms750m -Xmx750m</ES_JAVA_OPTS>
</env>
<ports>
<port>9200:9200</port>
diff --git a/fit/core-reference/src/main/resources/logic.properties b/fit/core-reference/src/main/resources/elasticsearch/logic.properties
similarity index 91%
copy from fit/core-reference/src/main/resources/logic.properties
copy to fit/core-reference/src/main/resources/elasticsearch/logic.properties
index 5e8f1ba06b..f4705deb91 100644
--- a/fit/core-reference/src/main/resources/logic.properties
+++ b/fit/core-reference/src/main/resources/elasticsearch/logic.properties
@@ -16,4 +16,4 @@
# under the License.
logicInvocationHandler=org.apache.syncope.core.logic.LogicInvocationHandler
classPathScanImplementationLookup=org.apache.syncope.fit.core.reference.ITImplementationLookup
-enable.jdbcAuditAppender=true
+default.audit.appender=org.apache.syncope.core.logic.audit.ElasticsearchAuditAppender
diff --git a/fit/core-reference/src/main/resources/elasticsearch/persistence.properties b/fit/core-reference/src/main/resources/elasticsearch/persistence.properties
index a4517014a8..3c38f2a21c 100644
--- a/fit/core-reference/src/main/resources/elasticsearch/persistence.properties
+++ b/fit/core-reference/src/main/resources/elasticsearch/persistence.properties
@@ -25,5 +25,5 @@ user.dao=org.apache.syncope.core.persistence.jpa.dao.JPAUserDAO
group.dao=org.apache.syncope.core.persistence.jpa.dao.JPAGroupDAO
anyObject.dao=org.apache.syncope.core.persistence.jpa.dao.JPAAnyObjectDAO
conf.dao=org.apache.syncope.core.persistence.jpa.dao.JPAConfDAO
-logger.dao=org.apache.syncope.core.persistence.jpa.dao.JPALoggerDAO
+logger.dao=org.apache.syncope.core.persistence.jpa.dao.ElasticsearchLoggerDAO
openjpa.RemoteCommitProvider=sjvm
diff --git a/fit/core-reference/src/main/resources/logic.properties b/fit/core-reference/src/main/resources/logic.properties
index 5e8f1ba06b..b746565dcc 100644
--- a/fit/core-reference/src/main/resources/logic.properties
+++ b/fit/core-reference/src/main/resources/logic.properties
@@ -16,4 +16,4 @@
# under the License.
logicInvocationHandler=org.apache.syncope.core.logic.LogicInvocationHandler
classPathScanImplementationLookup=org.apache.syncope.fit.core.reference.ITImplementationLookup
-enable.jdbcAuditAppender=true
+default.audit.appender=org.apache.syncope.core.logic.audit.JdbcAuditAppender
diff --git a/fit/core-reference/src/test/java/org/apache/syncope/fit/AbstractITCase.java b/fit/core-reference/src/test/java/org/apache/syncope/fit/AbstractITCase.java
index de41c63034..cb900625a7 100644
--- a/fit/core-reference/src/test/java/org/apache/syncope/fit/AbstractITCase.java
+++ b/fit/core-reference/src/test/java/org/apache/syncope/fit/AbstractITCase.java
@@ -710,6 +710,14 @@ public abstract class AbstractITCase {
}
protected static List<AuditEntry> query(final AuditQuery query, final int maxWaitSeconds) {
+ if (ElasticsearchDetector.isElasticSearchEnabled(syncopeService)) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException ex) {
+ // ignore
+ }
+ }
+
int i = 0;
List<AuditEntry> results = Collections.emptyList();
do {
diff --git a/src/main/asciidoc/reference-guide/concepts/audit.adoc b/src/main/asciidoc/reference-guide/concepts/audit.adoc
index b8adfe39e3..03e45bfa8f 100644
--- a/src/main/asciidoc/reference-guide/concepts/audit.adoc
+++ b/src/main/asciidoc/reference-guide/concepts/audit.adoc
@@ -20,7 +20,8 @@
The audit feature allows to capture <<audit-events,events>> occurring within the <<core>> and to log relevant information
about them. +
-By default, events are logged as entries into the `SYNCOPEAUDIT` table of the internal storage.
+By default, events are logged as entries into the `SYNCOPEAUDIT` table of the internal storage. +
+Audit events can also be processed differently, for example when using the <<elasticsearch>> extension.
Once events are reported, they can be used as input for external tools.
@@ -36,7 +37,7 @@ except for the admin console <<console-configuration-audit,tooling>>, which is n
==== Audit Appenders
-In addition to insertions into the `SYNCOPEAUDIT` table, events are also available for custom handling via Audit
+In addition to default processing, events are also available for custom handling via Audit
Appenders, based on https://logging.apache.org/log4j/2.x/manual/appenders.html[Apache Log4j 2 Appenders^]. +
This allows to empower the available implementations or to write new ones in order to route audit messages, with optional
transformation (rewrite), to files, queues, sockets, syslog, etc.
diff --git a/src/main/asciidoc/reference-guide/concepts/extensions.adoc b/src/main/asciidoc/reference-guide/concepts/extensions.adoc
index f1c7742ad4..d1c5a24b57 100644
--- a/src/main/asciidoc/reference-guide/concepts/extensions.adoc
+++ b/src/main/asciidoc/reference-guide/concepts/extensions.adoc
@@ -151,8 +151,8 @@ This extension adds features to all components and layers that are available, an
==== Elasticsearch
-This extension provides an alternate internal search engine for <<users-groups-and-any-objects>>, requiring an external
-https://www.elastic.co/[Elasticsearch^] cluster.
+This extension provides an alternate internal search engine for <<users-groups-and-any-objects>> and <<audit-events>>,
+requiring an external https://www.elastic.co/[Elasticsearch^] cluster.
[WARNING]
This extension supports Elasticsearch server versions starting from 7.x.
diff --git a/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc b/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc
index d38ee26bf1..2f13405f3c 100644
--- a/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc
+++ b/src/main/asciidoc/reference-guide/workingwithapachesyncope/customization.adoc
@@ -417,6 +417,11 @@ Add the following dependencies to `core/pom.xml`:
[source,xml,subs="verbatim,attributes"]
----
+<dependency>
+ <groupId>org.apache.syncope.ext.elasticsearch</groupId>
+ <artifactId>syncope-ext-elasticsearch-logic</artifactId>
+ <version>${syncope.version}</version>
+</dependency>
<dependency>
<groupId>org.apache.syncope.ext.elasticsearch</groupId>
<artifactId>syncope-ext-elasticsearch-provisioning-java</artifactId>
@@ -429,6 +434,20 @@ Add the following dependencies to `core/pom.xml`:
</dependency>
----
+Replace
+
+....
+default.audit.appender=org.apache.syncope.core.logic.audit.JdbcAuditAppender
+....
+
+with
+
+....
+default.audit.appender=org.apache.syncope.core.logic.audit.ElasticsearchAuditAppender
+....
+
+in `core/src/main/resources/logic.properties`.
+
Download
ifeval::["{snapshotOrRelease}" == "release"]
@@ -484,7 +503,7 @@ Then, create a new <<tasks-custom, custom task>>, select the implementation just
[TIP]
The `org.apache.syncope.core.provisioning.java.job.ElasticsearchReindex` custom task created above is not meant for
scheduled execution; rather, it can be run every time you want to blank and re-create the Elasticsearch indexes
-starting from Syncope's users, groups and any objects.
+starting from Syncope's internal storage.
[discrete]
===== Enable the <<SCIM>> extension