You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2018/07/09 18:26:00 UTC
[08/50] [abbrv] hadoop git commit: HDDS-198. Create AuditLogger
mechanism to be used by OM,
SCM and Datanode. Contributed by Dinesh Chitlangia.
HDDS-198. Create AuditLogger mechanism to be used by OM, SCM and Datanode.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0ef7e76
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0ef7e76
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0ef7e76
Branch: refs/heads/HDFS-12090
Commit: c0ef7e7680d882e2182f48f033109678a48742ab
Parents: 51654a3
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Jul 3 11:38:14 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Jul 3 11:39:50 2018 -0700
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 18 +++
.../apache/hadoop/ozone/audit/AuditAction.java | 30 +++++
.../hadoop/ozone/audit/AuditEventStatus.java | 36 +++++
.../apache/hadoop/ozone/audit/AuditLogger.java | 128 ++++++++++++++++++
.../hadoop/ozone/audit/AuditLoggerType.java | 37 ++++++
.../apache/hadoop/ozone/audit/AuditMarker.java | 38 ++++++
.../apache/hadoop/ozone/audit/Auditable.java | 32 +++++
.../apache/hadoop/ozone/audit/package-info.java | 123 +++++++++++++++++
.../apache/hadoop/ozone/audit/DummyAction.java | 51 ++++++++
.../apache/hadoop/ozone/audit/DummyEntity.java | 57 ++++++++
.../ozone/audit/TestOzoneAuditLogger.java | 131 +++++++++++++++++++
.../apache/hadoop/ozone/audit/package-info.java | 23 ++++
.../common/src/test/resources/log4j2.properties | 76 +++++++++++
.../org/apache/hadoop/ozone/audit/OMAction.java | 51 ++++++++
.../apache/hadoop/ozone/audit/package-info.java | 22 ++++
15 files changed, 853 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index a8a634c..4068522 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -31,6 +31,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<hadoop.component>hdds</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
+ <log4j2.version>2.11.0</log4j2.version>
+ <disruptor.version>3.4.2</disruptor.version>
</properties>
<dependencies>
@@ -81,6 +83,22 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>${disruptor.version}</version>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java
new file mode 100644
index 0000000..8c1d6f0
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.audit;
+
+/**
+ * Interface to define AuditAction.
+ */
+public interface AuditAction {
+ /**
+ * Implementation must override.
+ * @return String
+ */
+ String getAction();
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java
new file mode 100644
index 0000000..098ab6b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.audit;
+
+/**
+ * Enum to define AuditEventStatus values.
+ */
+public enum AuditEventStatus {
+ SUCCESS("SUCCESS"),
+ FAILURE("FAILURE");
+
+ private String status;
+
+ AuditEventStatus(String status){
+ this.status = status;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
new file mode 100644
index 0000000..46ffaab
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.audit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.message.StructuredDataMessage;
+import org.apache.logging.log4j.spi.ExtendedLogger;
+
+import java.util.Map;
+
+/**
+ * Class to define Audit Logger for Ozone.
+ */
+public class AuditLogger {
+
+ private ExtendedLogger logger;
+
+ private static final String SUCCESS = AuditEventStatus.SUCCESS.getStatus();
+ private static final String FAILURE = AuditEventStatus.FAILURE.getStatus();
+ private static final String FQCN = AuditLogger.class.getName();
+ private static final Marker WRITE_MARKER = AuditMarker.WRITE.getMarker();
+ private static final Marker READ_MARKER = AuditMarker.READ.getMarker();
+
+ /**
+ * Parametrized Constructor to initialize logger.
+ * @param type
+ */
+ public AuditLogger(AuditLoggerType type){
+ initializeLogger(type);
+ }
+
+ /**
+ * Initializes the logger with specific type.
+ * @param loggerType specified one of the values from enum AuditLoggerType.
+ */
+ private void initializeLogger(AuditLoggerType loggerType){
+ this.logger = LogManager.getContext(false).getLogger(loggerType.getType());
+ }
+
+ @VisibleForTesting
+ public ExtendedLogger getLogger() {
+ return logger;
+ }
+
+ public void logWriteSuccess(AuditAction type, Map<String, String> data) {
+ logWriteSuccess(type, data, Level.INFO);
+ }
+
+ public void logWriteSuccess(AuditAction type, Map<String, String> data, Level
+ level) {
+ StructuredDataMessage msg = new StructuredDataMessage("", SUCCESS,
+ type.getAction(), data);
+ this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, null);
+ }
+
+
+ public void logWriteFailure(AuditAction type, Map<String, String> data) {
+ logWriteFailure(type, data, Level.INFO, null);
+ }
+
+ public void logWriteFailure(AuditAction type, Map<String, String> data, Level
+ level) {
+ logWriteFailure(type, data, level, null);
+ }
+
+ public void logWriteFailure(AuditAction type, Map<String, String> data,
+ Throwable exception) {
+ logWriteFailure(type, data, Level.INFO, exception);
+ }
+
+ public void logWriteFailure(AuditAction type, Map<String, String> data, Level
+ level, Throwable exception) {
+ StructuredDataMessage msg = new StructuredDataMessage("", FAILURE,
+ type.getAction(), data);
+ this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, exception);
+ }
+
+ public void logReadSuccess(AuditAction type, Map<String, String> data) {
+ logReadSuccess(type, data, Level.INFO);
+ }
+
+ public void logReadSuccess(AuditAction type, Map<String, String> data, Level
+ level) {
+ StructuredDataMessage msg = new StructuredDataMessage("", SUCCESS,
+ type.getAction(), data);
+ this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, null);
+ }
+
+ public void logReadFailure(AuditAction type, Map<String, String> data) {
+ logReadFailure(type, data, Level.INFO, null);
+ }
+
+ public void logReadFailure(AuditAction type, Map<String, String> data, Level
+ level) {
+ logReadFailure(type, data, level, null);
+ }
+
+ public void logReadFailure(AuditAction type, Map<String, String> data,
+ Throwable exception) {
+ logReadFailure(type, data, Level.INFO, exception);
+ }
+
+ public void logReadFailure(AuditAction type, Map<String, String> data, Level
+ level, Throwable exception) {
+ StructuredDataMessage msg = new StructuredDataMessage("", FAILURE,
+ type.getAction(), data);
+ this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, exception);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java
new file mode 100644
index 0000000..18241c7
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.audit;
+
+/**
+ * Enumeration for defining types of Audit Loggers in Ozone.
+ */
+public enum AuditLoggerType {
+ DNLOGGER("DNAudit"),
+ OMLOGGER("OMAudit"),
+ SCMLOGGER("SCMAudit");
+
+ private String type;
+
+ public String getType() {
+ return type;
+ }
+
+ AuditLoggerType(String type){
+ this.type = type;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java
new file mode 100644
index 0000000..505b958
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.audit;
+
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
+
+/**
+ * Defines audit marker types.
+ */
+public enum AuditMarker {
+ WRITE(MarkerManager.getMarker("WRITE")),
+ READ(MarkerManager.getMarker("READ"));
+
+ private Marker marker;
+
+ AuditMarker(Marker marker){
+ this.marker = marker;
+ }
+
+ public Marker getMarker(){
+ return marker;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java
new file mode 100644
index 0000000..d388bca
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.audit;
+
+import java.util.Map;
+
+/**
+ * Interface to make an entity auditable.
+ */
+public interface Auditable {
+ /**
+ * Must override in implementation.
+ * @return Map<String, String> with values to be logged in audit.
+ */
+ Map<String, String> toAuditMap();
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
new file mode 100644
index 0000000..3743fdd
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.audit;
+/**
+ ******************************************************************************
+ * Important
+ * 1. Any changes to classes in this package can render the logging
+ * framework broken.
+ * 2. The logger framework has been designed keeping in mind future
+ * plans to build a log parser.
+ * 3. Please exercise great caution when attempting changes in this package.
+ ******************************************************************************
+ *
+ *
+ * This package lays the foundation for Audit logging in Ozone.
+ * AuditLogging in Ozone has been built using log4j2 which brings in new
+ * features that facilitate turning on/off selective audit events by using
+ * MarkerFilter, checking for change in logging configuration periodically
+ * and reloading the changes, use of disruptor framework for improved
+ * Asynchronous logging.
+ *
+ * The log4j2 configurations can be specified in XML, YAML, JSON and
+ * Properties file. For Ozone, we are using the Properties file due to sheer
+ * simplicity, readability and ease of modification.
+ *
+ * log4j2 configuration file can be passed to startup command with option
+ * -Dlog4j.configurationFile unlike -Dlog4j.configuration in log4j 1.x
+ *
+ ******************************************************************************
+ * Understanding the Audit Logging framework in Ozone.
+ ******************************************************************************
+ * **** Auditable ***
+ * This is an interface to mark an entity as auditable.
+ * This interface must be implemented by entities requiring audit logging.
+ * For example - KSMVolumeArgs, KSMBucketArgs.
+ * The implementing class must override toAuditMap() to return an
+ * instance of Map<Key, Value> where both Key and Value are String.
+ *
+ * Key: must not contain any spaces. If the key is multi word then use
+ * camel case.
+ * Value: if it is a collection/array, then it must be converted to a comma
+ * delimited string
+ *
+ * *** AuditAction ***
+ * This is an interface to define the various type of actions to be audited.
+ * To ensure separation of concern, for each sub-component you must create an
+ * Enum to implement AuditAction.
+ * Structure of Enum can be referred from the test class DummyAction.
+ *
+ * For starters, we expect following 3 implementations of AuditAction:
+ * OMAction - to define action types for Ozone Manager
+ * SCMAction - to define action types for Storage Container manager
+ * DNAction - to define action types for Datanode
+ *
+ * *** AuditEventStatus ***
+ * Enum to define Audit event status like success and failure.
+ * This is used in AuditLogger.logXXX() methods.
+ *
+ * * *** AuditLogger ***
+ * This is where the audit logging magic unfolds.
+ * The class has 2 Markers defined - READ and WRITE.
+ * These markers are used to tag when logging events.
+ *
+ * *** AuditLoggerType ***
+ * Enum to define the various AuditLoggers in Ozone
+ *
+ * *** AuditMarker ***
+ * Enum to define various Audit Markers used in AuditLogging.
+ *
+ * ****************************************************************************
+ * Usage
+ * ****************************************************************************
+ * Using the AuditLogger to log events:
+ * 1. Get a logger by specifying the appropriate logger type
+ * Example: ExtendedLogger AUDIT = new AuditLogger(AuditLoggerType.OMLogger)
+ *
+ * 2. Log Read/Write and Success/Failure event as needed.
+ * Example
+ * AUDIT.logWriteSuccess(AuditAction type, Map<String, String> data, Level
+ * level)
+ *
+ * If logging is done without specifying Level, then Level implicitly
+ * defaults to INFO
+ * AUDIT.logWriteSuccess(AuditAction type, Map<String, String> data)
+ *
+ * See sample invocations in src/test in the following class:
+ * org.apache.hadoop.ozone.audit.TestOzoneAuditLogger
+ *
+ * ****************************************************************************
+ * Defining new Logger types
+ * ****************************************************************************
+ * New Logger type can be added with following steps:
+ * 1. Update AuditLoggerType to add the new type
+ * 2. Create new Enum by implementing AuditAction if needed
+ * 3. Ensure the required entity implements Auditable
+ *
+ * ****************************************************************************
+ * Defining new Marker types
+ * ****************************************************************************
+ * New Markers can be configured as follows:
+ * 1. Define new markers in AuditMarker
+ * 2. Get the Marker in AuditLogger for use in the log methods, example:
+ * private static final Marker WRITE_MARKER = AuditMarker.WRITE.getMarker();
+ * 3. Define log methods in AuditLogger to use the new Marker type
+ * 4. Call these new methods from the required classes to audit with these
+ * new markers
+ * 5. The marker based filtering can be configured in log4j2 configurations
+ * Refer log4j2.properties in src/test/resources for a sample.
+ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
new file mode 100644
index 0000000..6044c0a
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.audit;
+
+/**
+ * Enum to define Dummy AuditAction Type for test.
+ */
+public enum DummyAction implements AuditAction {
+
+ CREATE_VOLUME("CREATE_VOLUME"),
+ CREATE_BUCKET("CREATE_BUCKET"),
+ CREATE_KEY("CREATE_KEY"),
+ READ_VOLUME("READ_VOLUME"),
+ READ_BUCKET("READ_BUCKET"),
+ READ_KEY("READ_BUCKET"),
+ UPDATE_VOLUME("UPDATE_VOLUME"),
+ UPDATE_BUCKET("UPDATE_BUCKET"),
+ UPDATE_KEY("UPDATE_KEY"),
+ DELETE_VOLUME("DELETE_VOLUME"),
+ DELETE_BUCKET("DELETE_BUCKET"),
+ DELETE_KEY("DELETE_KEY"),
+ SET_OWNER("SET_OWNER"),
+ SET_QUOTA("SET_QUOTA");
+
+ private String action;
+
+ DummyAction(String action) {
+ this.action = action;
+ }
+
+ @Override
+ public String getAction() {
+ return this.action;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java
new file mode 100644
index 0000000..0c2d98f
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.audit;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * DummyEntity that implements Auditable for test purpose.
+ */
+public class DummyEntity implements Auditable {
+
+ private String key1;
+ private String key2;
+
+ public DummyEntity(){
+ this.key1 = "value1";
+ this.key2 = "value2";
+ }
+ public String getKey1() {
+ return key1;
+ }
+
+ public void setKey1(String key1) {
+ this.key1 = key1;
+ }
+
+ public String getKey2() {
+ return key2;
+ }
+
+ public void setKey2(String key2) {
+ this.key2 = key2;
+ }
+
+ @Override
+ public Map<String, String> toAuditMap() {
+ Map<String, String> auditMap = new HashMap<>();
+ auditMap.put("key1", this.key1);
+ auditMap.put("key2", this.key2);
+ return auditMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
new file mode 100644
index 0000000..d3cc9e4
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.audit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.Level;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test Ozone Audit Logger.
+ */
+public class TestOzoneAuditLogger {
+
+ private static final Logger LOG = LoggerFactory.getLogger
+ (TestOzoneAuditLogger.class.getName());
+ private static AuditLogger AUDIT = new AuditLogger(AuditLoggerType.OMLOGGER);
+ public DummyEntity auditableObj = new DummyEntity();
+
+ @BeforeClass
+ public static void setUp(){
+ System.setProperty("log4j.configurationFile", "log4j2.properties");
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ File file = new File("audit.log");
+ if (FileUtils.deleteQuietly(file)) {
+ LOG.info(file.getName() +
+ " has been deleted as all tests have completed.");
+ } else {
+ LOG.info("audit.log could not be deleted.");
+ }
+ }
+
+ /**
+ * Ensures WriteSuccess events are logged @ INFO and above.
+ */
+ @Test
+ public void logInfoWriteSuccess() throws IOException {
+ AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.INFO);
+ String expected = "[INFO ] OMAudit - CREATE_VOLUME [ key1=\"value1\" " +
+ "key2=\"value2\"] SUCCESS";
+ verifyLog(expected);
+ }
+
+ /**
+ * Test to verify default log level is INFO
+ */
+ @Test
+ public void verifyDefaultLogLevel() throws IOException {
+ AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap());
+ String expected = "[INFO ] OMAudit - CREATE_VOLUME [ key1=\"value1\" " +
+ "key2=\"value2\"] SUCCESS";
+ verifyLog(expected);
+ }
+
+ /**
+ * Test to verify WriteFailure events are logged as ERROR.
+ */
+ @Test
+ public void logErrorWriteFailure() throws IOException {
+ AUDIT.logWriteFailure(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.ERROR);
+ String expected = "[ERROR] OMAudit - CREATE_VOLUME [ key1=\"value1\" " +
+ "key2=\"value2\"] FAILURE";
+ verifyLog(expected);
+ }
+
+ /**
+ * Test to verify no READ event is logged.
+ */
+ @Test
+ public void notLogReadEvents() throws IOException {
+ AUDIT.logReadSuccess(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.INFO);
+ AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.INFO);
+ AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.ERROR);
+ AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.ERROR,
+ new Exception("test"));
+ verifyLog(null);
+ }
+
+ /**
+ * Test to ensure DEBUG level messages are not logged when INFO is enabled.
+ */
+ @Test
+ public void notLogDebugEvents() throws IOException {
+ AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.DEBUG);
+ AUDIT.logReadSuccess(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.DEBUG);
+ verifyLog(null);
+ }
+
+ public void verifyLog(String expected) throws IOException {
+ File file = new File("audit.log");
+ List<String> lines = FileUtils.readLines(file, (String)null);
+ if(expected == null){
+ // When no log entry is expected, the log file must be empty
+ assertTrue(lines.size() == 0);
+ } else {
+ // When log entry is expected, the log file will contain one line and
+ // that must be equal to the expected string
+ assertTrue(expected.equalsIgnoreCase(lines.get(0)));
+ //empty the file
+ lines.remove(0);
+ FileUtils.writeLines(file, lines, false);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java
new file mode 100644
index 0000000..1222ad0
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.audit;
+/**
+ * Unit tests of Ozone Audit Logger.
+ * For test purpose, the log4j2 configuration is loaded from file at:
+ * src/test/resources/log4j2.properties
+ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-hdds/common/src/test/resources/log4j2.properties
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/resources/log4j2.properties b/hadoop-hdds/common/src/test/resources/log4j2.properties
new file mode 100644
index 0000000..d60df18
--- /dev/null
+++ b/hadoop-hdds/common/src/test/resources/log4j2.properties
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# <p>
+# http://www.apache.org/licenses/LICENSE-2.0
+# <p>
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+name=PropertiesConfig
+
+# Checks for config change periodically and reloads
+monitorInterval=5
+
+filter=read, write
+# filter.read.onMatch = DENY avoids logging all READ events
+# filter.read.onMatch = ACCEPT permits logging all READ events
+# The above two settings ignore the log levels in configuration
+# filter.read.onMatch = NEUTRAL permits logging of only those READ events
+# which are attempted at log level equal or greater than log level specified
+# in the configuration
+filter.read.type = MarkerFilter
+filter.read.marker = READ
+filter.read.onMatch = DENY
+filter.read.onMismatch = NEUTRAL
+
+# filter.write.onMatch = DENY avoids logging all WRITE events
+# filter.write.onMatch = ACCEPT permits logging all WRITE events
+# The above two settings ignore the log levels in configuration
+# filter.write.onMatch = NEUTRAL permits logging of only those WRITE events
+# which are attempted at log level equal or greater than log level specified
+# in the configuration
+filter.write.type = MarkerFilter
+filter.write.marker = WRITE
+filter.write.onMatch = NEUTRAL
+filter.write.onMismatch = NEUTRAL
+
+# Log Levels are organized from most specific to least:
+# OFF (most specific, no logging)
+# FATAL (most specific, little data)
+# ERROR
+# WARN
+# INFO
+# DEBUG
+# TRACE (least specific, a lot of data)
+# ALL (least specific, all data)
+
+appenders = console, audit
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%-5level] %c{1} - %msg%n
+
+appender.audit.type = File
+appender.audit.name = AUDITLOG
+appender.audit.fileName=audit.log
+appender.audit.layout.type=PatternLayout
+appender.audit.layout.pattern= [%-5level] %c{1} - %msg%n
+
+loggers=audit
+logger.audit.type=AsyncLogger
+logger.audit.name=OMAudit
+logger.audit.level = INFO
+logger.audit.appenderRefs = audit
+logger.audit.appenderRef.file.ref = AUDITLOG
+
+rootLogger.level = INFO
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
new file mode 100644
index 0000000..d780ea2
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.audit;
+
+/**
+ * Enum to define OM Action types for Audit.
+ */
+public enum OMAction implements AuditAction {
+
+ CREATE_VOLUME("CREATE_VOLUME"),
+ CREATE_BUCKET("CREATE_BUCKET"),
+ CREATE_KEY("CREATE_KEY"),
+ READ_VOLUME("READ_VOLUME"),
+ READ_BUCKET("READ_BUCKET"),
+ READ_KEY("READ_BUCKET"),
+ UPDATE_VOLUME("UPDATE_VOLUME"),
+ UPDATE_BUCKET("UPDATE_BUCKET"),
+ UPDATE_KEY("UPDATE_KEY"),
+ DELETE_VOLUME("DELETE_VOLUME"),
+ DELETE_BUCKET("DELETE_BUCKET"),
+ DELETE_KEY("DELETE_KEY"),
+ SET_OWNER("SET_OWNER"),
+ SET_QUOTA("SET_QUOTA");
+
+ private String action;
+
+ OMAction(String action) {
+ this.action = action;
+ }
+
+ @Override
+ public String getAction() {
+ return this.action;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ef7e76/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
new file mode 100644
index 0000000..0f88790
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.audit;
+/**
+ * This package defines OMAction - an implementation of AuditAction
+ * OMAction defines audit action types for various actions that will be
+ * audited in OzoneManager.
+ */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org