You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by iv...@apache.org on 2021/08/27 14:10:53 UTC
[pulsar] branch master updated: [Issue #11799][logging] Interface
for structured event logging (#11800)
This is an automated email from the ASF dual-hosted git repository.
ivank pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 879ab3a [Issue #11799][logging] Interface for structured event logging (#11800)
879ab3a is described below
commit 879ab3a174b6cc78c15199f8c6fe6aefb1959116
Author: Ivan Kelly <iv...@apache.org>
AuthorDate: Fri Aug 27 15:10:15 2021 +0100
[Issue #11799][logging] Interface for structured event logging (#11800)
This change has just the interface for structured event logging.
Implementation will follow.
Master issue: #11799
---
pom.xml | 2 +
structured-event-log/pom.xml | 56 +++++++
.../apache/pulsar/structuredeventlog/Event.java | 181 +++++++++++++++++++++
.../pulsar/structuredeventlog/EventResources.java | 52 ++++++
.../structuredeventlog/StructuredEventLog.java | 89 ++++++++++
5 files changed, 380 insertions(+)
diff --git a/pom.xml b/pom.xml
index 7b508ea..d852b1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1973,6 +1973,8 @@ flexible messaging model and an intuitive client API.</description>
<module>pulsar-client-auth-sasl</module>
<module>pulsar-config-validation</module>
+ <module>structured-event-log</module>
+
<!-- transaction related modules -->
<module>pulsar-transaction</module>
diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml
new file mode 100644
index 0000000..de872f8
--- /dev/null
+++ b/structured-event-log/pom.xml
@@ -0,0 +1,56 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar</artifactId>
+ <version>2.9.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>structured-event-log</artifactId>
+ <name>Structured event logger</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Event.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Event.java
new file mode 100644
index 0000000..3074617
--- /dev/null
+++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Event.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.structuredeventlog;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+/**
+ * Structured Logged Event interface.
+ *
+ * This interface is used to add context information to the log event and eventually log.
+ */
+public interface Event {
+ /**
+ * Create a new child event. The child event will inherit the trace ID of the event
+ * from which it was created. The child event parentId will be the ID of the event
+ * from which it was created.
+ * The child will inherit resources from its parent event, but not attributes.
+ *
+ * @return a new child event
+ */
+ Event newChildEvent();
+
+ /**
+ * Set the trace ID of the event. Normally this will be inherited from the parent
+ * event. In the case of a root event, the trace ID will be automatically generated.
+ * This method only needs to be used if a traceID has been received from elsewhere,
+ * such as from a user request.
+ * @param traceId the traceId
+ * @return this
+ */
+ Event traceId(String traceId);
+
+ /**
+ * Set the parent ID of the event. Normally this is set automatically for child events.
+ * For root events, it is normally empty unless provided by some other means, such
+ * as via a user request.
+ * @param parentId the parentId
+ * @return this
+ */
+ Event parentId(String parentId);
+
+ /**
+ * Mark this event as timed.
+ * Timed events will measure the duration between #timed() being called
+ * and logged being called. The duration will be in milliseconds.
+ *
+ * <pre>
+ * Event e = logger.newRootEvent().timed();
+ * // do something that takes time.
+ * e.log(Events.SOME_EVENT);
+ * </pre>
+ *
+ * @return this
+ */
+ Event timed();
+
+ /**
+ * Mark this event as sampled.
+ * Sampled events will only log once per specified duration.
+ * The sampling key is used to scope the rate limit. All events using the
+ * same sampling key will observe the same rate limit.
+ * Sampled events are most useful in the data plane, where, if there is an
+ * error for one event, there's likely to be a lot of other events which are
+ * almost identical.
+ *
+ * @param samplingKey a key by which to scope the rate limiting
+ * @param duration the duration for which one event will be logged
+ * @param unit the duration unit
+ * @return this
+ */
+ Event sampled(Object samplingKey, int duration, TimeUnit unit);
+
+ /**
+ * Add resources for the event from an EventResources object.
+ * @see #resource(java.lang.String,java.lang.Object)
+ * @return this
+ */
+ Event resources(EventResources attrs);
+
+ /**
+ * Add a resource for the event. Resources are inherited by
+ * child events.
+ * @param key the key to identify the resource
+ * @param value the value which will be logged for the resource.
+ * This is converted to a string before logging.
+ * @return this
+ */
+ Event resource(String key, Object value);
+
+ /**
+ * Add a resource for the event using a supplier. The supplier is
+ * used in the case that generating the string from the object is
+ * expensive or we want to generate a custom string.
+ * @param key the key to identify the resource
+ * @param value a supplier which returns the value to be logged for
+ * this resource
+ * @see #resource(java.lang.String,java.lang.Object)
+ */
+ Event resource(String key, Supplier<String> value);
+
+ /**
+ * Add an attribute for the event. Attributes are not inherited
+ * by child events.
+ * @param key the key to identify the attribute
+ * @param value the value which will be logged for the attribute.
+ * This is converted to a string, using Object#toString() before logging.
+ * @return this
+ */
+ Event attr(String key, Object value);
+
+ /**
+ * Add an attribute for the event using a supplier.
+ * @param key the key to identify the attribute
+ * @param value a supplier which returns the value to be logged for
+ * this attribute
+ * @return this
+ */
+ Event attr(String key, Supplier<String> value);
+
+ /**
+ * Attach an exception to the event.
+ * @param t the exception
+ * @return this
+ */
+ Event exception(Throwable t);
+
+ /**
+ * Log this event at the error level.
+ * @return this
+ */
+ Event atError();
+
+ /**
+ * Log this event at the info level (the default).
+ * @return this
+ */
+ Event atInfo();
+
+ /**
+ * Log this event at the warn level.
+ * @return this
+ */
+ Event atWarn();
+
+ /**
+ * Log the event, using an enum as the message.
+ * Logging with a enum allows documentation and annotations, such as subcomponent
+ * to be attached to the message.
+ * @param event the event message, in enum form
+ */
+ void log(Enum<?> event);
+
+ /**
+ * Log the event, using a string.
+ * @param event the event message.
+ */
+ void log(String event);
+
+ /**
+ * Stash this log event to bridge across an unmodifiable API call.
+ * @see StructuredEventLog#unstash()
+ */
+ void stash();
+}
diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/EventResources.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/EventResources.java
new file mode 100644
index 0000000..499525e
--- /dev/null
+++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/EventResources.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.structuredeventlog;
+
+import java.util.function.Supplier;
+
+/**
+ * A container for resources
+ * The container allows resources to be shared across multiple
+ * events with different root events. An example usage of this would be used
+ * for a server connection. We want all events initiated on the connection to
+ * share resource values, such as remote and local socket. However, each root
+ * event on the connection may be for a different trace.
+ */
+public interface EventResources {
+ /**
+ * Add a resource for the event. Resources are inherited by
+ * child events.
+ * @param key the key to identify the resource
+ * @param value the value which will be logged for the resource.
+ * This is converted to a string before logging.
+ * @return this
+ */
+ EventResources resource(String key, Object value);
+
+ /**
+ * Add a resource for the event using a supplier. The supplier is
+ * used in the case that generating the string from the object is
+ * expensive or we want to generate a custom string.
+ * @param key the key to identify the resource
+ * @param value a supplier which returns the value to be logged for
+ * this resource
+ * @see #resource(java.lang.String,java.lang.Object)
+ */
+ EventResources resource(String key, Supplier<String> value);
+}
diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java
new file mode 100644
index 0000000..b4f0a1d
--- /dev/null
+++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.structuredeventlog;
+
+/**
+ * Structured event logging interface
+ *
+ * Allows resources and attribute key value pairs to be attached to a logged event.
+ *
+ * Basic usage:
+ * <pre>
+ * StructuredEventLog logger = StructuredEventLog.newLogger();
+ * logger.newRootEvent()
+ * .resource("remote", remoteAddr)
+ * .resource("local", localAddr)
+ * .attr("path", request.getPath())
+ * .log(Events.READ_REQUEST);
+ * </pre>
+ */
+public interface StructuredEventLog {
+ /**
+ * Create a new root event. Root events occur in response to some external stimulus, such as
+ * a user request, a timer being triggered or a threshold being crossed.
+ *
+ * The level of the event is INFO by default.
+ *
+ * The root event will generate a new traceId, and will have a empty parent Id. If this
+ * information is provided, as can be the case with a user request, they can be set
+ * with Event#traceId(String) and Event#parentId(String).
+ */
+ Event newRootEvent();
+
+ /**
+ * Create an new event resources object, which can be used across multiple
+ * root events.
+ */
+ EventResources newEventResources();
+
+ /**
+ * Retrieves an event from the call stack. This can be used, along with Event#stash(),
+ * to bridge an event across an API without having to modify the API to pass the event
+ * object.
+ *
+ * For example, the child event, METHOD2, in the following example, will share the traceId
+ * with METHOD1, and METHOD1's id will be match the parentId of METHOD2.
+ *
+ * <pre>
+ * void method1() {
+ * Event e = logger.newRootEvent()
+ * .timed()
+ * .resource("foo", bar);
+ *
+ * e.stash();
+ * unmodifiableMethod();
+ * e.log(Events.METHOD1);
+ * }
+ *
+ * void unmodifiableMethod() {
+ * logger.unstash().newChildEvent().log(Events.METHOD2);
+ * }
+ * </pre>
+ *
+ * This should be used sparingly.
+ */
+ Event unstash();
+
+ /**
+ * Create a new logger object, from which root events can be created.
+ */
+ public static StructuredEventLog newLogger() {
+ return null;
+ }
+}