You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ud...@apache.org on 2023/06/14 18:43:37 UTC
[beam] branch master updated: Initial implementation of QuotaEvent (#27078)
This is an automated email from the ASF dual-hosted git repository.
udim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0822ab49697 Initial implementation of QuotaEvent (#27078)
0822ab49697 is described below
commit 0822ab49697d3cb0f3f0b069e64c9571b848b7e2
Author: Udi Meiri <ud...@users.noreply.github.com>
AuthorDate: Wed Jun 14 11:43:24 2023 -0700
Initial implementation of QuotaEvent (#27078)
* Initial implementation of QuotaEvent
* Fix style
* Fix typo
* Add benchmark
* Added clarification about close()
---
.../jmh/logging/BeamFnLoggingClientBenchmark.java | 18 ++++
.../fn/harness/logging/BeamFnLoggingClient.java | 4 +-
.../apache/beam/fn/harness/logging/QuotaEvent.java | 108 +++++++++++++++++++++
3 files changed, 129 insertions(+), 1 deletion(-)
diff --git a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java
index 3732bdfb189..d7c18059dc2 100644
--- a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java
+++ b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java
@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.beam.fn.harness.logging.BeamFnLoggingClient;
import org.apache.beam.fn.harness.logging.BeamFnLoggingMDC;
+import org.apache.beam.fn.harness.logging.QuotaEvent;
import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
@@ -206,6 +207,23 @@ public class BeamFnLoggingClientBenchmark {
}
}
+ @Benchmark
+ @Threads(16) // Use several threads since we expect contention during logging
+ public void testLoggingWithQuotaEvent(
+ ManyExpectedCallsLoggingClientAndService client, ManageExecutionState executionState)
+ throws Exception {
+ try (Closeable state =
+ executionState.executionStateTracker.enterState(executionState.simpleExecutionState)) {
+ try (AutoCloseable ac =
+ new QuotaEvent.Builder()
+ .withOperation("test")
+ .withFullResourceName("//test.googleapis.com/abc/123")
+ .create()) {
+ LOG.warn("log me");
+ }
+ }
+ }
+
@Benchmark
@Threads(16) // Use several threads since we expect contention during logging
public void testSkippedLogging(ZeroExpectedCallsLoggingClientAndService client) {
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
index f2ec67c2ac5..68ad6727c4d 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
@@ -126,7 +126,9 @@ public class BeamFnLoggingClient implements AutoCloseable {
logRecordHandler.setLevel(Level.ALL);
logRecordHandler.setFormatter(DEFAULT_FORMATTER);
logRecordHandler.executeOn(options.as(ExecutorOptions.class).getScheduledExecutorService());
- logRecordHandler.setLogMdc(options.as(SdkHarnessOptions.class).getLogMdc());
+ boolean logMdc = options.as(SdkHarnessOptions.class).getLogMdc();
+ logRecordHandler.setLogMdc(logMdc);
+ QuotaEvent.setEnabled(logMdc);
outboundObserver = (CallStreamObserver<BeamFnApi.LogEntry.List>) stub.logging(inboundObserver);
rootLogger.addHandler(logRecordHandler);
}
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/QuotaEvent.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/QuotaEvent.java
new file mode 100644
index 00000000000..2ebb8b9ee33
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/QuotaEvent.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.logging;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.slf4j.MDC;
+
+/**
+ * Adds MDC key:value pairs for a quota event. The contents of the MDC are added to each log entry.
+ *
+ * <p>Suggested usage: try (QuotaEventCloseable qec = new
+ * QuotaEvent.Builder().withOperation(op).withFullResourceName(name).create()) { LOG.info(...); }
+ *
+ * <p>MDC data is thread-local. After calling QuotaEventClosable.close, any overwritten
+ * quota_event.* keys will not be restored.
+ */
+public class QuotaEvent {
+ public static class QuotaEventCloseable implements AutoCloseable {
+ public QuotaEventCloseable(Map<String, String> data) {
+ this.data = data;
+ }
+
+ @Override
+ public void close() {
+ if (!enabled) {
+ return;
+ }
+ for (String key : data.keySet()) {
+ MDC.remove(key);
+ }
+ }
+
+ private final Map<String, String> data;
+ }
+
+ public static class Builder {
+ /** @param messageText Any additional description for the event. */
+ public Builder withMessageText(String messageText) {
+ data.put("quota_event.message_text", messageText);
+ return this;
+ }
+
+ /**
+ * @param fullResourceName The full path for the resource involved. For GCP, this would follow
+ * https://google.aip.dev/122#full-resource-names.
+ */
+ public Builder withFullResourceName(String fullResourceName) {
+ data.put("quota_event.full_resource_name", fullResourceName);
+ return this;
+ }
+
+ /**
+ * @param quotaName The full quota name. For GCP, this would look like:
+ * example.googleapis.com/quota/name
+ */
+ public Builder withQuotaName(String quotaName) {
+ data.put("quota_event.quota_name", quotaName);
+ return this;
+ }
+
+ /** @param operation Name of the operation that generated this quota event, in snake-case. */
+ public Builder withOperation(String operation) {
+ data.put("quota_event.operation", operation);
+ return this;
+ }
+
+ /**
+ * Puts the given values in the MDC as a quota event.
+ *
+ * @return An AutoCloseable that removes the added values from the MDC, or null if the feature
+ * is disabled.
+ */
+ public @Nullable QuotaEventCloseable create() {
+ if (!enabled) {
+ return null;
+ }
+
+ data.forEach(MDC::put);
+ return new QuotaEventCloseable(data);
+ }
+
+ private final Map<String, String> data = new HashMap<>();
+ }
+
+ /** Set whether this feature is enabled. Affects Builder.create(). */
+ public static void setEnabled(boolean enabled) {
+ QuotaEvent.enabled = enabled;
+ }
+
+ private static boolean enabled = false;
+}