You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ud...@apache.org on 2023/06/14 18:43:37 UTC

[beam] branch master updated: Initial implementation of QuotaEvent (#27078)

This is an automated email from the ASF dual-hosted git repository.

udim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 0822ab49697 Initial implementation of QuotaEvent (#27078)
0822ab49697 is described below

commit 0822ab49697d3cb0f3f0b069e64c9571b848b7e2
Author: Udi Meiri <ud...@users.noreply.github.com>
AuthorDate: Wed Jun 14 11:43:24 2023 -0700

    Initial implementation of QuotaEvent (#27078)
    
    * Initial implementation of QuotaEvent
    
    * Fix style
    
    * Fix typo
    
    * Add benchmark
    
    * Added clarification about close()
---
 .../jmh/logging/BeamFnLoggingClientBenchmark.java  |  18 ++++
 .../fn/harness/logging/BeamFnLoggingClient.java    |   4 +-
 .../apache/beam/fn/harness/logging/QuotaEvent.java | 108 +++++++++++++++++++++
 3 files changed, 129 insertions(+), 1 deletion(-)

diff --git a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java
index 3732bdfb189..d7c18059dc2 100644
--- a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java
+++ b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/logging/BeamFnLoggingClientBenchmark.java
@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.beam.fn.harness.logging.BeamFnLoggingClient;
 import org.apache.beam.fn.harness.logging.BeamFnLoggingMDC;
+import org.apache.beam.fn.harness.logging.QuotaEvent;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi;
 import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
 import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
@@ -206,6 +207,23 @@ public class BeamFnLoggingClientBenchmark {
     }
   }
 
+  @Benchmark
+  @Threads(16) // Use several threads since we expect contention during logging
+  public void testLoggingWithQuotaEvent(
+      ManyExpectedCallsLoggingClientAndService client, ManageExecutionState executionState)
+      throws Exception {
+    try (Closeable state =
+        executionState.executionStateTracker.enterState(executionState.simpleExecutionState)) {
+      try (AutoCloseable ac =
+          new QuotaEvent.Builder()
+              .withOperation("test")
+              .withFullResourceName("//test.googleapis.com/abc/123")
+              .create()) {
+        LOG.warn("log me");
+      }
+    }
+  }
+
   @Benchmark
   @Threads(16) // Use several threads since we expect contention during logging
   public void testSkippedLogging(ZeroExpectedCallsLoggingClientAndService client) {
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
index f2ec67c2ac5..68ad6727c4d 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
@@ -126,7 +126,9 @@ public class BeamFnLoggingClient implements AutoCloseable {
     logRecordHandler.setLevel(Level.ALL);
     logRecordHandler.setFormatter(DEFAULT_FORMATTER);
     logRecordHandler.executeOn(options.as(ExecutorOptions.class).getScheduledExecutorService());
-    logRecordHandler.setLogMdc(options.as(SdkHarnessOptions.class).getLogMdc());
+    boolean logMdc = options.as(SdkHarnessOptions.class).getLogMdc();
+    logRecordHandler.setLogMdc(logMdc);
+    QuotaEvent.setEnabled(logMdc);
     outboundObserver = (CallStreamObserver<BeamFnApi.LogEntry.List>) stub.logging(inboundObserver);
     rootLogger.addHandler(logRecordHandler);
   }
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/QuotaEvent.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/QuotaEvent.java
new file mode 100644
index 00000000000..2ebb8b9ee33
--- /dev/null
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/QuotaEvent.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.fn.harness.logging;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.slf4j.MDC;
+
+/**
+ * Adds MDC key:value pairs for a quota event. The contents of the MDC are added to each log entry.
+ *
+ * <p>Suggested usage: try (QuotaEventCloseable qec = new
+ * QuotaEvent.Builder().withOperation(op).withFullResourceName(name).create()) { LOG.info(...); }
+ *
+ * <p>MDC data is thread-local. After calling QuotaEventClosable.close, any overwritten
+ * quota_event.* keys will not be restored.
+ */
+public class QuotaEvent {
+  public static class QuotaEventCloseable implements AutoCloseable {
+    public QuotaEventCloseable(Map<String, String> data) {
+      this.data = data;
+    }
+
+    @Override
+    public void close() {
+      if (!enabled) {
+        return;
+      }
+      for (String key : data.keySet()) {
+        MDC.remove(key);
+      }
+    }
+
+    private final Map<String, String> data;
+  }
+
+  public static class Builder {
+    /** @param messageText Any additional description for the event. */
+    public Builder withMessageText(String messageText) {
+      data.put("quota_event.message_text", messageText);
+      return this;
+    }
+
+    /**
+     * @param fullResourceName The full path for the resource involved. For GCP, this would follow
+     *     https://google.aip.dev/122#full-resource-names.
+     */
+    public Builder withFullResourceName(String fullResourceName) {
+      data.put("quota_event.full_resource_name", fullResourceName);
+      return this;
+    }
+
+    /**
+     * @param quotaName The full quota name. For GCP, this would look like:
+     *     example.googleapis.com/quota/name
+     */
+    public Builder withQuotaName(String quotaName) {
+      data.put("quota_event.quota_name", quotaName);
+      return this;
+    }
+
+    /** @param operation Name of the operation that generated this quota event, in snake-case. */
+    public Builder withOperation(String operation) {
+      data.put("quota_event.operation", operation);
+      return this;
+    }
+
+    /**
+     * Puts the given values in the MDC as a quota event.
+     *
+     * @return An AutoCloseable that removes the added values from the MDC, or null if the feature
+     *     is disabled.
+     */
+    public @Nullable QuotaEventCloseable create() {
+      if (!enabled) {
+        return null;
+      }
+
+      data.forEach(MDC::put);
+      return new QuotaEventCloseable(data);
+    }
+
+    private final Map<String, String> data = new HashMap<>();
+  }
+
+  /** Set whether this feature is enabled. Affects Builder.create(). */
+  public static void setEnabled(boolean enabled) {
+    QuotaEvent.enabled = enabled;
+  }
+
+  private static boolean enabled = false;
+}