You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2024/07/26 18:05:33 UTC

(beam) branch revert-31825-conditional-string-sets created (now b25cbe22100)

This is an automated email from the ASF dual-hosted git repository.

yhu pushed a change to branch revert-31825-conditional-string-sets
in repository https://gitbox.apache.org/repos/asf/beam.git


      at b25cbe22100 Revert "Avoid publishing string set metrics on the Dataflow legacy runner. (#…"

This branch includes the following new commits:

     new b25cbe22100 Revert "Avoid publishing string set metrics on the Dataflow legacy runner. (#…"

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(beam) 01/01: Revert "Avoid publishing string set metrics on the Dataflow legacy runner. (#…"

Posted by yh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yhu pushed a commit to branch revert-31825-conditional-string-sets
in repository https://gitbox.apache.org/repos/asf/beam.git

commit b25cbe22100d2b41596ac4813f050d76e029e6ee
Author: Yi Hu <hu...@gmail.com>
AuthorDate: Fri Jul 26 14:05:27 2024 -0400

    Revert "Avoid publishing string set metrics on the Dataflow legacy runner. (#…"
    
    This reverts commit a22609494318e3125255105b6d2568bbba30a54f.
---
 .../dataflow/worker/BatchModeExecutionContext.java    | 19 ++++---------------
 .../worker/StreamingStepMetricsContainer.java         |  7 +------
 .../worker/StreamingStepMetricsContainerTest.java     |  2 --
 3 files changed, 5 insertions(+), 23 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java
index 41bbae7cfdb..aeef7784c2c 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java
@@ -19,7 +19,6 @@ package org.apache.beam.runners.dataflow.worker;
 
 import com.google.api.services.dataflow.model.CounterUpdate;
 import com.google.api.services.dataflow.model.SideInputInfo;
-import java.util.Collections;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import org.apache.beam.runners.core.InMemoryStateInternals;
@@ -78,9 +77,6 @@ public class BatchModeExecutionContext
   protected static final String BIGQUERY_READ_THROTTLE_TIME_NAMESPACE =
       "org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl$StorageClientImpl";
 
-  // TODO(BEAM-31814): Remove once Dataflow legacy runner supports this.
-  private final boolean populateStringSetMetrics;
-
   private BatchModeExecutionContext(
       CounterFactory counterFactory,
       Cache<?, WeightedValue<?>> dataCache,
@@ -88,8 +84,7 @@ public class BatchModeExecutionContext
       ReaderFactory readerFactory,
       PipelineOptions options,
       DataflowExecutionStateTracker executionStateTracker,
-      DataflowExecutionStateRegistry executionStateRegistry,
-      boolean populateStringSetMetrics) {
+      DataflowExecutionStateRegistry executionStateRegistry) {
     super(
         counterFactory,
         createMetricsContainerRegistry(),
@@ -102,7 +97,6 @@ public class BatchModeExecutionContext
     this.dataCache = dataCache;
     this.containerRegistry =
         (MetricsContainerRegistry<MetricsContainerImpl>) getMetricsContainerRegistry();
-    this.populateStringSetMetrics = populateStringSetMetrics;
   }
 
   private static MetricsContainerRegistry<MetricsContainerImpl> createMetricsContainerRegistry() {
@@ -138,8 +132,7 @@ public class BatchModeExecutionContext
             counterFactory,
             options,
             "test-work-item-id"),
-        stateRegistry,
-        true);
+        stateRegistry);
   }
 
   public static BatchModeExecutionContext forTesting(PipelineOptions options, String stageName) {
@@ -252,8 +245,7 @@ public class BatchModeExecutionContext
             counterFactory,
             options,
             workItemId),
-        executionStateRegistry,
-        false);
+        executionStateRegistry);
   }
 
   /** Create a new {@link StepContext}. */
@@ -523,10 +515,7 @@ public class BatchModeExecutionContext
                           update ->
                               MetricsToCounterUpdateConverter.fromDistribution(
                                   update.getKey(), true, update.getUpdate())),
-                  FluentIterable.from(
-                          populateStringSetMetrics
-                              ? updates.stringSetUpdates()
-                              : Collections.emptyList())
+                  FluentIterable.from(updates.stringSetUpdates())
                       .transform(
                           update ->
                               MetricsToCounterUpdateConverter.fromStringSet(
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java
index 04f983c4fa7..7cc0dc68f7e 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java
@@ -23,7 +23,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -89,9 +88,6 @@ public class StreamingStepMetricsContainer implements MetricsContainer {
 
   private final Clock clock;
 
-  // TODO(BEAM-31814): Remove once Dataflow legacy runner supports this.
-  @VisibleForTesting boolean populateStringSetUpdates = false;
-
   private StreamingStepMetricsContainer(String stepName) {
     this.stepName = stepName;
     this.perWorkerCountersByFirstStaleTime = new ConcurrentHashMap<>();
@@ -191,8 +187,7 @@ public class StreamingStepMetricsContainer implements MetricsContainer {
   public Iterable<CounterUpdate> extractUpdates() {
     return counterUpdates()
         .append(distributionUpdates())
-        .append(gaugeUpdates())
-        .append(populateStringSetUpdates ? stringSetUpdates() : Collections.emptyList());
+        .append(gaugeUpdates().append(stringSetUpdates()));
   }
 
   private FluentIterable<CounterUpdate> counterUpdates() {
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java
index d128255cd23..2d5a8d8266a 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java
@@ -292,7 +292,6 @@ public class StreamingStepMetricsContainerTest {
             .setCumulative(false)
             .setStringList(new StringList().setElements(Arrays.asList("ab", "cd", "ef", "gh")));
 
-    ((StreamingStepMetricsContainer) c1).populateStringSetUpdates = true;
     Iterable<CounterUpdate> updates = StreamingStepMetricsContainer.extractMetricUpdates(registry);
     assertThat(updates, containsInAnyOrder(name1Update));
 
@@ -315,7 +314,6 @@ public class StreamingStepMetricsContainerTest {
             .setCumulative(false)
             .setStringList(new StringList().setElements(Arrays.asList("ij", "kl", "mn")));
 
-    ((StreamingStepMetricsContainer) c2).populateStringSetUpdates = true;
     updates = StreamingStepMetricsContainer.extractMetricUpdates(registry);
     assertThat(updates, containsInAnyOrder(name1Update, name2Update));