You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2021/04/01 17:38:44 UTC
[beam] branch master updated: Update V1Beta3 API and add
DebugOptions
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2354edd Update V1Beta3 API and add DebugOptions
new b43952c Merge pull request #14380 from [BEAM-10994] Java: Update V1Beta3 API and add DebugOptions
2354edd is described below
commit 2354eddcaf8659b521789b25878780f89c92956f
Author: Sam R <ro...@gmail.com>
AuthorDate: Tue Mar 30 21:41:07 2021 +0000
Update V1Beta3 API and add DebugOptions
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 6 ++++++
.../dataflow/DataflowPipelineTranslatorTest.java | 22 ++++++++++++++++++++++
3 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 5acc301..4aded65 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -512,7 +512,7 @@ class BeamModulePlugin implements Plugin<Project> {
google_api_services_bigquery : "com.google.apis:google-api-services-bigquery:v2-rev20210219-$google_clients_version",
google_api_services_clouddebugger : "com.google.apis:google-api-services-clouddebugger:v2-rev20200807-$google_clients_version",
google_api_services_cloudresourcemanager : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210222-$google_clients_version",
- google_api_services_dataflow : "com.google.apis:google-api-services-dataflow:v1b3-rev20210217-$google_clients_version",
+ google_api_services_dataflow : "com.google.apis:google-api-services-dataflow:v1b3-rev20210301-$google_clients_version",
google_api_services_healthcare : "com.google.apis:google-api-services-healthcare:v1beta1-rev20210217-$google_clients_version",
google_api_services_pubsub : "com.google.apis:google-api-services-pubsub:v1-rev20210208-$google_clients_version",
google_api_services_storage : "com.google.apis:google-api-services-storage:v1-rev20210127-$google_clients_version",
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 0fd8d80..23147e7 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -36,6 +36,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.services.dataflow.model.AutoscalingSettings;
import com.google.api.services.dataflow.model.DataflowPackage;
+import com.google.api.services.dataflow.model.DebugOptions;
import com.google.api.services.dataflow.model.Disk;
import com.google.api.services.dataflow.model.Environment;
import com.google.api.services.dataflow.model.Job;
@@ -435,6 +436,11 @@ public class DataflowPipelineTranslator {
if (options.getDataflowKmsKey() != null) {
environment.setServiceKmsKeyName(options.getDataflowKmsKey());
}
+ if (options.isHotKeyLoggingEnabled()) {
+ DebugOptions debugOptions = new DebugOptions();
+ debugOptions.setEnableHotKeyLogging(true);
+ environment.setDebugOptions(debugOptions);
+ }
pipeline.traverseTopologically(this);
return job;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 8015ab3..ca306cf 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -1317,6 +1317,28 @@ public class DataflowPipelineTranslatorTest implements Serializable {
assertEquals(serviceOptions, job.getEnvironment().getServiceOptions());
}
+ @Test
+ public void testHotKeyLoggingEnabledOption() throws IOException {
+ DataflowPipelineOptions options = buildPipelineOptions();
+ options.setHotKeyLoggingEnabled(true);
+
+ Pipeline p = buildPipeline(options);
+ p.traverseTopologically(new RecordingPipelineVisitor());
+ SdkComponents sdkComponents = createSdkComponents(options);
+ RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p, sdkComponents, true);
+ Job job =
+ DataflowPipelineTranslator.fromOptions(options)
+ .translate(
+ p,
+ pipelineProto,
+ sdkComponents,
+ DataflowRunner.fromOptions(options),
+ Collections.emptyList())
+ .getJob();
+
+ assertTrue(job.getEnvironment().getDebugOptions().getEnableHotKeyLogging());
+ }
+
private static void assertAllStepOutputsHaveUniqueIds(Job job) throws Exception {
List<String> outputIds = new ArrayList<>();
for (Step step : job.getSteps()) {