You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/04/14 18:43:04 UTC

[1/2] beam git commit: ApexRunner: register standard IOs when deserializing pipeline options

Repository: beam
Updated Branches:
  refs/heads/master eea96ffa3 -> 735a82bda


ApexRunner: register standard IOs when deserializing pipeline options

This needs to be done once per JVM using the Job's pipeline options. The following
is a hack that is tolerant of multiple initialization.

Is there a better way to execute something like this exactly once?


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/dfbe9549
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/dfbe9549
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/dfbe9549

Branch: refs/heads/master
Commit: dfbe95493302555d969751232ec4c93a63b8c6de
Parents: eea96ff
Author: Dan Halperin <dh...@google.com>
Authored: Thu Apr 13 14:07:26 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 14 11:42:51 2017 -0700

----------------------------------------------------------------------
 .../translation/utils/SerializablePipelineOptions.java | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/dfbe9549/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
index d0dce2b..1a47ed5 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
@@ -18,20 +18,24 @@
 package org.apache.beam.runners.apex.translation.utils;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-
 import java.io.Externalizable;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
-
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.beam.runners.apex.ApexPipelineOptions;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.IOChannelUtils;
 
 /**
  * A wrapper to enable serialization of {@link PipelineOptions}.
  */
 public class SerializablePipelineOptions implements Externalizable {
 
+  /* Used to ensure we initialize file systems exactly once, because it's a slow operation. */
+  private static final AtomicBoolean FILE_SYSTEMS_INTIIALIZED = new AtomicBoolean(false);
+
   private transient ApexPipelineOptions pipelineOptions;
 
   public SerializablePipelineOptions(ApexPipelineOptions pipelineOptions) {
@@ -55,6 +59,11 @@ public class SerializablePipelineOptions implements Externalizable {
     String s = in.readUTF();
     this.pipelineOptions = new ObjectMapper().readValue(s, PipelineOptions.class)
         .as(ApexPipelineOptions.class);
+
+    if (FILE_SYSTEMS_INTIIALIZED.compareAndSet(false, true)) {
+      IOChannelUtils.registerIOFactoriesAllowOverride(pipelineOptions);
+      FileSystems.setDefaultConfigInWorkers(pipelineOptions);
+    }
   }
 
 }


[2/2] beam git commit: This closes #2528

Posted by dh...@apache.org.
This closes #2528


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/735a82bd
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/735a82bd
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/735a82bd

Branch: refs/heads/master
Commit: 735a82bda010ae9b67587215543b9dc1af447221
Parents: eea96ff dfbe954
Author: Dan Halperin <dh...@google.com>
Authored: Fri Apr 14 11:42:56 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 14 11:42:56 2017 -0700

----------------------------------------------------------------------
 .../translation/utils/SerializablePipelineOptions.java | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------