You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/05/08 04:58:46 UTC

[1/3] beam git commit: [BEAM-831] ParDo Fusion of Apex Runner

Repository: beam
Updated Branches:
  refs/heads/master 53dd0a529 -> 1a4ca9acd


[BEAM-831] ParDo Fusion of Apex Runner


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/55a7cd4f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/55a7cd4f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/55a7cd4f

Branch: refs/heads/master
Commit: 55a7cd4f97f2e3fee709f034a0748284c9030df7
Parents: 019d300
Author: chinmaykolhatkar <ch...@apache.org>
Authored: Wed Mar 1 16:59:46 2017 +0530
Committer: Thomas Weise <th...@apache.org>
Committed: Sat May 6 19:52:13 2017 -0700

----------------------------------------------------------------------
 .../beam/runners/apex/ApexPipelineOptions.java  |   5 +
 .../apex/translation/TranslationContext.java    | 103 +++++++++++++++++--
 2 files changed, 97 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/55a7cd4f/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
index f37e874..92f6e8f 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
@@ -62,4 +62,9 @@ public interface ApexPipelineOptions extends PipelineOptions, java.io.Serializab
   @Default.String("classpath:/beam-runners-apex.properties")
   String getConfigFile();
 
+  @Description("configure whether to perform ParDo fusion")
+  void setParDoFusionEnabled(boolean enabled);
+
+  @Default.Boolean(true)
+  boolean isParDoFusionEnabled();
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/55a7cd4f/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index 9c20449..1224e25 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -37,6 +37,8 @@ import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -54,7 +56,8 @@ class TranslationContext {
 
   private final ApexPipelineOptions pipelineOptions;
   private AppliedPTransform<?, ?, ?> currentTransform;
-  private final Map<PCollection, Pair<OutputPort<?>, List<InputPort<?>>>> streams = new HashMap<>();
+  private final Map<PCollection, Pair<OutputPortInfo, List<InputPortInfo>>> streams =
+          new HashMap<>();
   private final Map<String, Operator> operators = new HashMap<>();
   private final Map<PCollectionView<?>, PInput> viewInputs = new HashMap<>();
   private Map<PInput, PInput> aliasCollections = new HashMap<>();
@@ -122,8 +125,10 @@ class TranslationContext {
         addOperator(operator, portEntry.getValue(), portEntry.getKey());
         first = false;
       } else {
-        this.streams.put(portEntry.getKey(), (Pair) new ImmutablePair<>(portEntry.getValue(),
-            new ArrayList<>()));
+        this.streams.put(portEntry.getKey(),
+                         (Pair) new ImmutablePair<>(new OutputPortInfo(portEntry.getValue(),
+                                                                       getCurrentTransform()),
+                                                    new ArrayList<>()));
       }
     }
   }
@@ -142,16 +147,19 @@ class TranslationContext {
       name = getCurrentTransform().getFullName() + i;
     }
     this.operators.put(name, operator);
-    this.streams.put(output, (Pair) new ImmutablePair<>(port, new ArrayList<>()));
+    this.streams.put(output, (Pair) new ImmutablePair<>(
+            new OutputPortInfo(port, getCurrentTransform()),
+            new ArrayList<>()));
   }
 
   public void addStream(PInput input, InputPort inputPort) {
     while (aliasCollections.containsKey(input)) {
       input = aliasCollections.get(input);
     }
-    Pair<OutputPort<?>, List<InputPort<?>>> stream = this.streams.get(input);
-    checkArgument(stream != null, "no upstream operator defined for %s", input);
-    stream.getRight().add(inputPort);
+
+    Pair<OutputPortInfo, List<InputPortInfo>> stream = this.streams.get(input);
+    checkArgument(stream != null, "no upstream operator defined for " + input);
+    stream.getRight().add(new InputPortInfo(inputPort, getCurrentTransform()));
   }
 
   /**
@@ -168,13 +176,23 @@ class TranslationContext {
     for (Map.Entry<String, Operator> nameAndOperator : this.operators.entrySet()) {
       dag.addOperator(nameAndOperator.getKey(), nameAndOperator.getValue());
     }
+
     int streamIndex = 0;
-    for (Map.Entry<PCollection, Pair<OutputPort<?>, List<InputPort<?>>>> streamEntry : this.
+    for (Map.Entry<PCollection, Pair<OutputPortInfo, List<InputPortInfo>>> streamEntry : this.
         streams.entrySet()) {
-      List<InputPort<?>> sinksList = streamEntry.getValue().getRight();
-      InputPort[] sinks = sinksList.toArray(new InputPort[sinksList.size()]);
+      List<InputPortInfo> destInfo = streamEntry.getValue().getRight();
+      InputPort[] sinks = new InputPort[destInfo.size()];
+      for (int i = 0; i < sinks.length; i++) {
+        sinks[i] = destInfo.get(i).port;
+      }
+
       if (sinks.length > 0) {
-        dag.addStream("stream" + streamIndex++, streamEntry.getValue().getLeft(), sinks);
+        DAG.StreamMeta streamMeta = dag.addStream("stream" + streamIndex++,
+                                                  streamEntry.getValue().getLeft().port, sinks);
+        if (pipelineOptions.isParDoFusionEnabled()) {
+          optimizeStreams(streamMeta, streamEntry);
+        }
+
         for (InputPort port : sinks) {
           PCollection pc = streamEntry.getKey();
           Coder coder = pc.getCoder();
@@ -191,6 +209,49 @@ class TranslationContext {
     }
   }
 
+  private void optimizeStreams(DAG.StreamMeta streamMeta,
+                               Map.Entry<PCollection,
+                                         Pair<OutputPortInfo, List<InputPortInfo>>> streamEntry) {
+    DAG.Locality loc = null;
+
+    List<InputPortInfo> sinks = streamEntry.getValue().getRight();
+    OutputPortInfo source = streamEntry.getValue().getLeft();
+    PTransform sourceTransform = source.transform.getTransform();
+    if (sourceTransform instanceof ParDo.Bound
+        || sourceTransform instanceof ParDo.BoundMulti) {
+      // Source is ParDo.. Check sink(s)
+      for (InputPortInfo sink : sinks) {
+        PTransform transform = sink.transform.getTransform();
+        if (transform instanceof ParDo.Bound) {
+          ParDo.Bound t = (ParDo.Bound) transform;
+          if (t.getSideInputs().size() > 0) {
+            loc = DAG.Locality.CONTAINER_LOCAL;
+            break;
+          } else {
+            loc = DAG.Locality.THREAD_LOCAL;
+          }
+        } else if (transform instanceof ParDo.BoundMulti) {
+          ParDo.BoundMulti t = (ParDo.BoundMulti) transform;
+          if (t.getSideInputs().size() > 0) {
+            loc = DAG.Locality.CONTAINER_LOCAL;
+            break;
+          } else {
+            loc = DAG.Locality.THREAD_LOCAL;
+          }
+        } else {
+          // Sink is not ParDo.. set null locality.
+          loc = null;
+          break;
+        }
+      }
+    } else {
+      // Source is not ParDo... set null locality
+      loc = null;
+    }
+
+    streamMeta.setLocality(loc);
+  }
+
   /**
    * Return the state backend for the pipeline translation.
    * @return
@@ -198,4 +259,24 @@ class TranslationContext {
   public ApexStateBackend getStateBackend() {
     return new ApexStateInternals.ApexStateBackend();
   }
+
+  static class InputPortInfo {
+    InputPort port;
+    AppliedPTransform transform;
+
+    public InputPortInfo(InputPort port, AppliedPTransform transform) {
+      this.port = port;
+      this.transform = transform;
+    }
+  }
+
+  static class OutputPortInfo {
+    OutputPort port;
+    AppliedPTransform transform;
+
+    public OutputPortInfo(OutputPort port, AppliedPTransform transform) {
+      this.port = port;
+      this.transform = transform;
+    }
+  }
 }


[2/3] beam git commit: [BEAM-831] Fix chaining, add test. closes #2216

Posted by ke...@apache.org.
[BEAM-831] Fix chaining, add test.
closes #2216


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b64c8ce7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b64c8ce7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b64c8ce7

Branch: refs/heads/master
Commit: b64c8ce7dc55426e0c4110a64ce5ec8e83151ab0
Parents: 55a7cd4
Author: Thomas Weise <th...@apache.org>
Authored: Fri May 5 06:45:34 2017 -0700
Committer: Thomas Weise <th...@apache.org>
Committed: Sat May 6 20:27:36 2017 -0700

----------------------------------------------------------------------
 .../apache/beam/runners/apex/ApexRunner.java    | 11 +++--
 .../beam/runners/apex/TestApexRunner.java       |  8 ++++
 .../apex/translation/TranslationContext.java    | 26 ++++-------
 .../beam/runners/apex/ApexRunnerTest.java       | 47 ++++++++++++++++----
 .../FlattenPCollectionTranslatorTest.java       | 13 ++----
 5 files changed, 67 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/b64c8ce7/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index a50e10e..366308e 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -70,11 +70,11 @@ import org.apache.hadoop.conf.Configuration;
  * pipeline to an Apex DAG and executes it on an Apex cluster.
  *
  */
-@SuppressWarnings({"rawtypes", "unchecked"})
 public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
 
   private final ApexPipelineOptions options;
   public static final String CLASSPATH_SCHEME = "classpath";
+  protected boolean translateOnly = false;
 
   /**
    * TODO: this isn't thread safe and may cause issues when tests run in parallel
@@ -93,6 +93,7 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
     return new ApexRunner(apexPipelineOptions);
   }
 
+  @SuppressWarnings({"rawtypes"})
   private List<PTransformOverride> getOverrides() {
     return ImmutableList.<PTransformOverride>builder()
         .add(
@@ -156,7 +157,7 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
     }
 
     if (options.isEmbeddedExecution()) {
-      Launcher<AppHandle> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
+      EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
       Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
       launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
       if (options.isEmbeddedExecutionDebugMode()) {
@@ -166,11 +167,15 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
       Configuration conf = new Configuration(false);
       ApexYarnLauncher.addProperties(conf, configProperties);
       try {
+        if (translateOnly) {
+          launcher.prepareDAG(apexApp, conf);
+          return new ApexRunnerResult(launcher.getDAG(), null);
+        }
         ApexRunner.ASSERTION_ERROR.set(null);
         AppHandle apexAppResult = launcher.launchApp(apexApp, conf, launchAttributes);
         return new ApexRunnerResult(apexDAG.get(), apexAppResult);
       } catch (Exception e) {
-        Throwables.propagateIfPossible(e);
+        Throwables.throwIfUnchecked(e);
         throw new RuntimeException(e);
       }
     } else {

http://git-wip-us.apache.org/repos/asf/beam/blob/b64c8ce7/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
index e068db0..b68d3da 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.runners.apex;
 
+import com.datatorrent.api.DAG;
 import java.io.IOException;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineRunner;
@@ -44,6 +45,13 @@ public class TestApexRunner extends PipelineRunner<ApexRunnerResult> {
     return new TestApexRunner(apexOptions);
   }
 
+  public static DAG translate(Pipeline pipeline, ApexPipelineOptions options) {
+    ApexRunner delegate = new ApexRunner(options);
+    delegate.translateOnly = true;
+    DAG dag = delegate.run(pipeline).getApexDAG();
+    return dag;
+  }
+
   @Override
   public ApexRunnerResult run(Pipeline pipeline) {
     ApexRunnerResult result = delegate.run(pipeline);

http://git-wip-us.apache.org/repos/asf/beam/blob/b64c8ce7/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index 1224e25..a5e3028 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -39,6 +39,7 @@ import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -217,36 +218,27 @@ class TranslationContext {
     List<InputPortInfo> sinks = streamEntry.getValue().getRight();
     OutputPortInfo source = streamEntry.getValue().getLeft();
     PTransform sourceTransform = source.transform.getTransform();
-    if (sourceTransform instanceof ParDo.Bound
-        || sourceTransform instanceof ParDo.BoundMulti) {
-      // Source is ParDo.. Check sink(s)
+    if (sourceTransform instanceof ParDo.MultiOutput
+        || sourceTransform instanceof Window.Assign) {
+      // source qualifies for chaining, check sink(s)
       for (InputPortInfo sink : sinks) {
         PTransform transform = sink.transform.getTransform();
-        if (transform instanceof ParDo.Bound) {
-          ParDo.Bound t = (ParDo.Bound) transform;
-          if (t.getSideInputs().size() > 0) {
-            loc = DAG.Locality.CONTAINER_LOCAL;
-            break;
-          } else {
-            loc = DAG.Locality.THREAD_LOCAL;
-          }
-        } else if (transform instanceof ParDo.BoundMulti) {
-          ParDo.BoundMulti t = (ParDo.BoundMulti) transform;
+        if (transform instanceof ParDo.MultiOutput) {
+          ParDo.MultiOutput t = (ParDo.MultiOutput) transform;
           if (t.getSideInputs().size() > 0) {
             loc = DAG.Locality.CONTAINER_LOCAL;
             break;
           } else {
             loc = DAG.Locality.THREAD_LOCAL;
           }
+        } else if (transform instanceof Window.Assign) {
+          loc = DAG.Locality.THREAD_LOCAL;
         } else {
-          // Sink is not ParDo.. set null locality.
+          // cannot chain, if there is any other sink
           loc = null;
           break;
         }
       }
-    } else {
-      // Source is not ParDo... set null locality
-      loc = null;
     }
 
     streamMeta.setLocality(loc);

http://git-wip-us.apache.org/repos/asf/beam/blob/b64c8ce7/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
index e9e9a5b..c5521d1 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
@@ -18,15 +18,23 @@
 package org.apache.beam.runners.apex;
 
 import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
 import com.datatorrent.api.DAG.OperatorMeta;
 import com.datatorrent.stram.engine.OperatorContext;
+import com.google.common.collect.Sets;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.util.Properties;
+import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.PCollection;
+import org.hamcrest.Matchers;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -41,16 +49,13 @@ public class ApexRunnerTest {
     String operName = "testProperties";
     ApexPipelineOptions options = PipelineOptionsFactory.create()
         .as(ApexPipelineOptions.class);
-    options.setRunner(ApexRunner.class);
 
     // default configuration from class path
-    Pipeline p = Pipeline.create(options);
+    Pipeline p = Pipeline.create();
     Create.Values<Void> empty = Create.empty(VoidCoder.of());
     p.apply(operName, empty);
-    ApexRunnerResult result = (ApexRunnerResult) p.run();
-    result.cancel();
 
-    DAG dag = result.getApexDAG();
+    DAG dag = TestApexRunner.translate(p, options);
     OperatorMeta t1Meta = dag.getOperatorMeta(operName);
     Assert.assertNotNull(t1Meta);
     Assert.assertEquals(new Integer(32), t1Meta.getValue(OperatorContext.MEMORY_MB));
@@ -63,14 +68,40 @@ public class ApexRunnerTest {
       props.store(fos, "");
     }
     options.setConfigFile(tmp.getAbsolutePath());
-    result = (ApexRunnerResult) p.run();
-    result.cancel();
+    dag = TestApexRunner.translate(p, options);
     tmp.delete();
-    dag = result.getApexDAG();
+
     t1Meta = dag.getOperatorMeta(operName);
     Assert.assertNotNull(t1Meta);
     Assert.assertEquals(new Integer(64), t1Meta.getValue(OperatorContext.MEMORY_MB));
 
   }
 
+  @Test
+  public void testParDoChaining() throws Exception {
+    Pipeline p = Pipeline.create();
+    long numElements = 1000;
+    PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements));
+    PAssert.thatSingleton(input.apply("Count", Count.<Long>globally())).isEqualTo(numElements);
+
+    ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
+    DAG dag = TestApexRunner.translate(p, options);
+
+    String[] expectedThreadLocal = { "/CreateActual/FilterActuals/Window.Assign" };
+    Set<String> actualThreadLocal = Sets.newHashSet();
+    for (DAG.StreamMeta sm : dag.getAllStreamsMeta()) {
+      DAG.OutputPortMeta opm = sm.getSource();
+      if (sm.getLocality() == Locality.THREAD_LOCAL) {
+         String name = opm.getOperatorMeta().getName();
+         String prefix = "PAssert$";
+         if (name.startsWith(prefix)) {
+           // remove indeterministic prefix
+           name = name.substring(prefix.length() + 1);
+         }
+         actualThreadLocal.add(name);
+      }
+    }
+    Assert.assertThat(actualThreadLocal, Matchers.hasItems(expectedThreadLocal));
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/b64c8ce7/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
index 64ca0ee..929778a 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
@@ -26,12 +26,10 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import org.apache.apex.api.EmbeddedAppLauncher;
-import org.apache.apex.api.Launcher;
-import org.apache.apex.api.Launcher.LaunchMode;
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.runners.apex.ApexRunner;
 import org.apache.beam.runners.apex.ApexRunnerResult;
+import org.apache.beam.runners.apex.TestApexRunner;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -100,16 +98,11 @@ public class FlattenPCollectionTranslatorTest {
   @Test
   public void testFlattenSingleCollection() {
     ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
-    options.setRunner(ApexRunner.class);
-    ApexPipelineTranslator translator = new ApexPipelineTranslator(options);
-    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
-    DAG dag = launcher.getDAG();
-
-    Pipeline p = Pipeline.create(options);
+    Pipeline p = Pipeline.create();
     PCollection<String> single = p.apply(Create.of(Collections.singletonList("1")));
     PCollectionList.of(single).apply(Flatten.<String>pCollections())
       .apply(ParDo.of(new EmbeddedCollector()));
-    translator.translate(p, dag);
+    DAG dag = TestApexRunner.translate(p, options);
     Assert.assertNotNull(
         dag.getOperatorMeta("ParDo(EmbeddedCollector)/ParMultiDo(EmbeddedCollector)"));
   }


[3/3] beam git commit: This closes #2927: ParDo Chaining in Apex Runner

Posted by ke...@apache.org.
This closes #2927: ParDo Chaining in Apex Runner

  [BEAM-831] Fix chaining, add test. closes #2216
  [BEAM-831] ParDo Fusion of Apex Runner


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1a4ca9ac
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1a4ca9ac
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1a4ca9ac

Branch: refs/heads/master
Commit: 1a4ca9acd53861c33279091ee08827775cd12348
Parents: 53dd0a5 b64c8ce
Author: Kenneth Knowles <kl...@google.com>
Authored: Sun May 7 21:57:27 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Sun May 7 21:57:27 2017 -0700

----------------------------------------------------------------------
 .../beam/runners/apex/ApexPipelineOptions.java  |  5 ++
 .../apache/beam/runners/apex/ApexRunner.java    | 11 ++-
 .../beam/runners/apex/TestApexRunner.java       |  8 ++
 .../apex/translation/TranslationContext.java    | 95 +++++++++++++++++---
 .../beam/runners/apex/ApexRunnerTest.java       | 47 ++++++++--
 .../FlattenPCollectionTranslatorTest.java       | 13 +--
 6 files changed, 147 insertions(+), 32 deletions(-)
----------------------------------------------------------------------