You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2016/04/14 06:48:10 UTC

[23/74] [partial] incubator-beam git commit: Rename com/google/cloud/dataflow->org/apache/beam

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java
new file mode 100644
index 0000000..d2c08c8
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.dataflow.sdk.util;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.api.client.googleapis.json.GoogleJsonError.ErrorInfo;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.json.GenericJson;
+import com.google.api.client.json.Json;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.testing.http.HttpTesting;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import com.google.api.services.dataflow.model.DataflowPackage;
+import com.google.cloud.dataflow.sdk.options.GcsOptions;
+import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory;
+import com.google.cloud.dataflow.sdk.testing.ExpectedLogs;
+import com.google.cloud.dataflow.sdk.testing.FastNanoClockAndSleeper;
+import com.google.cloud.dataflow.sdk.util.PackageUtil.PackageAttributes;
+import com.google.cloud.dataflow.sdk.util.gcsfs.GcsPath;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import com.google.common.io.LineReader;
+
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.channels.Pipe;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/** Tests for PackageUtil. */
+@RunWith(JUnit4.class)
+public class PackageUtilTest {
+  @Rule public ExpectedLogs logged = ExpectedLogs.none(PackageUtil.class);
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+  @Rule
+  public FastNanoClockAndSleeper fastNanoClockAndSleeper = new FastNanoClockAndSleeper();
+
+  @Mock
+  GcsUtil mockGcsUtil;
+
+  // 128 bits, base64 encoded is 171 bits, rounds to 22 bytes
+  private static final String HASH_PATTERN = "[a-zA-Z0-9+-]{22}";
+
+  // Hamcrest matcher to assert a string matches a pattern
+  private static class RegexMatcher extends BaseMatcher<String> {
+    private final Pattern pattern;
+
+    public RegexMatcher(String regex) {
+      this.pattern = Pattern.compile(regex);
+    }
+
+    @Override
+    public boolean matches(Object o) {
+      if (!(o instanceof String)) {
+        return false;
+      }
+      return pattern.matcher((String) o).matches();
+    }
+
+    @Override
+    public void describeTo(Description description) {
+      description.appendText(String.format("matches regular expression %s", pattern));
+    }
+
+    public static RegexMatcher matches(String regex) {
+      return new RegexMatcher(regex);
+    }
+  }
+
+  @Before
+  public void setUp() {
+    MockitoAnnotations.initMocks(this);
+
+    GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
+    pipelineOptions.setGcsUtil(mockGcsUtil);
+
+    IOChannelUtils.registerStandardIOFactories(pipelineOptions);
+  }
+
+  private File makeFileWithContents(String name, String contents) throws Exception {
+    File tmpFile = tmpFolder.newFile(name);
+    Files.write(contents, tmpFile, StandardCharsets.UTF_8);
+    tmpFile.setLastModified(0);  // required for determinism with directories
+    return tmpFile;
+  }
+
+  static final String STAGING_PATH = GcsPath.fromComponents("somebucket", "base/path").toString();
+  private static PackageAttributes makePackageAttributes(File file, String overridePackageName) {
+    return PackageUtil.createPackageAttributes(file, STAGING_PATH, overridePackageName);
+  }
+
+  @Test
+  public void testFileWithExtensionPackageNamingAndSize() throws Exception {
+    String contents = "This is a test!";
+    File tmpFile = makeFileWithContents("file.txt", contents);
+    PackageAttributes attr = makePackageAttributes(tmpFile, null);
+    DataflowPackage target = attr.getDataflowPackage();
+
+    assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt"));
+    assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
+    assertThat(attr.getSize(), equalTo((long) contents.length()));
+  }
+
+  @Test
+  public void testPackageNamingWithFileNoExtension() throws Exception {
+    File tmpFile = makeFileWithContents("file", "This is a test!");
+    DataflowPackage target = makePackageAttributes(tmpFile, null).getDataflowPackage();
+
+    assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN));
+    assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
+  }
+
+  @Test
+  public void testPackageNamingWithDirectory() throws Exception {
+    File tmpDirectory = tmpFolder.newFolder("folder");
+    DataflowPackage target = makePackageAttributes(tmpDirectory, null).getDataflowPackage();
+
+    assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar"));
+    assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
+  }
+
+  @Test
+  public void testPackageNamingWithFilesHavingSameContentsAndSameNames() throws Exception {
+    File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
+    makeFileWithContents("folder1/folderA/sameName", "This is a test!");
+    DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
+
+    File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
+    makeFileWithContents("folder2/folderA/sameName", "This is a test!");
+    DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
+
+    assertEquals(target1.getName(), target2.getName());
+    assertEquals(target1.getLocation(), target2.getLocation());
+  }
+
+  @Test
+  public void testPackageNamingWithFilesHavingSameContentsButDifferentNames() throws Exception {
+    File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
+    makeFileWithContents("folder1/folderA/uniqueName1", "This is a test!");
+    DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
+
+    File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
+    makeFileWithContents("folder2/folderA/uniqueName2", "This is a test!");
+    DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
+
+    assertNotEquals(target1.getName(), target2.getName());
+    assertNotEquals(target1.getLocation(), target2.getLocation());
+  }
+
+  @Test
+  public void testPackageNamingWithDirectoriesHavingSameContentsButDifferentNames()
+      throws Exception {
+    File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
+    tmpFolder.newFolder("folder1", "folderA", "uniqueName1");
+    DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
+
+    File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
+    tmpFolder.newFolder("folder2", "folderA", "uniqueName2");
+    DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
+
+    assertNotEquals(target1.getName(), target2.getName());
+    assertNotEquals(target1.getLocation(), target2.getLocation());
+  }
+
+  @Test
+  public void testPackageUploadWithLargeClasspathLogsWarning() throws Exception {
+    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
+    // all files will be present and cached so no upload needed.
+    when(mockGcsUtil.fileSize(any(GcsPath.class))).thenReturn(tmpFile.length());
+
+    List<String> classpathElements = Lists.newLinkedList();
+    for (int i = 0; i < 1005; ++i) {
+      String eltName = "element" + i;
+      classpathElements.add(eltName + '=' + tmpFile.getAbsolutePath());
+    }
+
+    PackageUtil.stageClasspathElements(classpathElements, STAGING_PATH);
+
+    logged.verifyWarn("Your classpath contains 1005 elements, which Google Cloud Dataflow");
+  }
+
+  @Test
+  public void testPackageUploadWithFileSucceeds() throws Exception {
+    Pipe pipe = Pipe.open();
+    String contents = "This is a test!";
+    File tmpFile = makeFileWithContents("file.txt", contents);
+    when(mockGcsUtil.fileSize(any(GcsPath.class)))
+        .thenThrow(new FileNotFoundException("some/path"));
+    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
+
+    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
+        ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH);
+    DataflowPackage target = Iterables.getOnlyElement(targets);
+
+    verify(mockGcsUtil).fileSize(any(GcsPath.class));
+    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
+    verifyNoMoreInteractions(mockGcsUtil);
+
+    assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt"));
+    assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
+    assertThat(new LineReader(Channels.newReader(pipe.source(), "UTF-8")).readLine(),
+        equalTo(contents));
+  }
+
+  @Test
+  public void testPackageUploadWithDirectorySucceeds() throws Exception {
+    Pipe pipe = Pipe.open();
+    File tmpDirectory = tmpFolder.newFolder("folder");
+    tmpFolder.newFolder("folder", "empty_directory");
+    tmpFolder.newFolder("folder", "directory");
+    makeFileWithContents("folder/file.txt", "This is a test!");
+    makeFileWithContents("folder/directory/file.txt", "This is also a test!");
+
+    when(mockGcsUtil.fileSize(any(GcsPath.class)))
+        .thenThrow(new FileNotFoundException("some/path"));
+    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
+
+    PackageUtil.stageClasspathElements(
+        ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH);
+
+    verify(mockGcsUtil).fileSize(any(GcsPath.class));
+    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
+    verifyNoMoreInteractions(mockGcsUtil);
+
+    ZipInputStream inputStream = new ZipInputStream(Channels.newInputStream(pipe.source()));
+    List<String> zipEntryNames = new ArrayList<>();
+    for (ZipEntry entry = inputStream.getNextEntry(); entry != null;
+        entry = inputStream.getNextEntry()) {
+      zipEntryNames.add(entry.getName());
+    }
+
+    assertThat(zipEntryNames,
+        containsInAnyOrder("directory/file.txt", "empty_directory/", "file.txt"));
+  }
+
+  @Test
+  public void testPackageUploadWithEmptyDirectorySucceeds() throws Exception {
+    Pipe pipe = Pipe.open();
+    File tmpDirectory = tmpFolder.newFolder("folder");
+
+    when(mockGcsUtil.fileSize(any(GcsPath.class)))
+        .thenThrow(new FileNotFoundException("some/path"));
+    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
+
+    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
+        ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH);
+    DataflowPackage target = Iterables.getOnlyElement(targets);
+
+    verify(mockGcsUtil).fileSize(any(GcsPath.class));
+    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
+    verifyNoMoreInteractions(mockGcsUtil);
+
+    assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar"));
+    assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
+    assertNull(new ZipInputStream(Channels.newInputStream(pipe.source())).getNextEntry());
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testPackageUploadFailsWhenIOExceptionThrown() throws Exception {
+    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
+    when(mockGcsUtil.fileSize(any(GcsPath.class)))
+        .thenThrow(new FileNotFoundException("some/path"));
+    when(mockGcsUtil.create(any(GcsPath.class), anyString()))
+        .thenThrow(new IOException("Fake Exception: Upload error"));
+
+    try {
+      PackageUtil.stageClasspathElements(
+          ImmutableList.of(tmpFile.getAbsolutePath()),
+          STAGING_PATH, fastNanoClockAndSleeper);
+    } finally {
+      verify(mockGcsUtil).fileSize(any(GcsPath.class));
+      verify(mockGcsUtil, times(5)).create(any(GcsPath.class), anyString());
+      verifyNoMoreInteractions(mockGcsUtil);
+    }
+  }
+
+  @Test
+  public void testPackageUploadFailsWithPermissionsErrorGivesDetailedMessage() throws Exception {
+    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
+    when(mockGcsUtil.fileSize(any(GcsPath.class)))
+        .thenThrow(new FileNotFoundException("some/path"));
+    when(mockGcsUtil.create(any(GcsPath.class), anyString()))
+        .thenThrow(new IOException("Failed to write to GCS path " + STAGING_PATH,
+            googleJsonResponseException(
+                HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Permission denied", "Test message")));
+
+    try {
+      PackageUtil.stageClasspathElements(
+          ImmutableList.of(tmpFile.getAbsolutePath()),
+          STAGING_PATH, fastNanoClockAndSleeper);
+      fail("Expected RuntimeException");
+    } catch (RuntimeException e) {
+      assertTrue("Expected IOException containing detailed message.",
+          e.getCause() instanceof IOException);
+      assertThat(e.getCause().getMessage(),
+          Matchers.allOf(
+              Matchers.containsString("Uploaded failed due to permissions error"),
+              Matchers.containsString(
+                  "Stale credentials can be resolved by executing 'gcloud auth login'")));
+    } finally {
+      verify(mockGcsUtil).fileSize(any(GcsPath.class));
+      verify(mockGcsUtil).create(any(GcsPath.class), anyString());
+      verifyNoMoreInteractions(mockGcsUtil);
+    }
+  }
+
+  @Test
+  public void testPackageUploadEventuallySucceeds() throws Exception {
+    Pipe pipe = Pipe.open();
+    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
+    when(mockGcsUtil.fileSize(any(GcsPath.class)))
+        .thenThrow(new FileNotFoundException("some/path"));
+    when(mockGcsUtil.create(any(GcsPath.class), anyString()))
+        .thenThrow(new IOException("Fake Exception: 410 Gone")) // First attempt fails
+        .thenReturn(pipe.sink());                               // second attempt succeeds
+
+    try {
+      PackageUtil.stageClasspathElements(
+                                              ImmutableList.of(tmpFile.getAbsolutePath()),
+                                              STAGING_PATH,
+                                              fastNanoClockAndSleeper);
+    } finally {
+      verify(mockGcsUtil).fileSize(any(GcsPath.class));
+      verify(mockGcsUtil, times(2)).create(any(GcsPath.class), anyString());
+      verifyNoMoreInteractions(mockGcsUtil);
+    }
+  }
+
+  @Test
+  public void testPackageUploadIsSkippedWhenFileAlreadyExists() throws Exception {
+    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
+    when(mockGcsUtil.fileSize(any(GcsPath.class))).thenReturn(tmpFile.length());
+
+    PackageUtil.stageClasspathElements(
+        ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH);
+
+    verify(mockGcsUtil).fileSize(any(GcsPath.class));
+    verifyNoMoreInteractions(mockGcsUtil);
+  }
+
+  @Test
+  public void testPackageUploadIsNotSkippedWhenSizesAreDifferent() throws Exception {
+    Pipe pipe = Pipe.open();
+    File tmpDirectory = tmpFolder.newFolder("folder");
+    tmpFolder.newFolder("folder", "empty_directory");
+    tmpFolder.newFolder("folder", "directory");
+    makeFileWithContents("folder/file.txt", "This is a test!");
+    makeFileWithContents("folder/directory/file.txt", "This is also a test!");
+    when(mockGcsUtil.fileSize(any(GcsPath.class))).thenReturn(Long.MAX_VALUE);
+    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
+
+    PackageUtil.stageClasspathElements(
+        ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH);
+
+    verify(mockGcsUtil).fileSize(any(GcsPath.class));
+    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
+    verifyNoMoreInteractions(mockGcsUtil);
+  }
+
+  @Test
+  public void testPackageUploadWithExplicitPackageName() throws Exception {
+    Pipe pipe = Pipe.open();
+    File tmpFile = makeFileWithContents("file.txt", "This is a test!");
+    final String overriddenName = "alias.txt";
+
+    when(mockGcsUtil.fileSize(any(GcsPath.class)))
+        .thenThrow(new FileNotFoundException("some/path"));
+    when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
+
+    List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
+        ImmutableList.of(overriddenName + "=" + tmpFile.getAbsolutePath()), STAGING_PATH);
+    DataflowPackage target = Iterables.getOnlyElement(targets);
+
+    verify(mockGcsUtil).fileSize(any(GcsPath.class));
+    verify(mockGcsUtil).create(any(GcsPath.class), anyString());
+    verifyNoMoreInteractions(mockGcsUtil);
+
+    assertThat(target.getName(), equalTo(overriddenName));
+    assertThat(target.getLocation(),
+        RegexMatcher.matches(STAGING_PATH + "/file-" + HASH_PATTERN + ".txt"));
+  }
+
+  @Test
+  public void testPackageUploadIsSkippedWithNonExistentResource() throws Exception {
+    String nonExistentFile =
+        IOChannelUtils.resolve(tmpFolder.getRoot().getPath(), "non-existent-file");
+    assertEquals(Collections.EMPTY_LIST, PackageUtil.stageClasspathElements(
+        ImmutableList.of(nonExistentFile), STAGING_PATH));
+  }
+
+  /**
+   * Builds a fake GoogleJsonResponseException for testing API error handling.
+   */
+  private static GoogleJsonResponseException googleJsonResponseException(
+      final int status, final String reason, final String message) throws IOException {
+    final JsonFactory jsonFactory = new JacksonFactory();
+    HttpTransport transport = new MockHttpTransport() {
+      @Override
+      public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
+        ErrorInfo errorInfo = new ErrorInfo();
+        errorInfo.setReason(reason);
+        errorInfo.setMessage(message);
+        errorInfo.setFactory(jsonFactory);
+        GenericJson error = new GenericJson();
+        error.set("code", status);
+        error.set("errors", Arrays.asList(errorInfo));
+        error.setFactory(jsonFactory);
+        GenericJson errorResponse = new GenericJson();
+        errorResponse.set("error", error);
+        errorResponse.setFactory(jsonFactory);
+        return new MockLowLevelHttpRequest().setResponse(
+            new MockLowLevelHttpResponse().setContent(errorResponse.toPrettyString())
+            .setContentType(Json.MEDIA_TYPE).setStatusCode(status));
+        }
+    };
+    HttpRequest request =
+        transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL);
+    request.setThrowExceptionOnExecuteError(false);
+    HttpResponse response = request.execute();
+    return GoogleJsonResponseException.from(jsonFactory, response);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/Pipeline.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/Pipeline.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/Pipeline.java
deleted file mode 100644
index db20d41..0000000
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/Pipeline.java
+++ /dev/null
@@ -1,503 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.dataflow.sdk;
-
-import com.google.cloud.dataflow.sdk.coders.CoderRegistry;
-import com.google.cloud.dataflow.sdk.io.Read;
-import com.google.cloud.dataflow.sdk.options.PipelineOptions;
-import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory;
-import com.google.cloud.dataflow.sdk.runners.PipelineRunner;
-import com.google.cloud.dataflow.sdk.runners.TransformHierarchy;
-import com.google.cloud.dataflow.sdk.runners.TransformTreeNode;
-import com.google.cloud.dataflow.sdk.transforms.AppliedPTransform;
-import com.google.cloud.dataflow.sdk.transforms.Create;
-import com.google.cloud.dataflow.sdk.transforms.PTransform;
-import com.google.cloud.dataflow.sdk.util.UserCodeException;
-import com.google.cloud.dataflow.sdk.values.PBegin;
-import com.google.cloud.dataflow.sdk.values.PCollection;
-import com.google.cloud.dataflow.sdk.values.PInput;
-import com.google.cloud.dataflow.sdk.values.POutput;
-import com.google.cloud.dataflow.sdk.values.PValue;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Multimap;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * A {@link Pipeline} manages a directed acyclic graph of {@link PTransform PTransforms}, and the
- * {@link PCollection PCollections} that the {@link PTransform}s consume and produce.
- *
- * <p>A {@link Pipeline} is initialized with a {@link PipelineRunner} that will later
- * execute the {@link Pipeline}.
- *
- * <p>{@link Pipeline Pipelines} are independent, so they can be constructed and executed
- * concurrently.
- *
- * <p>Each {@link Pipeline} is self-contained and isolated from any other
- * {@link Pipeline}. The {@link PValue PValues} that are inputs and outputs of each of a
- * {@link Pipeline Pipeline's} {@link PTransform PTransforms} are also owned by that
- * {@link Pipeline}. A {@link PValue} owned by one {@link Pipeline} can be read only by
- * {@link PTransform PTransforms} also owned by that {@link Pipeline}.
- *
- * <p>Here is a typical example of use:
- * <pre> {@code
- * // Start by defining the options for the pipeline.
- * PipelineOptions options = PipelineOptionsFactory.create();
- * // Then create the pipeline. The runner is determined by the options.
- * Pipeline p = Pipeline.create(options);
- *
- * // A root PTransform, like TextIO.Read or Create, gets added
- * // to the Pipeline by being applied:
- * PCollection<String> lines =
- *     p.apply(TextIO.Read.from("gs://bucket/dir/file*.txt"));
- *
- * // A Pipeline can have multiple root transforms:
- * PCollection<String> moreLines =
- *     p.apply(TextIO.Read.from("gs://bucket/other/dir/file*.txt"));
- * PCollection<String> yetMoreLines =
- *     p.apply(Create.of("yet", "more", "lines").withCoder(StringUtf8Coder.of()));
- *
- * // Further PTransforms can be applied, in an arbitrary (acyclic) graph.
- * // Subsequent PTransforms (and intermediate PCollections etc.) are
- * // implicitly part of the same Pipeline.
- * PCollection<String> allLines =
- *     PCollectionList.of(lines).and(moreLines).and(yetMoreLines)
- *     .apply(new Flatten<String>());
- * PCollection<KV<String, Integer>> wordCounts =
- *     allLines
- *     .apply(ParDo.of(new ExtractWords()))
- *     .apply(new Count<String>());
- * PCollection<String> formattedWordCounts =
- *     wordCounts.apply(ParDo.of(new FormatCounts()));
- * formattedWordCounts.apply(TextIO.Write.to("gs://bucket/dir/counts.txt"));
- *
- * // PTransforms aren't executed when they're applied, rather they're
- * // just added to the Pipeline.  Once the whole Pipeline of PTransforms
- * // is constructed, the Pipeline's PTransforms can be run using a
- * // PipelineRunner.  The default PipelineRunner executes the Pipeline
- * // directly, sequentially, in this one process, which is useful for
- * // unit tests and simple experiments:
- * p.run();
- *
- * } </pre>
- */
-public class Pipeline {
-  private static final Logger LOG = LoggerFactory.getLogger(Pipeline.class);
-
-  /**
-   * Thrown during execution of a {@link Pipeline}, whenever user code within that
-   * {@link Pipeline} throws an exception.
-   *
-   * <p>The original exception thrown by user code may be retrieved via {@link #getCause}.
-   */
-  public static class PipelineExecutionException extends RuntimeException {
-    /**
-     * Wraps {@code cause} into a {@link PipelineExecutionException}.
-     */
-    public PipelineExecutionException(Throwable cause) {
-      super(cause);
-    }
-  }
-
-  /////////////////////////////////////////////////////////////////////////////
-  // Public operations.
-
-  /**
-   * Constructs a pipeline from the provided options.
-   *
-   * @return The newly created pipeline.
-   */
-  public static Pipeline create(PipelineOptions options) {
-    Pipeline pipeline = new Pipeline(PipelineRunner.fromOptions(options), options);
-    LOG.debug("Creating {}", pipeline);
-    return pipeline;
-  }
-
-  /**
-   * Returns a {@link PBegin} owned by this Pipeline.  This is useful
-   * as the input of a root PTransform such as {@link Read} or
-   * {@link Create}.
-   */
-  public PBegin begin() {
-    return PBegin.in(this);
-  }
-
-  /**
-   * Like {@link #apply(String, PTransform)} but the transform node in the {@link Pipeline}
-   * graph will be named according to {@link PTransform#getName}.
-   *
-   * @see #apply(String, PTransform)
-   */
-  public <OutputT extends POutput> OutputT apply(
-      PTransform<? super PBegin, OutputT> root) {
-    return begin().apply(root);
-  }
-
-  /**
-   * Adds a root {@link PTransform}, such as {@link Read} or {@link Create},
-   * to this {@link Pipeline}.
-   *
-   * <p>The node in the {@link Pipeline} graph will use the provided {@code name}.
-   * This name is used in various places, including the monitoring UI, logging,
-   * and to stably identify this node in the {@link Pipeline} graph upon update.
-   *
-   * <p>Alias for {@code begin().apply(name, root)}.
-   */
-  public <OutputT extends POutput> OutputT apply(
-      String name, PTransform<? super PBegin, OutputT> root) {
-    return begin().apply(name, root);
-  }
-
-  /**
-   * Runs the {@link Pipeline} using its {@link PipelineRunner}.
-   */
-  public PipelineResult run() {
-    LOG.debug("Running {} via {}", this, runner);
-    try {
-      return runner.run(this);
-    } catch (UserCodeException e) {
-      // This serves to replace the stack with one that ends here and
-      // is caused by the caught UserCodeException, thereby splicing
-      // out all the stack frames in between the PipelineRunner itself
-      // and where the worker calls into the user's code.
-      throw new PipelineExecutionException(e.getCause());
-    }
-  }
-
-
-  /////////////////////////////////////////////////////////////////////////////
-  // Below here are operations that aren't normally called by users.
-
-  /**
-   * Returns the {@link CoderRegistry} that this {@link Pipeline} uses.
-   */
-  public CoderRegistry getCoderRegistry() {
-    if (coderRegistry == null) {
-      coderRegistry = new CoderRegistry();
-      coderRegistry.registerStandardCoders();
-    }
-    return coderRegistry;
-  }
-
-  /**
-   * Sets the {@link CoderRegistry} that this {@link Pipeline} uses.
-   */
-  public void setCoderRegistry(CoderRegistry coderRegistry) {
-    this.coderRegistry = coderRegistry;
-  }
-
-  /**
-   * A {@link PipelineVisitor} can be passed into
-   * {@link Pipeline#traverseTopologically} to be called for each of the
-   * transforms and values in the {@link Pipeline}.
-   */
-  public interface PipelineVisitor {
-    /**
-     * Called for each composite transform after all topological predecessors have been visited
-     * but before any of its component transforms.
-     */
-    public void enterCompositeTransform(TransformTreeNode node);
-
-    /**
-     * Called for each composite transform after all of its component transforms and their outputs
-     * have been visited.
-     */
-    public void leaveCompositeTransform(TransformTreeNode node);
-
-    /**
-     * Called for each primitive transform after all of its topological predecessors
-     * and inputs have been visited.
-     */
-    public void visitTransform(TransformTreeNode node);
-
-    /**
-     * Called for each value after the transform that produced the value has been
-     * visited.
-     */
-    public void visitValue(PValue value, TransformTreeNode producer);
-  }
-
-  /**
-   * Invokes the {@link PipelineVisitor PipelineVisitor's}
-   * {@link PipelineVisitor#visitTransform} and
-   * {@link PipelineVisitor#visitValue} operations on each of this
-   * {@link Pipeline Pipeline's} transform and value nodes, in forward
-   * topological order.
-   *
-   * <p>Traversal of the {@link Pipeline} causes {@link PTransform PTransforms} and
-   * {@link PValue PValues} owned by the {@link Pipeline} to be marked as finished,
-   * at which point they may no longer be modified.
-   *
-   * <p>Typically invoked by {@link PipelineRunner} subclasses.
-   */
-  public void traverseTopologically(PipelineVisitor visitor) {
-    Set<PValue> visitedValues = new HashSet<>();
-    // Visit all the transforms, which should implicitly visit all the values.
-    transforms.visit(visitor, visitedValues);
-    if (!visitedValues.containsAll(values)) {
-      throw new RuntimeException(
-          "internal error: should have visited all the values "
-          + "after visiting all the transforms");
-    }
-  }
-
-  /**
-   * Like {@link #applyTransform(String, PInput, PTransform)} but defaulting to the name
-   * provided by the {@link PTransform}.
-   */
-  public static <InputT extends PInput, OutputT extends POutput>
-  OutputT applyTransform(InputT input,
-      PTransform<? super InputT, OutputT> transform) {
-    return input.getPipeline().applyInternal(transform.getName(), input, transform);
-  }
-
-  /**
-   * Applies the given {@code PTransform} to this input {@code InputT} and returns
-   * its {@code OutputT}. This uses {@code name} to identify this specific application
-   * of the transform. This name is used in various places, including the monitoring UI,
-   * logging, and to stably identify this application node in the {@link Pipeline} graph during
-   * update.
-   *
-   * <p>Each {@link PInput} subclass that provides an {@code apply} method should delegate to
-   * this method to ensure proper registration with the {@link PipelineRunner}.
-   */
-  public static <InputT extends PInput, OutputT extends POutput>
-  OutputT applyTransform(String name, InputT input,
-      PTransform<? super InputT, OutputT> transform) {
-    return input.getPipeline().applyInternal(name, input, transform);
-  }
-
-  /////////////////////////////////////////////////////////////////////////////
-  // Below here are internal operations, never called by users.
-
-  private final PipelineRunner<?> runner;
-  private final PipelineOptions options;
-  private final TransformHierarchy transforms = new TransformHierarchy();
-  private Collection<PValue> values = new ArrayList<>();
-  private Set<String> usedFullNames = new HashSet<>();
-  private CoderRegistry coderRegistry;
-  private Multimap<PTransform<?, ?>, AppliedPTransform<?, ?, ?>> transformApplicationsForTesting =
-      HashMultimap.create();
-
-  /**
-   * @deprecated replaced by {@link #Pipeline(PipelineRunner, PipelineOptions)}
-   */
-  @Deprecated
-  protected Pipeline(PipelineRunner<?> runner) {
-    this(runner, PipelineOptionsFactory.create());
-  }
-
-  protected Pipeline(PipelineRunner<?> runner, PipelineOptions options) {
-    this.runner = runner;
-    this.options = options;
-  }
-
-  @Override
-  public String toString() {
-    return "Pipeline#" + hashCode();
-  }
-
-  /**
-   * Applies a {@link PTransform} to the given {@link PInput}.
-   *
-   * @see Pipeline#apply
-   */
-  private <InputT extends PInput, OutputT extends POutput>
-  OutputT applyInternal(String name, InputT input,
-      PTransform<? super InputT, OutputT> transform) {
-    input.finishSpecifying();
-
-    TransformTreeNode parent = transforms.getCurrent();
-    String namePrefix = parent.getFullName();
-    String fullName = uniquifyInternal(namePrefix, name);
-
-    boolean nameIsUnique = fullName.equals(buildName(namePrefix, name));
-
-    if (!nameIsUnique) {
-      switch (getOptions().getStableUniqueNames()) {
-        case OFF:
-          break;
-        case WARNING:
-          LOG.warn("Transform {} does not have a stable unique name. "
-              + "This will prevent updating of pipelines.", fullName);
-          break;
-        case ERROR:
-          throw new IllegalStateException(
-              "Transform " + fullName + " does not have a stable unique name. "
-              + "This will prevent updating of pipelines.");
-        default:
-          throw new IllegalArgumentException(
-              "Unrecognized value for stable unique names: " + getOptions().getStableUniqueNames());
-      }
-    }
-
-    TransformTreeNode child =
-        new TransformTreeNode(parent, transform, fullName, input);
-    parent.addComposite(child);
-
-    transforms.addInput(child, input);
-
-    LOG.debug("Adding {} to {}", transform, this);
-    try {
-      transforms.pushNode(child);
-      transform.validate(input);
-      OutputT output = runner.apply(transform, input);
-      transforms.setOutput(child, output);
-
-      AppliedPTransform<?, ?, ?> applied = AppliedPTransform.of(
-          child.getFullName(), input, output, transform);
-      transformApplicationsForTesting.put(transform, applied);
-      // recordAsOutput is a NOOP if already called;
-      output.recordAsOutput(applied);
-      verifyOutputState(output, child);
-      return output;
-    } finally {
-      transforms.popNode();
-    }
-  }
-
-  /**
-   * Returns all producing transforms for the {@link PValue PValues} contained
-   * in {@code output}.
-   */
-  private List<AppliedPTransform<?, ?, ?>> getProducingTransforms(POutput output) {
-    List<AppliedPTransform<?, ?, ?>> producingTransforms = new ArrayList<>();
-    for (PValue value : output.expand()) {
-      AppliedPTransform<?, ?, ?> transform = value.getProducingTransformInternal();
-      if (transform != null) {
-        producingTransforms.add(transform);
-      }
-    }
-    return producingTransforms;
-  }
-
-  /**
-   * Verifies that the output of a {@link PTransform} is correctly configured in its
-   * {@link TransformTreeNode} in the {@link Pipeline} graph.
-   *
-   * <p>A non-composite {@link PTransform} must have all
-   * of its outputs registered as produced by that {@link PTransform}.
-   *
-   * <p>A composite {@link PTransform} must have all of its outputs
-   * registered as produced by the contained primitive {@link PTransform PTransforms}.
-   * They have each had the above check performed already, when
-   * they were applied, so the only possible failure state is
-   * that the composite {@link PTransform} has returned a primitive output.
-   */
-  private void verifyOutputState(POutput output, TransformTreeNode node) {
-    if (!node.isCompositeNode()) {
-      PTransform<?, ?> thisTransform = node.getTransform();
-      List<AppliedPTransform<?, ?, ?>> producingTransforms = getProducingTransforms(output);
-      for (AppliedPTransform<?, ?, ?> producingTransform : producingTransforms) {
-        // Using != because object identity indicates that the transforms
-        // are the same node in the pipeline
-        if (thisTransform != producingTransform.getTransform()) {
-          throw new IllegalArgumentException("Output of non-composite transform "
-              + thisTransform + " is registered as being produced by"
-              + " a different transform: " + producingTransform);
-        }
-      }
-    } else {
-      PTransform<?, ?> thisTransform = node.getTransform();
-      List<AppliedPTransform<?, ?, ?>> producingTransforms = getProducingTransforms(output);
-      for (AppliedPTransform<?, ?, ?> producingTransform : producingTransforms) {
-        // Using == because object identity indicates that the transforms
-        // are the same node in the pipeline
-        if (thisTransform == producingTransform.getTransform()) {
-          throw new IllegalStateException("Output of composite transform "
-              + thisTransform + " is registered as being produced by it,"
-              + " but the output of every composite transform should be"
-              + " produced by a primitive transform contained therein.");
-        }
-      }
-    }
-  }
-
-  /**
-   * Returns the configured {@link PipelineRunner}.
-   */
-  public PipelineRunner<?> getRunner() {
-    return runner;
-  }
-
-  /**
-   * Returns the configured {@link PipelineOptions}.
-   */
-  public PipelineOptions getOptions() {
-    return options;
-  }
-
-  /**
-   * @deprecated this method is no longer compatible with the design of {@link Pipeline},
-   * as {@link PTransform PTransforms} can be applied multiple times, with different names
-   * each time.
-   */
-  @Deprecated
-  public String getFullNameForTesting(PTransform<?, ?> transform) {
-    Collection<AppliedPTransform<?, ?, ?>> uses =
-        transformApplicationsForTesting.get(transform);
-    Preconditions.checkState(uses.size() > 0, "Unknown transform: " + transform);
-    Preconditions.checkState(uses.size() <= 1, "Transform used multiple times: " + transform);
-    return Iterables.getOnlyElement(uses).getFullName();
-  }
-
-  /**
-   * Returns a unique name for a transform with the given prefix (from
-   * enclosing transforms) and initial name.
-   *
-   * <p>For internal use only.
-   */
-  private String uniquifyInternal(String namePrefix, String origName) {
-    String name = origName;
-    int suffixNum = 2;
-    while (true) {
-      String candidate = buildName(namePrefix, name);
-      if (usedFullNames.add(candidate)) {
-        return candidate;
-      }
-      // A duplicate!  Retry.
-      name = origName + suffixNum++;
-    }
-  }
-
-  /**
-   * Builds a name from a "/"-delimited prefix and a name.
-   */
-  private String buildName(String namePrefix, String name) {
-    return namePrefix.isEmpty() ? name : namePrefix + "/" + name;
-  }
-
-  /**
-   * Adds the given {@link PValue} to this {@link Pipeline}.
-   *
-   * <p>For internal use only.
-   */
-  public void addValueInternal(PValue value) {
-    this.values.add(value);
-    LOG.debug("Adding {} to {}", value, this);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/PipelineResult.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/PipelineResult.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/PipelineResult.java
deleted file mode 100644
index 90d09ec..0000000
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/PipelineResult.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.dataflow.sdk;
-
-import com.google.cloud.dataflow.sdk.runners.AggregatorRetrievalException;
-import com.google.cloud.dataflow.sdk.runners.AggregatorValues;
-import com.google.cloud.dataflow.sdk.transforms.Aggregator;
-
-/**
- * Result of {@link Pipeline#run()}.
- */
-public interface PipelineResult {
-
-  /**
-   * Retrieves the current state of the pipeline execution.
-   *
-   * @return the {@link State} representing the state of this pipeline.
-   */
-  State getState();
-
-  /**
-   * Retrieves the current value of the provided {@link Aggregator}.
-   *
-   * @param aggregator the {@link Aggregator} to retrieve values for.
-   * @return the current values of the {@link Aggregator},
-   * which may be empty if there are no values yet.
-   * @throws AggregatorRetrievalException if the {@link Aggregator} values could not be retrieved.
-   */
-  <T> AggregatorValues<T> getAggregatorValues(Aggregator<?, T> aggregator)
-      throws AggregatorRetrievalException;
-
-  // TODO: method to retrieve error messages.
-
-  /** Named constants for common values for the job state. */
-  public enum State {
-
-    /** The job state could not be obtained or was not specified. */
-    UNKNOWN(false, false),
-
-    /** The job has been paused, or has not yet started. */
-    STOPPED(false, false),
-
-    /** The job is currently running. */
-    RUNNING(false, false),
-
-    /** The job has successfully completed. */
-    DONE(true, false),
-
-    /** The job has failed. */
-    FAILED(true, false),
-
-    /** The job has been explicitly cancelled. */
-    CANCELLED(true, false),
-
-    /** The job has been updated. */
-    UPDATED(true, true);
-
-    private final boolean terminal;
-
-    private final boolean hasReplacement;
-
-    private State(boolean terminal, boolean hasReplacement) {
-      this.terminal = terminal;
-      this.hasReplacement = hasReplacement;
-    }
-
-    /**
-     * @return {@code true} if the job state can no longer complete work.
-     */
-    public final boolean isTerminal() {
-      return terminal;
-    }
-
-    /**
-     * @return {@code true} if this job state indicates that a replacement job exists.
-     */
-    public final boolean hasReplacementJob() {
-      return hasReplacement;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AtomicCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AtomicCoder.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AtomicCoder.java
deleted file mode 100644
index adda5a2..0000000
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AtomicCoder.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.dataflow.sdk.coders;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * A {@link Coder} that has no component {@link Coder Coders} or other state.
- *
- * <p>Note that, unless the behavior is overridden, atomic coders are presumed to be deterministic
- * and all instances are considered equal.
- *
- * @param <T> the type of the values being transcoded
- */
-public abstract class AtomicCoder<T> extends DeterministicStandardCoder<T> {
-  protected AtomicCoder() { }
-
-  @Override
-  public List<Coder<?>> getCoderArguments() {
-    return null;
-  }
-
-  /**
-   * Returns a list of values contained in the provided example
-   * value, one per type parameter. If there are no type parameters,
-   * returns an empty list.
-   *
-   * <p>Because {@link AtomicCoder} has no components, always returns an empty list.
-   *
-   * @param exampleValue unused, but part of the latent interface expected by
-   * {@link CoderFactories#fromStaticMethods}
-   */
-  public static <T> List<Object> getInstanceComponents(T exampleValue) {
-    return Collections.emptyList();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AvroCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AvroCoder.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AvroCoder.java
deleted file mode 100644
index e7ac0e3..0000000
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/AvroCoder.java
+++ /dev/null
@@ -1,715 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.dataflow.sdk.coders;
-
-import static com.google.cloud.dataflow.sdk.util.Structs.addString;
-
-import com.google.cloud.dataflow.sdk.util.CloudObject;
-import com.google.cloud.dataflow.sdk.values.TypeDescriptor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.generic.IndexedRecord;
-import org.apache.avro.io.BinaryDecoder;
-import org.apache.avro.io.BinaryEncoder;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.io.DecoderFactory;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.reflect.AvroEncode;
-import org.apache.avro.reflect.AvroName;
-import org.apache.avro.reflect.AvroSchema;
-import org.apache.avro.reflect.ReflectData;
-import org.apache.avro.reflect.ReflectDatumReader;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.avro.reflect.Union;
-import org.apache.avro.specific.SpecificData;
-import org.apache.avro.util.ClassUtils;
-import org.apache.avro.util.Utf8;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-
-import javax.annotation.Nullable;
-
-/**
- * A {@link Coder} using Avro binary format.
- *
- * <p>Each instance of {@code AvroCoder<T>} encapsulates an Avro schema for objects of type
- * {@code T}.
- *
- * <p>The Avro schema may be provided explicitly via {@link AvroCoder#of(Class, Schema)} or
- * omitted via {@link AvroCoder#of(Class)}, in which case it will be inferred
- * using Avro's {@link org.apache.avro.reflect.ReflectData}.
- *
- * <p>For complete details about schema generation and how it can be controlled please see
- * the {@link org.apache.avro.reflect} package.
- * Only concrete classes with a no-argument constructor can be mapped to Avro records.
- * All inherited fields that are not static or transient are included. Fields are not permitted to
- * be null unless annotated by {@link Nullable} or a {@link Union} schema
- * containing {@code "null"}.
- *
- * <p>To use, specify the {@code Coder} type on a PCollection:
- * <pre>
- * {@code
- * PCollection<MyCustomElement> records =
- *     input.apply(...)
- *          .setCoder(AvroCoder.of(MyCustomElement.class);
- * }
- * </pre>
- *
- * <p>or annotate the element class using {@code @DefaultCoder}.
- * <pre><code>
- * {@literal @}DefaultCoder(AvroCoder.class)
- * public class MyCustomElement {
- *   ...
- * }
- * </code></pre>
- *
- * <p>The implementation attempts to determine if the Avro encoding of the given type will satisfy
- * the criteria of {@link Coder#verifyDeterministic} by inspecting both the type and the
- * Schema provided or generated by Avro. Only coders that are deterministic can be used in
- * {@link com.google.cloud.dataflow.sdk.transforms.GroupByKey} operations.
- *
- * @param <T> the type of elements handled by this coder
- */
-public class AvroCoder<T> extends StandardCoder<T> {
-
-  /**
-   * Returns an {@code AvroCoder} instance for the provided element type.
-   * @param <T> the element type
-   */
-  public static <T> AvroCoder<T> of(TypeDescriptor<T> type) {
-    @SuppressWarnings("unchecked")
-    Class<T> clazz = (Class<T>) type.getRawType();
-    return of(clazz);
-  }
-
-  /**
-   * Returns an {@code AvroCoder} instance for the provided element class.
-   * @param <T> the element type
-   */
-  public static <T> AvroCoder<T> of(Class<T> clazz) {
-    return new AvroCoder<>(clazz, ReflectData.get().getSchema(clazz));
-  }
-
-  /**
-   * Returns an {@code AvroCoder} instance for the Avro schema. The implicit
-   * type is GenericRecord.
-   */
-  public static AvroCoder<GenericRecord> of(Schema schema) {
-    return new AvroCoder<>(GenericRecord.class, schema);
-  }
-
-  /**
-   * Returns an {@code AvroCoder} instance for the provided element type
-   * using the provided Avro schema.
-   *
-   * <p>If the type argument is GenericRecord, the schema may be arbitrary.
-   * Otherwise, the schema must correspond to the type provided.
-   *
-   * @param <T> the element type
-   */
-  public static <T> AvroCoder<T> of(Class<T> type, Schema schema) {
-    return new AvroCoder<>(type, schema);
-  }
-
-  @SuppressWarnings({"unchecked", "rawtypes"})
-  @JsonCreator
-  public static AvroCoder<?> of(
-      @JsonProperty("type") String classType,
-      @JsonProperty("schema") String schema) throws ClassNotFoundException {
-    Schema.Parser parser = new Schema.Parser();
-    return new AvroCoder(Class.forName(classType), parser.parse(schema));
-  }
-
-  public static final CoderProvider PROVIDER = new CoderProvider() {
-    @Override
-    public <T> Coder<T> getCoder(TypeDescriptor<T> typeDescriptor) {
-      // This is a downcast from `? super T` to T. However, because
-      // it comes from a TypeDescriptor<T>, the class object itself
-      // is the same so the supertype in question shares the same
-      // generated AvroCoder schema.
-      @SuppressWarnings("unchecked")
-      Class<T> rawType = (Class<T>) typeDescriptor.getRawType();
-      return AvroCoder.of(rawType);
-    }
-  };
-
-  private final Class<T> type;
-  private final Schema schema;
-
-  private final List<String> nonDeterministicReasons;
-
-  // Factories allocated by .get() are thread-safe and immutable.
-  private static final EncoderFactory ENCODER_FACTORY = EncoderFactory.get();
-  private static final DecoderFactory DECODER_FACTORY = DecoderFactory.get();
-  // Cache the old encoder/decoder and let the factories reuse them when possible. To be threadsafe,
-  // these are ThreadLocal. This code does not need to be re-entrant as AvroCoder does not use
-  // an inner coder.
-  private final ThreadLocal<BinaryDecoder> decoder;
-  private final ThreadLocal<BinaryEncoder> encoder;
-  private final ThreadLocal<DatumWriter<T>> writer;
-  private final ThreadLocal<DatumReader<T>> reader;
-
-  protected AvroCoder(Class<T> type, Schema schema) {
-    this.type = type;
-    this.schema = schema;
-
-    nonDeterministicReasons = new AvroDeterminismChecker().check(TypeDescriptor.of(type), schema);
-
-    // Decoder and Encoder start off null for each thread. They are allocated and potentially
-    // reused inside encode/decode.
-    this.decoder = new ThreadLocal<>();
-    this.encoder = new ThreadLocal<>();
-
-    // Reader and writer are allocated once per thread and are "final" for thread-local Coder
-    // instance.
-    this.reader = new ThreadLocal<DatumReader<T>>() {
-      @Override
-      public DatumReader<T> initialValue() {
-        return createDatumReader();
-      }
-    };
-    this.writer = new ThreadLocal<DatumWriter<T>>() {
-      @Override
-      public DatumWriter<T> initialValue() {
-        return createDatumWriter();
-      }
-    };
-  }
-
-  /**
-   * The encoding identifier is designed to support evolution as per the design of Avro
-   * In order to use this class effectively, carefully read the Avro
-   * documentation at
-   * <a href="https://avro.apache.org/docs/1.7.7/spec.html#Schema+Resolution">Schema Resolution</a>
-   * to ensure that the old and new schema <i>match</i>.
-   *
-   * <p>In particular, this encoding identifier is guaranteed to be the same for {@code AvroCoder}
-   * instances of the same principal class, and otherwise distinct. The schema is not included
-   * in the identifier.
-   *
-   * <p>When modifying a class to be encoded as Avro, here are some guidelines; see the above link
-   * for greater detail.
-   *
-   * <ul>
-   * <li>Avoid changing field names.
-   * <li>Never remove a <code>required</code> field.
-   * <li>Only add <code>optional</code> fields, with sensible defaults.
-   * <li>When changing the type of a field, consult the Avro documentation to ensure the new and
-   * old types are interchangeable.
-   * </ul>
-   *
-   * <p>Code consuming this message class should be prepared to support <i>all</i> versions of
-   * the class until it is certain that no remaining serialized instances exist.
-   *
-   * <p>If backwards incompatible changes must be made, the best recourse is to change the name
-   * of your class.
-   */
-  @Override
-  public String getEncodingId() {
-    return type.getName();
-  }
-
-  /**
-   * Returns the type this coder encodes/decodes.
-   */
-  public Class<T> getType() {
-    return type;
-  }
-
-  private Object writeReplace() {
-    // When serialized by Java, instances of AvroCoder should be replaced by
-    // a SerializedAvroCoderProxy.
-    return new SerializedAvroCoderProxy<>(type, schema.toString());
-  }
-
-  @Override
-  public void encode(T value, OutputStream outStream, Context context) throws IOException {
-    // Get a BinaryEncoder instance from the ThreadLocal cache and attempt to reuse it.
-    BinaryEncoder encoderInstance = ENCODER_FACTORY.directBinaryEncoder(outStream, encoder.get());
-    // Save the potentially-new instance for reuse later.
-    encoder.set(encoderInstance);
-    writer.get().write(value, encoderInstance);
-    // Direct binary encoder does not buffer any data and need not be flushed.
-  }
-
-  @Override
-  public T decode(InputStream inStream, Context context) throws IOException {
-    // Get a BinaryDecoder instance from the ThreadLocal cache and attempt to reuse it.
-    BinaryDecoder decoderInstance = DECODER_FACTORY.directBinaryDecoder(inStream, decoder.get());
-    // Save the potentially-new instance for later.
-    decoder.set(decoderInstance);
-    return reader.get().read(null, decoderInstance);
-  }
-
-  @Override
-    public List<? extends Coder<?>> getCoderArguments() {
-    return null;
-  }
-
-  @Override
-  public CloudObject asCloudObject() {
-    CloudObject result = super.asCloudObject();
-    addString(result, "type", type.getName());
-    addString(result, "schema", schema.toString());
-    return result;
-  }
-
-  /**
-   * @throws NonDeterministicException when the type may not be deterministically
-   * encoded using the given {@link Schema}, the {@code directBinaryEncoder}, and the
-   * {@link ReflectDatumWriter} or {@link GenericDatumWriter}.
-   */
-  @Override
-  public void verifyDeterministic() throws NonDeterministicException {
-    if (!nonDeterministicReasons.isEmpty()) {
-      throw new NonDeterministicException(this, nonDeterministicReasons);
-    }
-  }
-
-  /**
-   * Returns a new {@link DatumReader} that can be used to read from an Avro file directly. Assumes
-   * the schema used to read is the same as the schema that was used when writing.
-   *
-   * @deprecated For {@code AvroCoder} internal use only.
-   */
-  // TODO: once we can remove this deprecated function, inline in constructor.
-  @Deprecated
-  public DatumReader<T> createDatumReader() {
-    if (type.equals(GenericRecord.class)) {
-      return new GenericDatumReader<>(schema);
-    } else {
-      return new ReflectDatumReader<>(schema);
-    }
-  }
-
-  /**
-   * Returns a new {@link DatumWriter} that can be used to write to an Avro file directly.
-   *
-   * @deprecated For {@code AvroCoder} internal use only.
-   */
-  // TODO: once we can remove this deprecated function, inline in constructor.
-  @Deprecated
-  public DatumWriter<T> createDatumWriter() {
-    if (type.equals(GenericRecord.class)) {
-      return new GenericDatumWriter<>(schema);
-    } else {
-      return new ReflectDatumWriter<>(schema);
-    }
-  }
-
-  /**
-   * Returns the schema used by this coder.
-   */
-  public Schema getSchema() {
-    return schema;
-  }
-
-  /**
-   * Proxy to use in place of serializing the {@link AvroCoder}. This allows the fields
-   * to remain final.
-   */
-  private static class SerializedAvroCoderProxy<T> implements Serializable {
-    private final Class<T> type;
-    private final String schemaStr;
-
-    public SerializedAvroCoderProxy(Class<T> type, String schemaStr) {
-      this.type = type;
-      this.schemaStr = schemaStr;
-    }
-
-    private Object readResolve() {
-      // When deserialized, instances of this object should be replaced by
-      // constructing an AvroCoder.
-      Schema.Parser parser = new Schema.Parser();
-      return new AvroCoder<T>(type, parser.parse(schemaStr));
-    }
-  }
-
-  /**
-   * Helper class encapsulating the various pieces of state maintained by the
-   * recursive walk used for checking if the encoding will be deterministic.
-   */
-  private static class AvroDeterminismChecker {
-
-    // Reasons that the original type are not deterministic. This accumulates
-    // the actual output.
-    private List<String> reasons = new ArrayList<>();
-
-    // Types that are currently "open". Used to make sure we don't have any
-    // recursive types. Note that we assume that all occurrences of a given type
-    // are equal, rather than tracking pairs of type + schema.
-    private Set<TypeDescriptor<?>> activeTypes = new HashSet<>();
-
-    // Similarly to how we record active types, we record the schemas we visit
-    // to make sure we don't encounter recursive fields.
-    private Set<Schema> activeSchemas = new HashSet<>();
-
-    /**
-     * Report an error in the current context.
-     */
-    private void reportError(String context, String fmt, Object... args) {
-      String message = String.format(fmt, args);
-      reasons.add(context + ": " + message);
-    }
-
-    /**
-     * Classes that are serialized by Avro as a String include
-     * <ul>
-     * <li>Subtypes of CharSequence (including String, Avro's mutable Utf8, etc.)
-     * <li>Several predefined classes (BigDecimal, BigInteger, URI, URL)
-     * <li>Classes annotated with @Stringable (uses their #toString() and a String constructor)
-     * </ul>
-     *
-     * <p>Rather than determine which of these cases are deterministic, we list some classes
-     * that definitely are, and treat any others as non-deterministic.
-     */
-    private static final Set<Class<?>> DETERMINISTIC_STRINGABLE_CLASSES = new HashSet<>();
-    static {
-      // CharSequences:
-      DETERMINISTIC_STRINGABLE_CLASSES.add(String.class);
-      DETERMINISTIC_STRINGABLE_CLASSES.add(Utf8.class);
-
-      // Explicitly Stringable:
-      DETERMINISTIC_STRINGABLE_CLASSES.add(java.math.BigDecimal.class);
-      DETERMINISTIC_STRINGABLE_CLASSES.add(java.math.BigInteger.class);
-      DETERMINISTIC_STRINGABLE_CLASSES.add(java.net.URI.class);
-      DETERMINISTIC_STRINGABLE_CLASSES.add(java.net.URL.class);
-
-      // Classes annotated with @Stringable:
-    }
-
-    /**
-     * Return true if the given type token is a subtype of *any* of the listed parents.
-     */
-    private static boolean isSubtypeOf(TypeDescriptor<?> type, Class<?>... parents) {
-      for (Class<?> parent : parents) {
-        if (type.isSubtypeOf(TypeDescriptor.of(parent))) {
-          return true;
-        }
-      }
-      return false;
-    }
-
-    protected AvroDeterminismChecker() {}
-
-    // The entry point for the check. Should not be recursively called.
-    public List<String> check(TypeDescriptor<?> type, Schema schema) {
-      recurse(type.getRawType().getName(), type, schema);
-      return reasons;
-    }
-
-    // This is the method that should be recursively called. It sets up the path
-    // and visited types correctly.
-    private void recurse(String context, TypeDescriptor<?> type, Schema schema) {
-      if (type.getRawType().isAnnotationPresent(AvroSchema.class)) {
-        reportError(context, "Custom schemas are not supported -- remove @AvroSchema.");
-        return;
-      }
-
-      if (!activeTypes.add(type)) {
-        reportError(context, "%s appears recursively", type);
-        return;
-      }
-
-      // If the the record isn't a true class, but rather a GenericRecord, SpecificRecord, etc.
-      // with a specified schema, then we need to make the decision based on the generated
-      // implementations.
-      if (isSubtypeOf(type, IndexedRecord.class)) {
-        checkIndexedRecord(context, schema, null);
-      } else {
-        doCheck(context, type, schema);
-      }
-
-      activeTypes.remove(type);
-    }
-
-    private void doCheck(String context, TypeDescriptor<?> type, Schema schema) {
-      switch (schema.getType()) {
-        case ARRAY:
-          checkArray(context, type, schema);
-          break;
-        case ENUM:
-          // Enums should be deterministic, since they depend only on the ordinal.
-          break;
-        case FIXED:
-          // Depending on the implementation of GenericFixed, we don't know how
-          // the given field will be encoded. So, we assume that it isn't
-          // deterministic.
-          reportError(context, "FIXED encodings are not guaranteed to be deterministic");
-          break;
-        case MAP:
-          checkMap(context, type, schema);
-          break;
-        case RECORD:
-          checkRecord(type, schema);
-          break;
-        case UNION:
-          checkUnion(context, type, schema);
-          break;
-        case STRING:
-          checkString(context, type);
-          break;
-        case BOOLEAN:
-        case BYTES:
-        case DOUBLE:
-        case INT:
-        case FLOAT:
-        case LONG:
-        case NULL:
-          // For types that Avro encodes using one of the above primitives, we assume they are
-          // deterministic.
-          break;
-        default:
-          // In any other case (eg., new types added to Avro) we cautiously return
-          // false.
-          reportError(context, "Unknown schema type %s may be non-deterministic", schema.getType());
-          break;
-      }
-    }
-
-    private void checkString(String context, TypeDescriptor<?> type) {
-      // For types that are encoded as strings, we need to make sure they're in an approved
-      // whitelist. For other types that are annotated @Stringable, Avro will just use the
-      // #toString() methods, which has no guarantees of determinism.
-      if (!DETERMINISTIC_STRINGABLE_CLASSES.contains(type.getRawType())) {
-        reportError(context, "%s may not have deterministic #toString()", type);
-      }
-    }
-
-   private static final Schema AVRO_NULL_SCHEMA = Schema.create(Schema.Type.NULL);
-
-   private void checkUnion(String context, TypeDescriptor<?> type, Schema schema) {
-      final List<Schema> unionTypes = schema.getTypes();
-
-      if (!type.getRawType().isAnnotationPresent(Union.class)) {
-        // First check for @Nullable field, which shows up as a union of field type and null.
-        if (unionTypes.size() == 2 && unionTypes.contains(AVRO_NULL_SCHEMA)) {
-          // Find the Schema that is not NULL and recursively check that it is deterministic.
-          Schema nullableFieldSchema = unionTypes.get(0).equals(AVRO_NULL_SCHEMA)
-              ? unionTypes.get(1) : unionTypes.get(0);
-          doCheck(context, type, nullableFieldSchema);
-          return;
-        }
-
-        // Otherwise report a schema error.
-        reportError(context, "Expected type %s to have @Union annotation", type);
-        return;
-      }
-
-      // Errors associated with this union will use the base class as their context.
-      String baseClassContext = type.getRawType().getName();
-
-      // For a union, we need to make sure that each possible instantiation is deterministic.
-      for (Schema concrete : unionTypes) {
-        @SuppressWarnings("unchecked")
-        TypeDescriptor<?> unionType = TypeDescriptor.of(ReflectData.get().getClass(concrete));
-
-        recurse(baseClassContext, unionType, concrete);
-      }
-    }
-
-    private void checkRecord(TypeDescriptor<?> type, Schema schema) {
-      // For a record, we want to make sure that all the fields are deterministic.
-      Class<?> clazz = type.getRawType();
-      for (org.apache.avro.Schema.Field fieldSchema : schema.getFields()) {
-        Field field = getField(clazz, fieldSchema.name());
-        String fieldContext = field.getDeclaringClass().getName() + "#" + field.getName();
-
-        if (field.isAnnotationPresent(AvroEncode.class)) {
-          reportError(fieldContext,
-              "Custom encoders may be non-deterministic -- remove @AvroEncode");
-          continue;
-        }
-
-        if (!IndexedRecord.class.isAssignableFrom(field.getType())
-            && field.isAnnotationPresent(AvroSchema.class)) {
-          // TODO: We should be able to support custom schemas on POJO fields, but we shouldn't
-          // need to, so we just allow it in the case of IndexedRecords.
-          reportError(fieldContext,
-              "Custom schemas are only supported for subtypes of IndexedRecord.");
-          continue;
-        }
-
-        TypeDescriptor<?> fieldType = type.resolveType(field.getGenericType());
-        recurse(fieldContext, fieldType, fieldSchema.schema());
-      }
-    }
-
-    private void checkIndexedRecord(String context, Schema schema,
-        @Nullable String specificClassStr) {
-
-      if (!activeSchemas.add(schema)) {
-        reportError(context, "%s appears recursively", schema.getName());
-        return;
-      }
-
-      switch (schema.getType()) {
-        case ARRAY:
-          // Generic Records use GenericData.Array to implement arrays, which is
-          // essentially an ArrayList, and therefore ordering is deterministic.
-          // The array is thus deterministic if the elements are deterministic.
-          checkIndexedRecord(context, schema.getElementType(), null);
-          break;
-        case ENUM:
-          // Enums are deterministic because they encode as a single integer.
-          break;
-        case FIXED:
-          // In the case of GenericRecords, FIXED is deterministic because it
-          // encodes/decodes as a Byte[].
-          break;
-        case MAP:
-          reportError(context,
-              "GenericRecord and SpecificRecords use a HashMap to represent MAPs,"
-              + " so it is non-deterministic");
-          break;
-        case RECORD:
-          for (org.apache.avro.Schema.Field field : schema.getFields()) {
-            checkIndexedRecord(
-                schema.getName() + "." + field.name(),
-                field.schema(),
-                field.getProp(SpecificData.CLASS_PROP));
-          }
-          break;
-        case STRING:
-          // GenericDatumWriter#findStringClass will use a CharSequence or a String
-          // for each string, so it is deterministic.
-
-          // SpecificCompiler#getStringType will use java.lang.String, org.apache.avro.util.Utf8,
-          // or java.lang.CharSequence, unless SpecificData.CLASS_PROP overrides that.
-          if (specificClassStr != null) {
-            Class<?> specificClass;
-            try {
-              specificClass = ClassUtils.forName(specificClassStr);
-              if (!DETERMINISTIC_STRINGABLE_CLASSES.contains(specificClass)) {
-                reportError(context, "Specific class %s is not known to be deterministic",
-                    specificClassStr);
-              }
-            } catch (ClassNotFoundException e) {
-              reportError(context, "Specific class %s is not known to be deterministic",
-                  specificClassStr);
-            }
-          }
-          break;
-        case UNION:
-          for (org.apache.avro.Schema subschema : schema.getTypes()) {
-            checkIndexedRecord(subschema.getName(), subschema, null);
-          }
-          break;
-        case BOOLEAN:
-        case BYTES:
-        case DOUBLE:
-        case INT:
-        case FLOAT:
-        case LONG:
-        case NULL:
-          // For types that Avro encodes using one of the above primitives, we assume they are
-          // deterministic.
-          break;
-        default:
-          reportError(context, "Unknown schema type %s may be non-deterministic", schema.getType());
-          break;
-      }
-
-      activeSchemas.remove(schema);
-    }
-
-    private void checkMap(String context, TypeDescriptor<?> type, Schema schema) {
-      if (!isSubtypeOf(type, SortedMap.class)) {
-        reportError(context, "%s may not be deterministically ordered", type);
-      }
-
-      // Avro (currently) asserts that all keys are strings.
-      // In case that changes, we double check that the key was a string:
-      Class<?> keyType = type.resolveType(Map.class.getTypeParameters()[0]).getRawType();
-      if (!String.class.equals(keyType)) {
-        reportError(context, "map keys should be Strings, but was %s", keyType);
-      }
-
-      recurse(context,
-          type.resolveType(Map.class.getTypeParameters()[1]),
-          schema.getValueType());
-    }
-
-    private void checkArray(String context, TypeDescriptor<?> type, Schema schema) {
-      TypeDescriptor<?> elementType = null;
-      if (type.isArray()) {
-        // The type is an array (with ordering)-> deterministic iff the element is deterministic.
-        elementType = type.getComponentType();
-      } else if (isSubtypeOf(type, Collection.class)) {
-        if (isSubtypeOf(type, List.class, SortedSet.class)) {
-          // Ordered collection -> deterministic iff the element is deterministic
-          elementType = type.resolveType(Collection.class.getTypeParameters()[0]);
-        } else {
-          // Not an ordered collection -> not deterministic
-          reportError(context, "%s may not be deterministically ordered", type);
-          return;
-        }
-      } else {
-        // If it was an unknown type encoded as an array, be conservative and assume
-        // that we don't know anything about the order.
-        reportError(context, "encoding %s as an ARRAY was unexpected");
-        return;
-      }
-
-      // If we get here, it's either a deterministically-ordered Collection, or
-      // an array. Either way, the type is deterministic iff the element type is
-      // deterministic.
-      recurse(context, elementType, schema.getElementType());
-    }
-
-    /**
-     * Extract a field from a class. We need to look at the declared fields so that we can
-     * see private fields. We may need to walk up to the parent to get classes from the parent.
-     */
-    private static Field getField(Class<?> clazz, String name) {
-      while (clazz != null) {
-        for (Field field : clazz.getDeclaredFields()) {
-          AvroName avroName = field.getAnnotation(AvroName.class);
-          if (avroName != null && name.equals(avroName.value())) {
-            return field;
-          } else if (avroName == null && name.equals(field.getName())) {
-            return field;
-          }
-        }
-        clazz = clazz.getSuperclass();
-      }
-
-      throw new IllegalArgumentException(
-          "Unable to get field " + name + " from class " + clazz);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianIntegerCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianIntegerCoder.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianIntegerCoder.java
deleted file mode 100644
index aea6dd5..0000000
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianIntegerCoder.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.dataflow.sdk.coders;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UTFDataFormatException;
-
-/**
- * A {@link BigEndianIntegerCoder} encodes {@link Integer Integers} in 4 bytes, big-endian.
- */
-public class BigEndianIntegerCoder extends AtomicCoder<Integer> {
-
-  @JsonCreator
-  public static BigEndianIntegerCoder of() {
-    return INSTANCE;
-  }
-
-  /////////////////////////////////////////////////////////////////////////////
-
-  private static final BigEndianIntegerCoder INSTANCE = new BigEndianIntegerCoder();
-
-  private BigEndianIntegerCoder() {}
-
-  @Override
-  public void encode(Integer value, OutputStream outStream, Context context)
-      throws IOException, CoderException {
-    if (value == null) {
-      throw new CoderException("cannot encode a null Integer");
-    }
-    new DataOutputStream(outStream).writeInt(value);
-  }
-
-  @Override
-  public Integer decode(InputStream inStream, Context context)
-      throws IOException, CoderException {
-    try {
-      return new DataInputStream(inStream).readInt();
-    } catch (EOFException | UTFDataFormatException exn) {
-      // These exceptions correspond to decoding problems, so change
-      // what kind of exception they're branded as.
-      throw new CoderException(exn);
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code true}. This coder is injective.
-   */
-  @Override
-  public boolean consistentWithEquals() {
-    return true;
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code true}, because {@link #getEncodedElementByteSize} runs in constant time.
-   */
-  @Override
-  public boolean isRegisterByteSizeObserverCheap(Integer value, Context context) {
-    return true;
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code 4}, the size in bytes of an integer's big endian encoding.
-   */
-  @Override
-  protected long getEncodedElementByteSize(Integer value, Context context)
-      throws Exception {
-    if (value == null) {
-      throw new CoderException("cannot encode a null Integer");
-    }
-    return 4;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianLongCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianLongCoder.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianLongCoder.java
deleted file mode 100644
index a0325ac..0000000
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/BigEndianLongCoder.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.dataflow.sdk.coders;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UTFDataFormatException;
-
-/**
- * A {@link BigEndianLongCoder} encodes {@link Long}s in 8 bytes, big-endian.
- */
-public class BigEndianLongCoder extends AtomicCoder<Long> {
-
-  @JsonCreator
-  public static BigEndianLongCoder of() {
-    return INSTANCE;
-  }
-
-  /////////////////////////////////////////////////////////////////////////////
-
-  private static final BigEndianLongCoder INSTANCE = new BigEndianLongCoder();
-
-  private BigEndianLongCoder() {}
-
-  @Override
-  public void encode(Long value, OutputStream outStream, Context context)
-      throws IOException, CoderException {
-    if (value == null) {
-      throw new CoderException("cannot encode a null Long");
-    }
-    new DataOutputStream(outStream).writeLong(value);
-  }
-
-  @Override
-  public Long decode(InputStream inStream, Context context)
-      throws IOException, CoderException {
-    try {
-      return new DataInputStream(inStream).readLong();
-    } catch (EOFException | UTFDataFormatException exn) {
-      // These exceptions correspond to decoding problems, so change
-      // what kind of exception they're branded as.
-      throw new CoderException(exn);
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code true}. This coder is injective.
-   */
-  @Override
-  public boolean consistentWithEquals() {
-    return true;
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code true}, since {@link #getEncodedElementByteSize} returns a constant.
-   */
-  @Override
-  public boolean isRegisterByteSizeObserverCheap(Long value, Context context) {
-    return true;
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code 8}, the byte size of a big-endian encoded {@code Long}.
-   */
-  @Override
-  protected long getEncodedElementByteSize(Long value, Context context)
-      throws Exception {
-    if (value == null) {
-      throw new CoderException("cannot encode a null Long");
-    }
-    return 8;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0393a791/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/ByteArrayCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/ByteArrayCoder.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/ByteArrayCoder.java
deleted file mode 100644
index 6b00a0e..0000000
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/coders/ByteArrayCoder.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.dataflow.sdk.coders;
-
-import com.google.cloud.dataflow.sdk.util.ExposedByteArrayOutputStream;
-import com.google.cloud.dataflow.sdk.util.StreamUtils;
-import com.google.cloud.dataflow.sdk.util.VarInt;
-import com.google.common.io.ByteStreams;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-/**
- * A {@link Coder} for {@code byte[]}.
- *
- * <p>The encoding format is as follows:
- * <ul>
- * <li>If in a non-nested context (the {@code byte[]} is the only value in the stream), the
- * bytes are read/written directly.</li>
- * <li>If in a nested context, the bytes are prefixed with the length of the array,
- * encoded via a {@link VarIntCoder}.</li>
- * </ul>
- */
-public class ByteArrayCoder extends AtomicCoder<byte[]> {
-
-  @JsonCreator
-  public static ByteArrayCoder of() {
-    return INSTANCE;
-  }
-
-
-  /////////////////////////////////////////////////////////////////////////////
-
-  private static final ByteArrayCoder INSTANCE = new ByteArrayCoder();
-
-  private ByteArrayCoder() {}
-
-  @Override
-  public void encode(byte[] value, OutputStream outStream, Context context)
-      throws IOException, CoderException {
-    if (value == null) {
-      throw new CoderException("cannot encode a null byte[]");
-    }
-    if (!context.isWholeStream) {
-      VarInt.encode(value.length, outStream);
-      outStream.write(value);
-    } else {
-      outStream.write(value);
-    }
-  }
-
-  /**
-   * Encodes the provided {@code value} with the identical encoding to {@link #encode}, but with
-   * optimizations that take ownership of the value.
-   *
-   * <p>Once passed to this method, {@code value} should never be observed or mutated again.
-   */
-  public void encodeAndOwn(byte[] value, OutputStream outStream, Context context)
-      throws IOException, CoderException {
-    if (!context.isWholeStream) {
-      VarInt.encode(value.length, outStream);
-      outStream.write(value);
-    } else {
-      if (outStream instanceof ExposedByteArrayOutputStream) {
-        ((ExposedByteArrayOutputStream) outStream).writeAndOwn(value);
-      } else {
-        outStream.write(value);
-      }
-    }
-  }
-
-  @Override
-  public byte[] decode(InputStream inStream, Context context)
-      throws IOException, CoderException {
-    if (context.isWholeStream) {
-      return StreamUtils.getBytes(inStream);
-    } else {
-      int length = VarInt.decodeInt(inStream);
-      if (length < 0) {
-        throw new IOException("invalid length " + length);
-      }
-      byte[] value = new byte[length];
-      ByteStreams.readFully(inStream, value);
-      return value;
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return objects that are equal if the two arrays contain the same bytes.
-   */
-  @Override
-  public Object structuralValue(byte[] value) {
-    return new StructuralByteArray(value);
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code true} since {@link #getEncodedElementByteSize} runs in
-   * constant time using the {@code length} of the provided array.
-   */
-  @Override
-  public boolean isRegisterByteSizeObserverCheap(byte[] value, Context context) {
-    return true;
-  }
-
-  @Override
-  protected long getEncodedElementByteSize(byte[] value, Context context)
-      throws Exception {
-    if (value == null) {
-      throw new CoderException("cannot encode a null byte[]");
-    }
-    long size = 0;
-    if (!context.isWholeStream) {
-      size += VarInt.getLength(value.length);
-    }
-    return size + value.length;
-  }
-}