You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/04/27 03:08:34 UTC
[02/21] incubator-beam git commit: Reorganize Java packages in the
sources of the Google Cloud Dataflow runner
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java
deleted file mode 100644
index 21bc60e..0000000
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/PackageUtilTest.java
+++ /dev/null
@@ -1,484 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
-import org.apache.beam.sdk.options.GcsOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.ExpectedLogs;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
-import org.apache.beam.sdk.util.PackageUtil.PackageAttributes;
-import org.apache.beam.sdk.util.gcsfs.GcsPath;
-
-import com.google.api.client.googleapis.json.GoogleJsonError.ErrorInfo;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.client.http.HttpRequest;
-import com.google.api.client.http.HttpResponse;
-import com.google.api.client.http.HttpStatusCodes;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.LowLevelHttpRequest;
-import com.google.api.client.json.GenericJson;
-import com.google.api.client.json.Json;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.client.testing.http.HttpTesting;
-import com.google.api.client.testing.http.MockHttpTransport;
-import com.google.api.client.testing.http.MockLowLevelHttpRequest;
-import com.google.api.client.testing.http.MockLowLevelHttpResponse;
-import com.google.api.services.dataflow.model.DataflowPackage;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.io.Files;
-import com.google.common.io.LineReader;
-
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.hamcrest.Matchers;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.channels.Channels;
-import java.nio.channels.Pipe;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.regex.Pattern;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-/** Tests for PackageUtil. */
-@RunWith(JUnit4.class)
-public class PackageUtilTest {
- @Rule public ExpectedLogs logged = ExpectedLogs.none(PackageUtil.class);
- @Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Rule
- public FastNanoClockAndSleeper fastNanoClockAndSleeper = new FastNanoClockAndSleeper();
-
- @Mock
- GcsUtil mockGcsUtil;
-
- // 128 bits, base64 encoded is 171 bits, rounds to 22 bytes
- private static final String HASH_PATTERN = "[a-zA-Z0-9+-]{22}";
-
- // Hamcrest matcher to assert a string matches a pattern
- private static class RegexMatcher extends BaseMatcher<String> {
- private final Pattern pattern;
-
- public RegexMatcher(String regex) {
- this.pattern = Pattern.compile(regex);
- }
-
- @Override
- public boolean matches(Object o) {
- if (!(o instanceof String)) {
- return false;
- }
- return pattern.matcher((String) o).matches();
- }
-
- @Override
- public void describeTo(Description description) {
- description.appendText(String.format("matches regular expression %s", pattern));
- }
-
- public static RegexMatcher matches(String regex) {
- return new RegexMatcher(regex);
- }
- }
-
- @Before
- public void setUp() {
- MockitoAnnotations.initMocks(this);
-
- GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
- pipelineOptions.setGcsUtil(mockGcsUtil);
-
- IOChannelUtils.registerStandardIOFactories(pipelineOptions);
- }
-
- private File makeFileWithContents(String name, String contents) throws Exception {
- File tmpFile = tmpFolder.newFile(name);
- Files.write(contents, tmpFile, StandardCharsets.UTF_8);
- tmpFile.setLastModified(0); // required for determinism with directories
- return tmpFile;
- }
-
- static final String STAGING_PATH = GcsPath.fromComponents("somebucket", "base/path").toString();
- private static PackageAttributes makePackageAttributes(File file, String overridePackageName) {
- return PackageUtil.createPackageAttributes(file, STAGING_PATH, overridePackageName);
- }
-
- @Test
- public void testFileWithExtensionPackageNamingAndSize() throws Exception {
- String contents = "This is a test!";
- File tmpFile = makeFileWithContents("file.txt", contents);
- PackageAttributes attr = makePackageAttributes(tmpFile, null);
- DataflowPackage target = attr.getDataflowPackage();
-
- assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt"));
- assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
- assertThat(attr.getSize(), equalTo((long) contents.length()));
- }
-
- @Test
- public void testPackageNamingWithFileNoExtension() throws Exception {
- File tmpFile = makeFileWithContents("file", "This is a test!");
- DataflowPackage target = makePackageAttributes(tmpFile, null).getDataflowPackage();
-
- assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN));
- assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
- }
-
- @Test
- public void testPackageNamingWithDirectory() throws Exception {
- File tmpDirectory = tmpFolder.newFolder("folder");
- DataflowPackage target = makePackageAttributes(tmpDirectory, null).getDataflowPackage();
-
- assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar"));
- assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
- }
-
- @Test
- public void testPackageNamingWithFilesHavingSameContentsAndSameNames() throws Exception {
- File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
- makeFileWithContents("folder1/folderA/sameName", "This is a test!");
- DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
-
- File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
- makeFileWithContents("folder2/folderA/sameName", "This is a test!");
- DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
-
- assertEquals(target1.getName(), target2.getName());
- assertEquals(target1.getLocation(), target2.getLocation());
- }
-
- @Test
- public void testPackageNamingWithFilesHavingSameContentsButDifferentNames() throws Exception {
- File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
- makeFileWithContents("folder1/folderA/uniqueName1", "This is a test!");
- DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
-
- File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
- makeFileWithContents("folder2/folderA/uniqueName2", "This is a test!");
- DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
-
- assertNotEquals(target1.getName(), target2.getName());
- assertNotEquals(target1.getLocation(), target2.getLocation());
- }
-
- @Test
- public void testPackageNamingWithDirectoriesHavingSameContentsButDifferentNames()
- throws Exception {
- File tmpDirectory1 = tmpFolder.newFolder("folder1", "folderA");
- tmpFolder.newFolder("folder1", "folderA", "uniqueName1");
- DataflowPackage target1 = makePackageAttributes(tmpDirectory1, null).getDataflowPackage();
-
- File tmpDirectory2 = tmpFolder.newFolder("folder2", "folderA");
- tmpFolder.newFolder("folder2", "folderA", "uniqueName2");
- DataflowPackage target2 = makePackageAttributes(tmpDirectory2, null).getDataflowPackage();
-
- assertNotEquals(target1.getName(), target2.getName());
- assertNotEquals(target1.getLocation(), target2.getLocation());
- }
-
- @Test
- public void testPackageUploadWithLargeClasspathLogsWarning() throws Exception {
- File tmpFile = makeFileWithContents("file.txt", "This is a test!");
- // all files will be present and cached so no upload needed.
- when(mockGcsUtil.fileSize(any(GcsPath.class))).thenReturn(tmpFile.length());
-
- List<String> classpathElements = Lists.newLinkedList();
- for (int i = 0; i < 1005; ++i) {
- String eltName = "element" + i;
- classpathElements.add(eltName + '=' + tmpFile.getAbsolutePath());
- }
-
- PackageUtil.stageClasspathElements(classpathElements, STAGING_PATH);
-
- logged.verifyWarn("Your classpath contains 1005 elements, which Google Cloud Dataflow");
- }
-
- @Test
- public void testPackageUploadWithFileSucceeds() throws Exception {
- Pipe pipe = Pipe.open();
- String contents = "This is a test!";
- File tmpFile = makeFileWithContents("file.txt", contents);
- when(mockGcsUtil.fileSize(any(GcsPath.class)))
- .thenThrow(new FileNotFoundException("some/path"));
- when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
-
- List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH);
- DataflowPackage target = Iterables.getOnlyElement(targets);
-
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
-
- assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt"));
- assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
- assertThat(new LineReader(Channels.newReader(pipe.source(), "UTF-8")).readLine(),
- equalTo(contents));
- }
-
- @Test
- public void testPackageUploadWithDirectorySucceeds() throws Exception {
- Pipe pipe = Pipe.open();
- File tmpDirectory = tmpFolder.newFolder("folder");
- tmpFolder.newFolder("folder", "empty_directory");
- tmpFolder.newFolder("folder", "directory");
- makeFileWithContents("folder/file.txt", "This is a test!");
- makeFileWithContents("folder/directory/file.txt", "This is also a test!");
-
- when(mockGcsUtil.fileSize(any(GcsPath.class)))
- .thenThrow(new FileNotFoundException("some/path"));
- when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
-
- PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH);
-
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
-
- ZipInputStream inputStream = new ZipInputStream(Channels.newInputStream(pipe.source()));
- List<String> zipEntryNames = new ArrayList<>();
- for (ZipEntry entry = inputStream.getNextEntry(); entry != null;
- entry = inputStream.getNextEntry()) {
- zipEntryNames.add(entry.getName());
- }
-
- assertThat(zipEntryNames,
- containsInAnyOrder("directory/file.txt", "empty_directory/", "file.txt"));
- }
-
- @Test
- public void testPackageUploadWithEmptyDirectorySucceeds() throws Exception {
- Pipe pipe = Pipe.open();
- File tmpDirectory = tmpFolder.newFolder("folder");
-
- when(mockGcsUtil.fileSize(any(GcsPath.class)))
- .thenThrow(new FileNotFoundException("some/path"));
- when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
-
- List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH);
- DataflowPackage target = Iterables.getOnlyElement(targets);
-
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
-
- assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar"));
- assertThat(target.getLocation(), equalTo(STAGING_PATH + '/' + target.getName()));
- assertNull(new ZipInputStream(Channels.newInputStream(pipe.source())).getNextEntry());
- }
-
- @Test(expected = RuntimeException.class)
- public void testPackageUploadFailsWhenIOExceptionThrown() throws Exception {
- File tmpFile = makeFileWithContents("file.txt", "This is a test!");
- when(mockGcsUtil.fileSize(any(GcsPath.class)))
- .thenThrow(new FileNotFoundException("some/path"));
- when(mockGcsUtil.create(any(GcsPath.class), anyString()))
- .thenThrow(new IOException("Fake Exception: Upload error"));
-
- try {
- PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpFile.getAbsolutePath()),
- STAGING_PATH, fastNanoClockAndSleeper);
- } finally {
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil, times(5)).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
- }
- }
-
- @Test
- public void testPackageUploadFailsWithPermissionsErrorGivesDetailedMessage() throws Exception {
- File tmpFile = makeFileWithContents("file.txt", "This is a test!");
- when(mockGcsUtil.fileSize(any(GcsPath.class)))
- .thenThrow(new FileNotFoundException("some/path"));
- when(mockGcsUtil.create(any(GcsPath.class), anyString()))
- .thenThrow(new IOException("Failed to write to GCS path " + STAGING_PATH,
- googleJsonResponseException(
- HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Permission denied", "Test message")));
-
- try {
- PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpFile.getAbsolutePath()),
- STAGING_PATH, fastNanoClockAndSleeper);
- fail("Expected RuntimeException");
- } catch (RuntimeException e) {
- assertTrue("Expected IOException containing detailed message.",
- e.getCause() instanceof IOException);
- assertThat(e.getCause().getMessage(),
- Matchers.allOf(
- Matchers.containsString("Uploaded failed due to permissions error"),
- Matchers.containsString(
- "Stale credentials can be resolved by executing 'gcloud auth login'")));
- } finally {
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
- }
- }
-
- @Test
- public void testPackageUploadEventuallySucceeds() throws Exception {
- Pipe pipe = Pipe.open();
- File tmpFile = makeFileWithContents("file.txt", "This is a test!");
- when(mockGcsUtil.fileSize(any(GcsPath.class)))
- .thenThrow(new FileNotFoundException("some/path"));
- when(mockGcsUtil.create(any(GcsPath.class), anyString()))
- .thenThrow(new IOException("Fake Exception: 410 Gone")) // First attempt fails
- .thenReturn(pipe.sink()); // second attempt succeeds
-
- try {
- PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpFile.getAbsolutePath()),
- STAGING_PATH,
- fastNanoClockAndSleeper);
- } finally {
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil, times(2)).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
- }
- }
-
- @Test
- public void testPackageUploadIsSkippedWhenFileAlreadyExists() throws Exception {
- File tmpFile = makeFileWithContents("file.txt", "This is a test!");
- when(mockGcsUtil.fileSize(any(GcsPath.class))).thenReturn(tmpFile.length());
-
- PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH);
-
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verifyNoMoreInteractions(mockGcsUtil);
- }
-
- @Test
- public void testPackageUploadIsNotSkippedWhenSizesAreDifferent() throws Exception {
- Pipe pipe = Pipe.open();
- File tmpDirectory = tmpFolder.newFolder("folder");
- tmpFolder.newFolder("folder", "empty_directory");
- tmpFolder.newFolder("folder", "directory");
- makeFileWithContents("folder/file.txt", "This is a test!");
- makeFileWithContents("folder/directory/file.txt", "This is also a test!");
- when(mockGcsUtil.fileSize(any(GcsPath.class))).thenReturn(Long.MAX_VALUE);
- when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
-
- PackageUtil.stageClasspathElements(
- ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH);
-
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
- }
-
- @Test
- public void testPackageUploadWithExplicitPackageName() throws Exception {
- Pipe pipe = Pipe.open();
- File tmpFile = makeFileWithContents("file.txt", "This is a test!");
- final String overriddenName = "alias.txt";
-
- when(mockGcsUtil.fileSize(any(GcsPath.class)))
- .thenThrow(new FileNotFoundException("some/path"));
- when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink());
-
- List<DataflowPackage> targets = PackageUtil.stageClasspathElements(
- ImmutableList.of(overriddenName + "=" + tmpFile.getAbsolutePath()), STAGING_PATH);
- DataflowPackage target = Iterables.getOnlyElement(targets);
-
- verify(mockGcsUtil).fileSize(any(GcsPath.class));
- verify(mockGcsUtil).create(any(GcsPath.class), anyString());
- verifyNoMoreInteractions(mockGcsUtil);
-
- assertThat(target.getName(), equalTo(overriddenName));
- assertThat(target.getLocation(),
- RegexMatcher.matches(STAGING_PATH + "/file-" + HASH_PATTERN + ".txt"));
- }
-
- @Test
- public void testPackageUploadIsSkippedWithNonExistentResource() throws Exception {
- String nonExistentFile =
- IOChannelUtils.resolve(tmpFolder.getRoot().getPath(), "non-existent-file");
- assertEquals(Collections.EMPTY_LIST, PackageUtil.stageClasspathElements(
- ImmutableList.of(nonExistentFile), STAGING_PATH));
- }
-
- /**
- * Builds a fake GoogleJsonResponseException for testing API error handling.
- */
- private static GoogleJsonResponseException googleJsonResponseException(
- final int status, final String reason, final String message) throws IOException {
- final JsonFactory jsonFactory = new JacksonFactory();
- HttpTransport transport = new MockHttpTransport() {
- @Override
- public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
- ErrorInfo errorInfo = new ErrorInfo();
- errorInfo.setReason(reason);
- errorInfo.setMessage(message);
- errorInfo.setFactory(jsonFactory);
- GenericJson error = new GenericJson();
- error.set("code", status);
- error.set("errors", Arrays.asList(errorInfo));
- error.setFactory(jsonFactory);
- GenericJson errorResponse = new GenericJson();
- errorResponse.set("error", error);
- errorResponse.setFactory(jsonFactory);
- return new MockLowLevelHttpRequest().setResponse(
- new MockLowLevelHttpResponse().setContent(errorResponse.toPrettyString())
- .setContentType(Json.MEDIA_TYPE).setStatusCode(status));
- }
- };
- HttpRequest request =
- transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL);
- request.setThrowExceptionOnExecuteError(false);
- HttpResponse response = request.execute();
- return GoogleJsonResponseException.from(jsonFactory, response);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index cdd3b84..a51a24e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -56,7 +56,7 @@ import javax.annotation.Nullable;
* <li>System property "beamTestPipelineOptions" must contain a JSON delimited list of pipeline
* options. For example:
* <pre>{@code [
- * "--runner=org.apache.beam.sdk.testing.TestDataflowPipelineRunner",
+ * "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowPipelineRunner",
* "--project=mygcpproject",
* "--stagingLocation=gs://mygcsbucket/path"
* ]}</pre>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java
index 4b2bb7b..3e4fc86 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java
@@ -17,11 +17,11 @@
*/
package ${package};
+import org.apache.beam.runners.dataflow.BlockingDataflowPipelineRunner;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
-import org.apache.beam.sdk.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.runners.BlockingDataflowPipelineRunner;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java
index f997521..7dea9fe 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java
@@ -17,11 +17,8 @@
*/
package ${package};
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.api.services.bigquery.model.TableReference;
-import com.google.api.services.bigquery.model.TableRow;
-import com.google.api.services.bigquery.model.TableSchema;
import ${package}.common.DataflowExampleUtils;
+
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.io.BigQueryIO;
@@ -37,6 +34,11 @@ import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
+
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java
index 8c89283..fc1f4b5 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java
@@ -17,9 +17,9 @@
*/
package ${package};
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
-import org.apache.beam.sdk.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.DefaultValueFactory;
import org.apache.beam.sdk.options.Description;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleOptions.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleOptions.java
index 9e7136d..e3bf7c5 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleOptions.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleOptions.java
@@ -17,7 +17,7 @@
*/
package ${package}.common;
-import org.apache.beam.sdk.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java
index 0ad449a..4914d4c 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java
@@ -17,6 +17,11 @@
*/
package ${package}.common;
+import org.apache.beam.runners.dataflow.DataflowPipelineJob;
+import org.apache.beam.runners.dataflow.DataflowPipelineRunner;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.util.MonitoringUtil;
+
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.services.AbstractGoogleClientRequest;
import com.google.api.services.bigquery.Bigquery;
@@ -34,12 +39,8 @@ import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.BigQueryOptions;
-import org.apache.beam.sdk.options.DataflowPipelineOptions;
-import org.apache.beam.sdk.runners.DataflowPipelineJob;
-import org.apache.beam.sdk.runners.DataflowPipelineRunner;
import org.apache.beam.sdk.runners.DirectPipelineRunner;
import org.apache.beam.sdk.transforms.IntraBundleParallelization;
-import org.apache.beam.sdk.util.MonitoringUtil;
import org.apache.beam.sdk.util.Transport;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExampleBigQueryTableOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExampleBigQueryTableOptions.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExampleBigQueryTableOptions.java
index 9e65c4f..279f2e0 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExampleBigQueryTableOptions.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExampleBigQueryTableOptions.java
@@ -17,13 +17,14 @@
*/
package ${package}.common;
-import com.google.api.services.bigquery.model.TableSchema;
-import org.apache.beam.sdk.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.DefaultValueFactory;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptions;
+import com.google.api.services.bigquery.model.TableSchema;
+
/**
* Options that can be used to configure BigQuery tables in Dataflow examples.
* The project defaults to the project being used to run the example.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExamplePubsubTopicOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExamplePubsubTopicOptions.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExamplePubsubTopicOptions.java
index b5acf69..8a7c9cf 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExamplePubsubTopicOptions.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/ExamplePubsubTopicOptions.java
@@ -17,7 +17,7 @@
*/
package ${package}.common;
-import org.apache.beam.sdk.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.DefaultValueFactory;
import org.apache.beam.sdk.options.Description;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02190985/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/PubsubFileInjector.java
----------------------------------------------------------------------
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/PubsubFileInjector.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/PubsubFileInjector.java
index d8bab1f..5c182b2 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/PubsubFileInjector.java
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/PubsubFileInjector.java
@@ -17,12 +17,9 @@
*/
package ${package}.common;
-import com.google.api.services.pubsub.Pubsub;
-import com.google.api.services.pubsub.model.PublishRequest;
-import com.google.api.services.pubsub.model.PubsubMessage;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
-import org.apache.beam.sdk.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -30,6 +27,10 @@ import org.apache.beam.sdk.options.Validation;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.IntraBundleParallelization;
import org.apache.beam.sdk.util.Transport;
+
+import com.google.api.services.pubsub.Pubsub;
+import com.google.api.services.pubsub.model.PublishRequest;
+import com.google.api.services.pubsub.model.PubsubMessage;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;