You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2016/04/08 23:32:33 UTC
[1/3] incubator-beam git commit: [BEAM-151] Update Dataflow worker
container label
Repository: incubator-beam
Updated Branches:
refs/heads/master 5d78420bf -> a43f9b820
[BEAM-151] Update Dataflow worker container label
This new worker container label includes the change required
to decouple the worker from depending on the deleted code
in PipelineOptionsFactory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e74194e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e74194e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e74194e7
Branch: refs/heads/master
Commit: e74194e71fb94b915eb7080c6037127caffbbc05
Parents: 02ee745
Author: Luke Cwik <lc...@google.com>
Authored: Fri Apr 8 10:14:12 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Apr 8 14:27:36 2016 -0700
----------------------------------------------------------------------
.../cloud/dataflow/sdk/runners/DataflowPipelineRunner.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e74194e7/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/runners/DataflowPipelineRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/runners/DataflowPipelineRunner.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/runners/DataflowPipelineRunner.java
index 50ca36f..a154848 100644
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/runners/DataflowPipelineRunner.java
+++ b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/runners/DataflowPipelineRunner.java
@@ -212,9 +212,9 @@ public class DataflowPipelineRunner extends PipelineRunner<DataflowPipelineJob>
// Default Docker container images that execute Dataflow worker harness, residing in Google
// Container Registry, separately for Batch and Streaming.
public static final String BATCH_WORKER_HARNESS_CONTAINER_IMAGE
- = "dataflow.gcr.io/v1beta3/java-batch:1.5.0";
+ = "dataflow.gcr.io/v1beta3/java-batch:1.5.1";
public static final String STREAMING_WORKER_HARNESS_CONTAINER_IMAGE
- = "dataflow.gcr.io/v1beta3/java-streaming:1.5.0";
+ = "dataflow.gcr.io/v1beta3/java-streaming:1.5.1";
// The limit of CreateJob request size.
private static final int CREATE_JOB_REQUEST_LIMIT_BYTES = 10 * 1024 * 1024;
[3/3] incubator-beam git commit: [BEAM-151] This closes #78
Posted by lc...@apache.org.
[BEAM-151] This closes #78
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a43f9b82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a43f9b82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a43f9b82
Branch: refs/heads/master
Commit: a43f9b82027b8ac8761e1558bf2ba151d47711a5
Parents: 5d78420 e74194e
Author: Luke Cwik <lc...@google.com>
Authored: Fri Apr 8 14:28:23 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Apr 8 14:28:23 2016 -0700
----------------------------------------------------------------------
.../sdk/options/PipelineOptionsFactory.java | 48 --------------------
.../sdk/runners/DataflowPipelineRunner.java | 4 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 17 -------
3 files changed, 2 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-beam git commit: [BEAM-151] Clean up reference to
DataflowWorkerHarnessOptions
Posted by lc...@apache.org.
[BEAM-151] Clean up reference to DataflowWorkerHarnessOptions
This prevents moving DataflowWorkerHarnessOptions to Dataflow Runner maven
module.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/02ee7457
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/02ee7457
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/02ee7457
Branch: refs/heads/master
Commit: 02ee74579171a8e47909b23425dcd906f22786cd
Parents: 5d78420
Author: Luke Cwik <lc...@google.com>
Authored: Fri Mar 25 15:28:43 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Apr 8 14:27:36 2016 -0700
----------------------------------------------------------------------
.../sdk/options/PipelineOptionsFactory.java | 48 --------------------
.../sdk/options/PipelineOptionsFactoryTest.java | 17 -------
2 files changed, 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02ee7457/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactory.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactory.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactory.java
index 988d346..dac7726 100644
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactory.java
+++ b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactory.java
@@ -831,54 +831,6 @@ public class PipelineOptionsFactory {
}
/**
- * Creates a set of Dataflow worker harness options based of a set of known system
- * properties. This is meant to only be used from the Dataflow worker harness as a method to
- * bootstrap the worker harness.
- *
- * <p>For internal use only.
- *
- * @return A {@link DataflowWorkerHarnessOptions} object configured for the
- * Dataflow worker harness.
- */
- public static DataflowWorkerHarnessOptions createFromSystemPropertiesInternal()
- throws IOException {
- return createFromSystemProperties();
- }
-
- /**
- * Creates a set of {@link DataflowWorkerHarnessOptions} based of a set of known system
- * properties. This is meant to only be used from the Dataflow worker harness as a method to
- * bootstrap the worker harness.
- *
- * @return A {@link DataflowWorkerHarnessOptions} object configured for the
- * Dataflow worker harness.
- * @deprecated for internal use only
- */
- @Deprecated
- public static DataflowWorkerHarnessOptions createFromSystemProperties() throws IOException {
- ObjectMapper objectMapper = new ObjectMapper();
- DataflowWorkerHarnessOptions options;
- if (System.getProperties().containsKey("sdk_pipeline_options")) {
- String serializedOptions = System.getProperty("sdk_pipeline_options");
- LOG.info("Worker harness starting with: " + serializedOptions);
- options = objectMapper.readValue(serializedOptions, PipelineOptions.class)
- .as(DataflowWorkerHarnessOptions.class);
- } else {
- options = PipelineOptionsFactory.as(DataflowWorkerHarnessOptions.class);
- }
-
- // These values will not be known at job submission time and must be provided.
- if (System.getProperties().containsKey("worker_id")) {
- options.setWorkerId(System.getProperty("worker_id"));
- }
- if (System.getProperties().containsKey("job_id")) {
- options.setJobId(System.getProperty("job_id"));
- }
-
- return options;
- }
-
- /**
* This method is meant to emulate the behavior of {@link Introspector#getBeanInfo(Class, int)}
* to construct the list of {@link PropertyDescriptor}.
*
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/02ee7457/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactoryTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactoryTest.java b/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactoryTest.java
index d8ba8e3..6ba1e00 100644
--- a/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactoryTest.java
+++ b/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/options/PipelineOptionsFactoryTest.java
@@ -78,23 +78,6 @@ public class PipelineOptionsFactoryTest {
}
@Test
- public void testCreationFromSystemProperties() throws Exception {
- System.getProperties().putAll(ImmutableMap
- .<String, String>builder()
- .put("worker_id", "test_worker_id")
- .put("job_id", "test_job_id")
- // Set a non-default value for testing
- .put("sdk_pipeline_options", "{\"options\":{\"numWorkers\":999}}")
- .build());
-
- @SuppressWarnings("deprecation") // testing deprecated functionality
- DataflowWorkerHarnessOptions options = PipelineOptionsFactory.createFromSystemProperties();
- assertEquals("test_worker_id", options.getWorkerId());
- assertEquals("test_job_id", options.getJobId());
- assertEquals(999, options.getNumWorkers());
- }
-
- @Test
public void testAppNameIsSet() {
ApplicationNameOptions options = PipelineOptionsFactory.as(ApplicationNameOptions.class);
assertEquals(PipelineOptionsFactoryTest.class.getSimpleName(), options.getAppName());