You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/08/19 18:57:00 UTC
[1/2] incubator-beam git commit: Incorporate private IP option for
Dataflow runner
Repository: incubator-beam
Updated Branches:
refs/heads/master 8d31ca0ca -> 6645dcd4a
Incorporate private IP option for Dataflow runner
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ac7e6f20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ac7e6f20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ac7e6f20
Branch: refs/heads/master
Commit: ac7e6f202812cd54ba72dd332c5bf7c2ace4c989
Parents: 8d31ca0
Author: sammcveety <sa...@gmail.com>
Authored: Thu Aug 18 18:02:06 2016 -0400
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Aug 19 11:56:42 2016 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
.../runners/dataflow/DataflowPipelineTranslator.java | 7 +++++++
.../options/DataflowPipelineWorkerPoolOptions.java | 14 ++++++++++++++
3 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ac7e6f20/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f39f94a..58adbe7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -105,7 +105,7 @@
<bigquery.version>v2-rev295-1.22.0</bigquery.version>
<pubsubgrpc.version>0.0.2</pubsubgrpc.version>
<clouddebugger.version>v2-rev8-1.22.0</clouddebugger.version>
- <dataflow.version>v1b3-rev30-1.22.0</dataflow.version>
+ <dataflow.version>v1b3-rev36-1.22.0</dataflow.version>
<dataflow.proto.version>0.5.160222</dataflow.proto.version>
<datastore.client.version>1.0.0-beta.2</datastore.client.version>
<datastore.proto.version>1.0.0-beta</datastore.proto.version>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ac7e6f20/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 0ca92c8..b8d9445 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -425,6 +425,13 @@ public class DataflowPipelineTranslator {
workerPool.setMachineType(options.getWorkerMachineType());
}
+ if (options.getUsePublicIps() != null) {
+ if (options.getUsePublicIps()) {
+ workerPool.setIpConfiguration("WORKER_IP_PUBLIC");
+ } else {
+ workerPool.setIpConfiguration("WORKER_IP_PRIVATE");
+ }
+ }
workerPool.setPackages(packages);
workerPool.setNumWorkers(options.getNumWorkers());
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ac7e6f20/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java
index e3a1a0f..9b7a9ce 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java
@@ -29,6 +29,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.List;
+import javax.annotation.Nullable;
+
/**
* Options that are used to configure the Dataflow pipeline worker pool.
*/
@@ -260,4 +262,16 @@ public interface DataflowPipelineWorkerPoolOptions extends PipelineOptions {
+ "https://cloud.google.com/compute/docs/reference/latest/diskTypes")
String getWorkerDiskType();
void setWorkerDiskType(String value);
+
+ /**
+ * Specifies whether worker pools should be started with public IP addresses.
+ *
+ * <p>WARNING: This feature is experimental. You must be whitelisted to use it.
+ */
+ @Description("Specifies whether worker pools should be started with public IP addresses. WARNING:"
+ + "This feature is experimental. You must be whitelisted to use it.")
+ @Experimental
+ @JsonIgnore
+ @Nullable Boolean getUsePublicIps();
+ void setUsePublicIps(@Nullable Boolean value);
}
[2/2] incubator-beam git commit: Closes #850
Posted by dh...@apache.org.
Closes #850
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6645dcd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6645dcd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6645dcd4
Branch: refs/heads/master
Commit: 6645dcd4acc1698e3a83ba99709e55a7933e64d8
Parents: 8d31ca0 ac7e6f2
Author: Dan Halperin <dh...@google.com>
Authored: Fri Aug 19 11:56:48 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Aug 19 11:56:48 2016 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
.../runners/dataflow/DataflowPipelineTranslator.java | 7 +++++++
.../options/DataflowPipelineWorkerPoolOptions.java | 14 ++++++++++++++
3 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------