You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ib...@apache.org on 2020/04/08 19:35:07 UTC
[beam] branch master updated: [BEAM-9714] [Go SDK] Require --region
flag in Dataflow runner.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c367536 [BEAM-9714] [Go SDK] Require --region flag in Dataflow runner.
new 76d7732 Merge pull request #11348 from ibzib/beam-9199-go
c367536 is described below
commit c367536269a59515e4881771712a118c9602b32a
Author: Kyle Weaver <kc...@google.com>
AuthorDate: Fri Apr 3 13:44:29 2020 -0400
[BEAM-9714] [Go SDK] Require --region flag in Dataflow runner.
---
sdks/go/README.md | 2 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 7 ++-----
sdks/go/test/run_integration_tests.sh | 7 +++++++
website/src/get-started/quickstart-go.md | 1 +
website/src/get-started/wordcount-example.md | 2 ++
5 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/sdks/go/README.md b/sdks/go/README.md
index f2d56d4..2d6faac 100644
--- a/sdks/go/README.md
+++ b/sdks/go/README.md
@@ -87,7 +87,7 @@ purse: 6
To run wordcount on dataflow runner do:
```
-$ go run wordcount.go --runner=dataflow --project=<YOUR_GCP_PROJECT> --staging_location=<YOUR_GCS_LOCATION>/staging --worker_harness_container_image=<YOUR_SDK_HARNESS_IMAGE_LOCATION> --output=<YOUR_GCS_LOCATION>/output
+$ go run wordcount.go --runner=dataflow --project=<YOUR_GCP_PROJECT> --region=<YOUR_GCP_REGION> --staging_location=<YOUR_GCS_LOCATION>/staging --worker_harness_container_image=<YOUR_SDK_HARNESS_IMAGE_LOCATION> --output=<YOUR_GCS_LOCATION>/output
```
The output is a GCS file in this case:
diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflow.go b/sdks/go/pkg/beam/runners/dataflow/dataflow.go
index 4543a91..8681ceb 100644
--- a/sdks/go/pkg/beam/runners/dataflow/dataflow.go
+++ b/sdks/go/pkg/beam/runners/dataflow/dataflow.go
@@ -53,7 +53,7 @@ var (
maxNumWorkers = flag.Int64("max_num_workers", 0, "Maximum number of workers during scaling (optional).")
autoscalingAlgorithm = flag.String("autoscaling_algorithm", "", "Autoscaling mode to use (optional).")
zone = flag.String("zone", "", "GCP zone (optional)")
- region = flag.String("region", "", "GCP Region (optional but encouraged)")
+ region = flag.String("region", "", "GCP Region (required)")
network = flag.String("network", "", "GCP network (optional)")
subnetwork = flag.String("subnetwork", "", "GCP subnetwork (optional)")
noUsePublicIPs = flag.Bool("no_use_public_ips", false, "Workers must not use public IP addresses (optional)")
@@ -93,10 +93,7 @@ func Execute(ctx context.Context, p *beam.Pipeline) error {
return errors.New("no GCS staging location specified. Use --staging_location=gs://<bucket>/<path>")
}
if *region == "" {
- *region = "us-central1"
- log.Warn(ctx, "--region not set; will default to us-central1. Future releases of Beam will "+
- "require the user to set the region explicitly. "+
- "https://cloud.google.com/compute/docs/regions-zones/regions-zones")
+ return errors.New("No Google Cloud region specified. Use --region=<region>. See https://cloud.google.com/dataflow/docs/concepts/regional-endpoints")
}
if *image == "" {
*image = getContainerImage(ctx)
diff --git a/sdks/go/test/run_integration_tests.sh b/sdks/go/test/run_integration_tests.sh
index 78aef1a..d07e33d 100755
--- a/sdks/go/test/run_integration_tests.sh
+++ b/sdks/go/test/run_integration_tests.sh
@@ -36,6 +36,7 @@ GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
# Project for the container and integration test
PROJECT=apache-beam-testing
DATAFLOW_PROJECT=apache-beam-testing
+REGION=us-central1
# Number of tests to run in parallel
PARALLEL=10
@@ -54,6 +55,11 @@ case $key in
shift # past argument
shift # past value
;;
+ --region)
+ REGION="$2"
+ shift # past argument
+ shift # past value
+ ;;
--dataflow_project)
DATAFLOW_PROJECT="$2"
shift # past argument
@@ -174,6 +180,7 @@ echo ">>> RUNNING $RUNNER INTEGRATION TESTS"
./sdks/go/build/bin/integration \
--runner=$RUNNER \
--project=$DATAFLOW_PROJECT \
+ --region=$REGION \
--environment_type=DOCKER \
--environment_config=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
diff --git a/website/src/get-started/quickstart-go.md b/website/src/get-started/quickstart-go.md
index ac3fcb4..c25b72c 100644
--- a/website/src/get-started/quickstart-go.md
+++ b/website/src/get-started/quickstart-go.md
@@ -69,6 +69,7 @@ $ wordcount --input gs://dataflow-samples/shakespeare/kinglear.txt \
--output gs://<your-gcs-bucket>/counts \
--runner dataflow \
--project your-gcp-project \
+ --region your-gcp-region \
--temp_location gs://<your-gcs-bucket>/tmp/ \
--staging_location gs://<your-gcs-bucket>/binaries/ \
--worker_harness_container_image=apache/beam_go_sdk:latest
diff --git a/website/src/get-started/wordcount-example.md b/website/src/get-started/wordcount-example.md
index 547111d..ffa605e 100644
--- a/website/src/get-started/wordcount-example.md
+++ b/website/src/get-started/wordcount-example.md
@@ -497,6 +497,7 @@ $ wordcount --input gs://dataflow-samples/shakespeare/kinglear.txt \
--output gs://<your-gcs-bucket>/counts \
--runner dataflow \
--project your-gcp-project \
+ --region your-gcp-region \
--temp_location gs://<your-gcs-bucket>/tmp/ \
--staging_location gs://<your-gcs-bucket>/binaries/ \
--worker_harness_container_image=apache/beam_go_sdk:latest
@@ -860,6 +861,7 @@ $ debugging_wordcount --input gs://dataflow-samples/shakespeare/kinglear.txt \
--output gs://<your-gcs-bucket>/counts \
--runner dataflow \
--project your-gcp-project \
+ --region your-gcp-region \
--temp_location gs://<your-gcs-bucket>/tmp/ \
--staging_location gs://<your-gcs-bucket>/binaries/ \
--worker_harness_container_image=apache-docker-beam-snapshots-docker.bintray.io/beam/go:20180515