You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2017/05/05 23:20:33 UTC
[1/2] beam git commit: Fix default temp location for DataflowRunner
Repository: beam
Updated Branches:
refs/heads/master 060889701 -> e9190834b
Fix default temp location for DataflowRunner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5fdb4055
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5fdb4055
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5fdb4055
Branch: refs/heads/master
Commit: 5fdb40554c7625a6e1ecfc5a8516885d112ae1d0
Parents: 0608897
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Thu May 4 18:30:35 2017 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Fri May 5 16:20:17 2017 -0700
----------------------------------------------------------------------
.../beam/runners/dataflow/options/DataflowPipelineOptions.java | 4 ++++
.../org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java | 4 ++--
.../apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java | 2 +-
3 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 11618af..4af420b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -33,6 +33,8 @@ import org.apache.beam.sdk.options.Hidden;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.options.Validation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Options that can be used to configure the {@link DataflowRunner}.
@@ -117,10 +119,12 @@ public interface DataflowPipelineOptions
* Returns a default staging location under {@link GcpOptions#getGcpTempLocation}.
*/
class StagingLocationFactory implements DefaultValueFactory<String> {
+ private static final Logger LOG = LoggerFactory.getLogger(StagingLocationFactory.class);
@Override
public String create(PipelineOptions options) {
GcsOptions gcsOptions = options.as(GcsOptions.class);
+ LOG.info("No stagingLocation provided, falling back to gcpTempLocation");
String gcpTempLocation;
try {
gcpTempLocation = gcsOptions.getGcpTempLocation();
http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index 126b795..a4128e8 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -278,7 +278,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
}
final String bucketName =
"dataflow-staging-" + region + "-" + projectNumber;
- LOG.info("No staging location provided, attempting to use default bucket: {}",
+ LOG.info("No tempLocation specified, attempting to use default bucket: {}",
bucketName);
Bucket bucket = new Bucket()
.setName(bucketName)
@@ -306,7 +306,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
throw new RuntimeException(
"Unable to determine the owner of the default bucket at gs://" + bucketName, e);
}
- return "gs://" + bucketName;
+ return "gs://" + bucketName + "/temp/";
}
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
index 68b3818..3cf6980 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
@@ -220,7 +220,7 @@ public class GcpOptionsTest {
when(mockGcsUtil.bucketOwner(any(GcsPath.class))).thenReturn(1L);
String bucket = GcpTempLocationFactory.tryCreateDefaultBucket(options, mockCrmClient);
- assertEquals("gs://dataflow-staging-us-north1-1", bucket);
+ assertEquals("gs://dataflow-staging-us-north1-1/temp/", bucket);
}
@Test
[2/2] beam git commit: This closes #2907
Posted by da...@apache.org.
This closes #2907
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/e9190834
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/e9190834
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/e9190834
Branch: refs/heads/master
Commit: e9190834b3cac20b8f61ff7e300f1126f8e087f1
Parents: 0608897 5fdb405
Author: Davor Bonaci <da...@google.com>
Authored: Fri May 5 16:20:28 2017 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Fri May 5 16:20:28 2017 -0700
----------------------------------------------------------------------
.../beam/runners/dataflow/options/DataflowPipelineOptions.java | 4 ++++
.../org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java | 4 ++--
.../apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java | 2 +-
3 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------