You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2016/04/22 07:36:52 UTC
[1/3] incubator-beam git commit: Fix how this SDK identifies itself
to the Cloud Dataflow service
Repository: incubator-beam
Updated Branches:
refs/heads/master 10e628471 -> 7465edb73
Fix how this SDK identifies itself to the Cloud Dataflow service
* Use Beam SDK naming, instead of Dataflow SDK.
* Fix version lookup, so it correctly identifies the version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f55bb1db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f55bb1db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f55bb1db
Branch: refs/heads/master
Commit: f55bb1db78dccd2110459fa84c6bc043f4cd8125
Parents: 10e6284
Author: Davor Bonaci <da...@google.com>
Authored: Wed Apr 20 16:02:46 2016 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Thu Apr 21 22:36:07 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f55bb1db/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java
index 8c096fb..e7d98b7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java
@@ -34,7 +34,7 @@ public final class DataflowReleaseInfo extends GenericJson {
private static final Logger LOG = LoggerFactory.getLogger(DataflowReleaseInfo.class);
private static final String DATAFLOW_PROPERTIES_PATH =
- "/org.apache.beam/sdk/sdk.properties";
+ "/org/apache/beam/sdk/sdk.properties";
private static class LazyInit {
private static final DataflowReleaseInfo INSTANCE =
@@ -48,7 +48,7 @@ public final class DataflowReleaseInfo extends GenericJson {
return LazyInit.INSTANCE;
}
- @Key private String name = "Google Cloud Dataflow Java SDK";
+ @Key private String name = "Apache Beam SDK for Java";
@Key private String version = "Unknown";
/** Provides the SDK name. */
[3/3] incubator-beam git commit: This closes #222
Posted by da...@apache.org.
This closes #222
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7465edb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7465edb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7465edb7
Branch: refs/heads/master
Commit: 7465edb73e5f3db4fa9a4330330540e993c1d4b1
Parents: 10e6284 390c5a8
Author: Davor Bonaci <da...@google.com>
Authored: Thu Apr 21 22:36:16 2016 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Thu Apr 21 22:36:16 2016 -0700
----------------------------------------------------------------------
.../sdk/runners/DataflowPipelineRunner.java | 6 +-
.../sdk/runners/DataflowPipelineRunnerTest.java | 6 +-
.../apache/beam/sdk/io/bigtable/BigtableIO.java | 4 +-
.../beam/sdk/util/DataflowReleaseInfo.java | 88 --------------------
.../org/apache/beam/sdk/util/ReleaseInfo.java | 88 ++++++++++++++++++++
5 files changed, 96 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-beam git commit: Rename DataflowReleaseInfo to
ReleaseInfo
Posted by da...@apache.org.
Rename DataflowReleaseInfo to ReleaseInfo
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/390c5a8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/390c5a8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/390c5a8b
Branch: refs/heads/master
Commit: 390c5a8bf4e5b53175301d8f59317aa2a326c855
Parents: f55bb1d
Author: Davor Bonaci <da...@google.com>
Authored: Thu Apr 21 10:08:53 2016 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Thu Apr 21 22:36:08 2016 -0700
----------------------------------------------------------------------
.../sdk/runners/DataflowPipelineRunner.java | 6 +-
.../sdk/runners/DataflowPipelineRunnerTest.java | 6 +-
.../apache/beam/sdk/io/bigtable/BigtableIO.java | 4 +-
.../beam/sdk/util/DataflowReleaseInfo.java | 88 --------------------
.../org/apache/beam/sdk/util/ReleaseInfo.java | 88 ++++++++++++++++++++
5 files changed, 96 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/390c5a8b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/DataflowPipelineRunner.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/DataflowPipelineRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/DataflowPipelineRunner.java
index 58733f3..c147f02 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/DataflowPipelineRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/DataflowPipelineRunner.java
@@ -89,7 +89,6 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.DataflowReleaseInfo;
import org.apache.beam.sdk.util.DataflowTransport;
import org.apache.beam.sdk.util.IOChannelUtils;
import org.apache.beam.sdk.util.InstanceBuilder;
@@ -97,6 +96,7 @@ import org.apache.beam.sdk.util.MonitoringUtil;
import org.apache.beam.sdk.util.PCollectionViews;
import org.apache.beam.sdk.util.PathValidator;
import org.apache.beam.sdk.util.PropertyNames;
+import org.apache.beam.sdk.util.ReleaseInfo;
import org.apache.beam.sdk.util.Reshuffle;
import org.apache.beam.sdk.util.SystemDoFnInternal;
import org.apache.beam.sdk.util.ValueWithRecordId;
@@ -507,10 +507,10 @@ public class DataflowPipelineRunner extends PipelineRunner<DataflowPipelineJob>
Job newJob = jobSpecification.getJob();
newJob.setClientRequestId(requestId);
- String version = DataflowReleaseInfo.getReleaseInfo().getVersion();
+ String version = ReleaseInfo.getReleaseInfo().getVersion();
System.out.println("Dataflow SDK version: " + version);
- newJob.getEnvironment().setUserAgent(DataflowReleaseInfo.getReleaseInfo());
+ newJob.getEnvironment().setUserAgent(ReleaseInfo.getReleaseInfo());
// The Dataflow Service may write to the temporary directory directly, so
// must be verified.
if (!Strings.isNullOrEmpty(options.getTempLocation())) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/390c5a8b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/runners/DataflowPipelineRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/runners/DataflowPipelineRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/runners/DataflowPipelineRunnerTest.java
index 6949128..8b5cbdb 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/runners/DataflowPipelineRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/runners/DataflowPipelineRunnerTest.java
@@ -69,9 +69,9 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.DataflowReleaseInfo;
import org.apache.beam.sdk.util.GcsUtil;
import org.apache.beam.sdk.util.NoopPathValidator;
+import org.apache.beam.sdk.util.ReleaseInfo;
import org.apache.beam.sdk.util.TestCredential;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
@@ -376,10 +376,10 @@ public class DataflowPipelineRunnerTest {
cloudDataflowDataset,
workflowJob.getEnvironment().getDataset());
assertEquals(
- DataflowReleaseInfo.getReleaseInfo().getName(),
+ ReleaseInfo.getReleaseInfo().getName(),
workflowJob.getEnvironment().getUserAgent().get("name"));
assertEquals(
- DataflowReleaseInfo.getReleaseInfo().getVersion(),
+ ReleaseInfo.getReleaseInfo().getVersion(),
workflowJob.getEnvironment().getUserAgent().get("version"));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/390c5a8b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/bigtable/BigtableIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/bigtable/BigtableIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/bigtable/BigtableIO.java
index b2d9cb3..5177262 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/bigtable/BigtableIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/bigtable/BigtableIO.java
@@ -35,7 +35,7 @@ import org.apache.beam.sdk.io.range.ByteKeyRangeTracker;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.DataflowReleaseInfo;
+import org.apache.beam.sdk.util.ReleaseInfo;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
@@ -978,7 +978,7 @@ public class BigtableIO {
*/
private static String getUserAgent() {
String javaVersion = System.getProperty("java.specification.version");
- DataflowReleaseInfo info = DataflowReleaseInfo.getReleaseInfo();
+ ReleaseInfo info = ReleaseInfo.getReleaseInfo();
return String.format(
"%s/%s (%s); %s",
info.getName(),
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/390c5a8b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java
deleted file mode 100644
index e7d98b7..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DataflowReleaseInfo.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import com.google.api.client.json.GenericJson;
-import com.google.api.client.util.Key;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Properties;
-
-/**
- * Utilities for working with the Dataflow distribution.
- */
-public final class DataflowReleaseInfo extends GenericJson {
- private static final Logger LOG = LoggerFactory.getLogger(DataflowReleaseInfo.class);
-
- private static final String DATAFLOW_PROPERTIES_PATH =
- "/org/apache/beam/sdk/sdk.properties";
-
- private static class LazyInit {
- private static final DataflowReleaseInfo INSTANCE =
- new DataflowReleaseInfo(DATAFLOW_PROPERTIES_PATH);
- }
-
- /**
- * Returns an instance of DataflowReleaseInfo.
- */
- public static DataflowReleaseInfo getReleaseInfo() {
- return LazyInit.INSTANCE;
- }
-
- @Key private String name = "Apache Beam SDK for Java";
- @Key private String version = "Unknown";
-
- /** Provides the SDK name. */
- public String getName() {
- return name;
- }
-
- /** Provides the SDK version. */
- public String getVersion() {
- return version;
- }
-
- private DataflowReleaseInfo(String resourcePath) {
- Properties properties = new Properties();
-
- InputStream in = DataflowReleaseInfo.class.getResourceAsStream(
- DATAFLOW_PROPERTIES_PATH);
- if (in == null) {
- LOG.warn("Dataflow properties resource not found: {}", resourcePath);
- return;
- }
-
- try {
- properties.load(in);
- } catch (IOException e) {
- LOG.warn("Error loading Dataflow properties resource: ", e);
- }
-
- for (String name : properties.stringPropertyNames()) {
- if (name.equals("name")) {
- // We don't allow the properties to override the SDK name.
- continue;
- }
- put(name, properties.getProperty(name));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/390c5a8b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java
new file mode 100644
index 0000000..77289ac
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import com.google.api.client.json.GenericJson;
+import com.google.api.client.util.Key;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+/**
+ * Utilities for working with release information.
+ */
+public final class ReleaseInfo extends GenericJson {
+ private static final Logger LOG = LoggerFactory.getLogger(ReleaseInfo.class);
+
+ private static final String PROPERTIES_PATH =
+ "/org/apache/beam/sdk/sdk.properties";
+
+ private static class LazyInit {
+ private static final ReleaseInfo INSTANCE =
+ new ReleaseInfo(PROPERTIES_PATH);
+ }
+
+ /**
+ * Returns an instance of DataflowReleaseInfo.
+ */
+ public static ReleaseInfo getReleaseInfo() {
+ return LazyInit.INSTANCE;
+ }
+
+ @Key private String name = "Apache Beam SDK for Java";
+ @Key private String version = "Unknown";
+
+ /** Provides the SDK name. */
+ public String getName() {
+ return name;
+ }
+
+ /** Provides the SDK version. */
+ public String getVersion() {
+ return version;
+ }
+
+ private ReleaseInfo(String resourcePath) {
+ Properties properties = new Properties();
+
+ InputStream in = ReleaseInfo.class.getResourceAsStream(
+ PROPERTIES_PATH);
+ if (in == null) {
+ LOG.warn("Dataflow properties resource not found: {}", resourcePath);
+ return;
+ }
+
+ try {
+ properties.load(in);
+ } catch (IOException e) {
+ LOG.warn("Error loading Dataflow properties resource: ", e);
+ }
+
+ for (String name : properties.stringPropertyNames()) {
+ if (name.equals("name")) {
+ // We don't allow the properties to override the SDK name.
+ continue;
+ }
+ put(name, properties.getProperty(name));
+ }
+ }
+}