You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2017/05/11 17:22:19 UTC
[1/2] beam git commit: Remove protobuf and http-client dependency
from runners/google-cloud-dataflow
Repository: beam
Updated Branches:
refs/heads/master f5ace5c6a -> ff589c711
Remove protobuf and http-client dependency from
runners/google-cloud-dataflow
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/aee91877
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/aee91877
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/aee91877
Branch: refs/heads/master
Commit: aee9187746a1dc0f10865e803a9eb57bdf296686
Parents: f5ace5c
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Wed May 10 17:39:01 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Thu May 11 10:21:47 2017 -0700
----------------------------------------------------------------------
runners/google-cloud-dataflow-java/pom.xml | 11 -----------
.../beam/runners/dataflow/internal/CustomSources.java | 12 ------------
2 files changed, 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/aee91877/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index f679976..073cee8 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -294,12 +294,6 @@
</dependency>
<dependency>
- <groupId>com.google.http-client</groupId>
- <artifactId>google-http-client-protobuf</artifactId>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-dataflow</artifactId>
</dependency>
@@ -353,11 +347,6 @@
</dependency>
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/aee91877/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
index 0d93566..ad2a666 100755
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
@@ -25,7 +25,6 @@ import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray;
import com.google.api.services.dataflow.model.SourceMetadata;
import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ByteString;
import java.util.ArrayList;
import java.util.List;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
@@ -47,20 +46,9 @@ import org.slf4j.LoggerFactory;
public class CustomSources {
private static final String SERIALIZED_SOURCE = "serialized_source";
@VisibleForTesting static final String SERIALIZED_SOURCE_SPLITS = "serialized_source_splits";
- /**
- * The current limit on the size of a ReportWorkItemStatus RPC to Google Cloud Dataflow, which
- * includes the initial splits, is 20 MB.
- */
- public static final long DATAFLOW_SPLIT_RESPONSE_API_SIZE_BYTES = 20 * (1 << 20);
private static final Logger LOG = LoggerFactory.getLogger(CustomSources.class);
- private static final ByteString firstSplitKey = ByteString.copyFromUtf8("0000000000000001");
-
- public static boolean isFirstUnboundedSourceSplit(ByteString splitKey) {
- return splitKey.equals(firstSplitKey);
- }
-
private static int getDesiredNumUnboundedSourceSplits(DataflowPipelineOptions options) {
int cores = 4; //TODO: decide at runtime?
if (options.getMaxNumWorkers() > 0) {
[2/2] beam git commit: Remove protobuf and http-client dependency
from Dataflow runner
Posted by lc...@apache.org.
Remove protobuf and http-client dependency from Dataflow runner
This closes #3072
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ff589c71
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ff589c71
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ff589c71
Branch: refs/heads/master
Commit: ff589c711014120e4cdef9c459879a5c03169a42
Parents: f5ace5c aee9187
Author: Luke Cwik <lc...@google.com>
Authored: Thu May 11 10:22:11 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Thu May 11 10:22:11 2017 -0700
----------------------------------------------------------------------
runners/google-cloud-dataflow-java/pom.xml | 11 -----------
.../beam/runners/dataflow/internal/CustomSources.java | 12 ------------
2 files changed, 23 deletions(-)
----------------------------------------------------------------------