You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/10/11 19:25:57 UTC
[01/12] beam git commit: Add model/ and submodules to Maven structure
and gen_protos.py
Repository: beam
Updated Branches:
refs/heads/master 4ccb3ba96 -> 92615ca4c
Add model/ and submodules to Maven structure and gen_protos.py
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fde0b93f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fde0b93f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fde0b93f
Branch: refs/heads/master
Commit: fde0b93fc5d497c7ecbc4b27c35dd3cf51e66f6f
Parents: 4ccb3ba
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Sep 29 12:09:35 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:52:26 2017 -0700
----------------------------------------------------------------------
model/fn-execution/pom.xml | 82 +++++++++++++++++++++++++++++++++++++++
model/job-management/pom.xml | 82 +++++++++++++++++++++++++++++++++++++++
model/pipeline/pom.xml | 82 +++++++++++++++++++++++++++++++++++++++
model/pom.xml | 40 +++++++++++++++++++
pom.xml | 26 +++++++++++++
sdks/python/gen_protos.py | 3 ++
6 files changed, 315 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/fde0b93f/model/fn-execution/pom.xml
----------------------------------------------------------------------
diff --git a/model/fn-execution/pom.xml b/model/fn-execution/pom.xml
new file mode 100644
index 0000000..807feb6
--- /dev/null
+++ b/model/fn-execution/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <packaging>jar</packaging>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-fn-execution</artifactId>
+ <name>Apache Beam :: Model :: Fn Execution</name>
+ <description>Portable definitions for execution user-defined functions</description>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>${project.build.directory}/original_sources_to_package</directory>
+ </resource>
+ </resources>
+
+ <plugins>
+ <!-- Skip the checkstyle plugin on generated code -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Skip the findbugs plugin on generated code -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/fde0b93f/model/job-management/pom.xml
----------------------------------------------------------------------
diff --git a/model/job-management/pom.xml b/model/job-management/pom.xml
new file mode 100644
index 0000000..3f7fd18
--- /dev/null
+++ b/model/job-management/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <packaging>jar</packaging>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-job-management</artifactId>
+ <name>Apache Beam :: Model :: Job Management</name>
+ <description>Portable definitions for submitting pipelines.</description>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>${project.build.directory}/original_sources_to_package</directory>
+ </resource>
+ </resources>
+
+ <plugins>
+ <!-- Skip the checkstyle plugin on generated code -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Skip the findbugs plugin on generated code -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/fde0b93f/model/pipeline/pom.xml
----------------------------------------------------------------------
diff --git a/model/pipeline/pom.xml b/model/pipeline/pom.xml
new file mode 100644
index 0000000..ed786c0
--- /dev/null
+++ b/model/pipeline/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <packaging>jar</packaging>
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-pipeline</artifactId>
+ <name>Apache Beam :: Model :: Pipeline</name>
+ <description>Portable definitions for building pipelines</description>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>${project.build.directory}/original_sources_to_package</directory>
+ </resource>
+ </resources>
+
+ <plugins>
+ <!-- Skip the checkstyle plugin on generated code -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Skip the findbugs plugin on generated code -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/fde0b93f/model/pom.xml
----------------------------------------------------------------------
diff --git a/model/pom.xml b/model/pom.xml
new file mode 100644
index 0000000..a7ffd3d
--- /dev/null
+++ b/model/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-parent</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-model-parent</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: Model</name>
+
+ <modules>
+ <module>pipeline</module>
+ <module>job-management</module>
+ <module>fn-execution</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/fde0b93f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 13f0f1b..72e2eff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -191,6 +191,7 @@
<!-- sdks/java/build-tools has project-wide configuration. To make these available
in all modules, link it directly to the parent pom.xml. -->
<module>sdks/java/build-tools</module>
+ <module>model</module>
<module>sdks</module>
<module>runners</module>
<module>examples</module>
@@ -402,6 +403,31 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-job-management</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-common-fn-api</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/fde0b93f/sdks/python/gen_protos.py
----------------------------------------------------------------------
diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
index d70158b..f5bb197 100644
--- a/sdks/python/gen_protos.py
+++ b/sdks/python/gen_protos.py
@@ -32,6 +32,9 @@ import warnings
GRPC_TOOLS = 'grpcio-tools>=1.3.5'
BEAM_PROTO_PATHS = [
+ os.path.join('..', '..', 'model', 'pipeline', 'src', 'main', 'proto'),
+ os.path.join('..', '..', 'model', 'job-management', 'src', 'main', 'proto'),
+ os.path.join('..', '..', 'model', 'fn-execution', 'src', 'main', 'proto'),
os.path.join('..', 'common', 'runner-api', 'src', 'main', 'proto'),
os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto')
]
[08/12] beam git commit: Re-namespace model/pipeline to model.pipeline
Posted by ke...@apache.org.
Re-namespace model/pipeline to model.pipeline
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7c563ef8
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7c563ef8
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7c563ef8
Branch: refs/heads/master
Commit: 7c563ef869b711915ed9706737d6098106b7d453
Parents: 3a9a435
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Sep 29 12:07:21 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:56:39 2017 -0700
----------------------------------------------------------------------
.../fn-execution/src/main/proto/beam_fn_api.proto | 18 +++++++++---------
.../src/main/proto/beam_job_api.proto | 4 ++--
.../pipeline/src/main/proto/beam_runner_api.proto | 4 ++--
model/pipeline/src/main/proto/endpoints.proto | 4 ++--
.../src/main/proto/standard_window_fns.proto | 4 ++--
.../core/construction/CoderTranslation.java | 6 +++---
.../core/construction/CombineTranslation.java | 12 ++++++------
.../CreatePCollectionViewTranslation.java | 4 ++--
.../core/construction/DisplayDataTranslation.java | 2 +-
.../core/construction/FlattenTranslator.java | 4 ++--
.../core/construction/GroupByKeyTranslation.java | 4 ++--
.../core/construction/PCollectionTranslation.java | 2 +-
.../core/construction/PTransformTranslation.java | 4 ++--
.../core/construction/ParDoTranslation.java | 16 ++++++++--------
.../core/construction/PipelineTranslation.java | 2 +-
.../core/construction/ReadTranslation.java | 10 +++++-----
.../core/construction/RehydratedComponents.java | 4 ++--
.../core/construction/RunnerPCollectionView.java | 2 +-
.../runners/core/construction/SdkComponents.java | 4 ++--
.../core/construction/TestStreamTranslation.java | 2 +-
.../core/construction/TriggerTranslation.java | 2 +-
.../core/construction/WindowIntoTranslation.java | 6 +++---
.../WindowingStrategyTranslation.java | 18 +++++++++---------
.../core/construction/WriteFilesTranslation.java | 10 +++++-----
.../core/construction/CoderTranslationTest.java | 4 ++--
.../core/construction/CombineTranslationTest.java | 4 ++--
.../CreatePCollectionViewTranslationTest.java | 4 ++--
.../construction/PCollectionTranslationTest.java | 2 +-
.../construction/PTransformTranslationTest.java | 6 +++---
.../core/construction/ParDoTranslationTest.java | 6 +++---
.../construction/PipelineTranslationTest.java | 2 +-
.../core/construction/ReadTranslationTest.java | 4 ++--
.../core/construction/SdkComponentsTest.java | 2 +-
.../construction/TestStreamTranslationTest.java | 6 +++---
.../construction/WindowIntoTranslationTest.java | 2 +-
.../WindowingStrategyTranslationTest.java | 2 +-
.../construction/WriteFilesTranslationTest.java | 4 ++--
.../core/triggers/TriggerStateMachines.java | 2 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
.../direct/GroupAlsoByWindowEvaluatorFactory.java | 2 +-
.../direct/TransformEvaluatorRegistry.java | 2 +-
.../flink/FlinkStreamingTransformTranslators.java | 2 +-
.../beam/runners/dataflow/DataflowRunner.java | 2 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 2 +-
.../dataflow/options/DataflowPipelineOptions.java | 1 -
sdks/java/core/pom.xml | 4 ++++
.../beam/fn/harness/BeamFnDataReadRunner.java | 4 ++--
.../beam/fn/harness/BeamFnDataWriteRunner.java | 4 ++--
.../beam/fn/harness/BoundedSourceRunner.java | 2 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 2 +-
.../org/apache/beam/fn/harness/FnHarness.java | 6 +++---
.../beam/fn/harness/PTransformRunnerFactory.java | 2 +-
.../fn/harness/channel/ManagedChannelFactory.java | 2 +-
.../fn/harness/control/BeamFnControlClient.java | 2 +-
.../fn/harness/control/ProcessBundleHandler.java | 4 ++--
.../beam/fn/harness/control/RegisterHandler.java | 2 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 2 +-
.../fn/harness/data/BeamFnDataGrpcClient.java | 2 +-
.../harness/data/BeamFnDataGrpcMultiplexer.java | 4 ++--
.../fn/harness/logging/BeamFnLoggingClient.java | 2 +-
.../harness/state/BeamFnStateGrpcClientCache.java | 4 ++--
.../fn/harness/state/StateFetchingIterators.java | 3 ++-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 6 +++---
.../fn/harness/BeamFnDataWriteRunnerTest.java | 6 +++---
.../beam/fn/harness/BoundedSourceRunnerTest.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 2 +-
.../org/apache/beam/fn/harness/FnHarnessTest.java | 2 +-
.../channel/ManagedChannelFactoryTest.java | 2 +-
.../harness/control/BeamFnControlClientTest.java | 2 +-
.../harness/control/ProcessBundleHandlerTest.java | 4 ++--
.../fn/harness/control/RegisterHandlerTest.java | 2 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 2 +-
.../data/BeamFnDataGrpcMultiplexerTest.java | 2 +-
.../harness/logging/BeamFnLoggingClientTest.java | 2 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 2 +-
sdks/python/gen_protos.py | 1 -
76 files changed, 151 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/model/fn-execution/src/main/proto/beam_fn_api.proto
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/main/proto/beam_fn_api.proto b/model/fn-execution/src/main/proto/beam_fn_api.proto
index 5a01077..f519581 100644
--- a/model/fn-execution/src/main/proto/beam_fn_api.proto
+++ b/model/fn-execution/src/main/proto/beam_fn_api.proto
@@ -74,7 +74,7 @@ message Target {
message RemoteGrpcPort {
// (Required) An API descriptor which describes where to
// connect to including any authentication that is required.
- org.apache.beam.portability.v1.ApiServiceDescriptor api_service_descriptor = 1;
+ org.apache.beam.model.pipeline.v1.ApiServiceDescriptor api_service_descriptor = 1;
}
/*
@@ -158,24 +158,24 @@ message ProcessBundleDescriptor {
string id = 1;
// (Required) A map from pipeline-scoped id to PTransform.
- map<string, org.apache.beam.runner_api.v1.PTransform> transforms = 2;
+ map<string, org.apache.beam.model.pipeline.v1.PTransform> transforms = 2;
// (Required) A map from pipeline-scoped id to PCollection.
- map<string, org.apache.beam.runner_api.v1.PCollection> pcollections = 3;
+ map<string, org.apache.beam.model.pipeline.v1.PCollection> pcollections = 3;
// (Required) A map from pipeline-scoped id to WindowingStrategy.
- map<string, org.apache.beam.runner_api.v1.WindowingStrategy> windowing_strategies = 4;
+ map<string, org.apache.beam.model.pipeline.v1.WindowingStrategy> windowing_strategies = 4;
// (Required) A map from pipeline-scoped id to Coder.
- map<string, org.apache.beam.runner_api.v1.Coder> coders = 5;
+ map<string, org.apache.beam.model.pipeline.v1.Coder> coders = 5;
// (Required) A map from pipeline-scoped id to Environment.
- map<string, org.apache.beam.runner_api.v1.Environment> environments = 6;
+ map<string, org.apache.beam.model.pipeline.v1.Environment> environments = 6;
// A descriptor describing the end point to use for State API
// calls. Required if the Runner intends to send remote references over the
// data plane or if any of the transforms rely on user state or side inputs.
- org.apache.beam.portability.v1.ApiServiceDescriptor state_api_service_descriptor = 7;
+ org.apache.beam.model.pipeline.v1.ApiServiceDescriptor state_api_service_descriptor = 7;
}
// A request to process a given bundle.
@@ -334,14 +334,14 @@ message PrimitiveTransformSplit {
//
// For example, a remote GRPC source will have a specific urn and data
// block containing an ElementCountRestriction.
- org.apache.beam.runner_api.v1.FunctionSpec completed_restriction = 2;
+ org.apache.beam.model.pipeline.v1.FunctionSpec completed_restriction = 2;
// (Required) A function specification describing the restriction
// representing the remainder of work for the primitive transform.
//
// FOr example, a remote GRPC source will have a specific urn and data
// block contain an ElemntCountSkipRestriction.
- org.apache.beam.runner_api.v1.FunctionSpec remaining_restriction = 3;
+ org.apache.beam.model.pipeline.v1.FunctionSpec remaining_restriction = 3;
}
message ProcessBundleSplitResponse {
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/model/job-management/src/main/proto/beam_job_api.proto
----------------------------------------------------------------------
diff --git a/model/job-management/src/main/proto/beam_job_api.proto b/model/job-management/src/main/proto/beam_job_api.proto
index 8f2ed0e..3830f5c 100644
--- a/model/job-management/src/main/proto/beam_job_api.proto
+++ b/model/job-management/src/main/proto/beam_job_api.proto
@@ -61,7 +61,7 @@ service JobService {
// Throws error ALREADY_EXISTS if the jobName is reused. Runners are permitted to deduplicate based on the name of the job.
// Throws error UNKNOWN for all other issues
message PrepareJobRequest {
- org.apache.beam.runner_api.v1.Pipeline pipeline = 1; // (required)
+ org.apache.beam.model.pipeline.v1.Pipeline pipeline = 1; // (required)
google.protobuf.Struct pipeline_options = 2; // (required)
string job_name = 3; // (required)
}
@@ -73,7 +73,7 @@ message PrepareJobResponse {
// An endpoint which exposes the Beam Artifact Staging API. Artifacts used by the job should be
// staged to this endpoint, and will be available during job execution.
- org.apache.beam.portability.v1.ApiServiceDescriptor artifact_staging_endpoint = 2;
+ org.apache.beam.model.pipeline.v1.ApiServiceDescriptor artifact_staging_endpoint = 2;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/model/pipeline/src/main/proto/beam_runner_api.proto
----------------------------------------------------------------------
diff --git a/model/pipeline/src/main/proto/beam_runner_api.proto b/model/pipeline/src/main/proto/beam_runner_api.proto
index 74f3897..36369dd 100644
--- a/model/pipeline/src/main/proto/beam_runner_api.proto
+++ b/model/pipeline/src/main/proto/beam_runner_api.proto
@@ -23,9 +23,9 @@
syntax = "proto3";
-package org.apache.beam.runner_api.v1;
+package org.apache.beam.model.pipeline.v1;
-option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_package = "org.apache.beam.model.pipeline.v1";
option java_outer_classname = "RunnerApi";
import "google/protobuf/any.proto";
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/model/pipeline/src/main/proto/endpoints.proto
----------------------------------------------------------------------
diff --git a/model/pipeline/src/main/proto/endpoints.proto b/model/pipeline/src/main/proto/endpoints.proto
index a642e63..0ed398e 100644
--- a/model/pipeline/src/main/proto/endpoints.proto
+++ b/model/pipeline/src/main/proto/endpoints.proto
@@ -22,9 +22,9 @@
syntax = "proto3";
-package org.apache.beam.portability.v1;
+package org.apache.beam.model.pipeline.v1;
-option java_package = "org.apache.beam.portability.v1";
+option java_package = "org.apache.beam.model.pipeline.v1";
option java_outer_classname = "Endpoints";
message ApiServiceDescriptor {
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/model/pipeline/src/main/proto/standard_window_fns.proto
----------------------------------------------------------------------
diff --git a/model/pipeline/src/main/proto/standard_window_fns.proto b/model/pipeline/src/main/proto/standard_window_fns.proto
index 0682044..4bccc6f 100644
--- a/model/pipeline/src/main/proto/standard_window_fns.proto
+++ b/model/pipeline/src/main/proto/standard_window_fns.proto
@@ -23,9 +23,9 @@
syntax = "proto3";
-package org.apache.beam.runner_api.v1;
+package org.apache.beam.model.pipeline.v1;
-option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_package = "org.apache.beam.model.pipeline.v1";
option java_outer_classname = "StandardWindowFns";
import "google/protobuf/duration.proto";
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
index 2246f81..2b00ce4 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
@@ -30,6 +30,9 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
@@ -37,9 +40,6 @@ import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.LengthPrefixCoder;
import org.apache.beam.sdk.coders.StructuredCoder;
import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow.IntervalWindowCoder;
import org.apache.beam.sdk.util.SerializableUtils;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
index 17c48dc..c3d9553 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
@@ -28,16 +28,16 @@ import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.CombinePayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.CombinePayload;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
index 1027ea2..4b8edcf 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
@@ -25,9 +25,9 @@ import com.google.protobuf.ByteString;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java
index 5186caf..8a9394d 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.core.construction;
import com.google.protobuf.Any;
import com.google.protobuf.BoolValue;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.transforms.display.DisplayData;
/** Utilities for going to/from DisplayData protos. */
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
index f1d553d..c9798e6 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
@@ -21,9 +21,9 @@ package org.apache.beam.runners.core.construction;
import com.google.auto.service.AutoService;
import java.util.Collections;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.PTransform;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java
index db73461..840bae2 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java
@@ -21,9 +21,9 @@ package org.apache.beam.runners.core.construction;
import com.google.auto.service.AutoService;
import java.util.Collections;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
index 84b3386..b85efe6 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
@@ -19,9 +19,9 @@
package org.apache.beam.runners.core.construction;
import java.io.IOException;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollection.IsBounded;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 41fac11..f9e7837 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -32,9 +32,9 @@ import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.display.DisplayData;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
index 714c59d..3886e47 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -38,17 +38,17 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Parameter.Type;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput.Builder;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Parameter.Type;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput.Builder;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.StateSpecs;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java
index 8a2faf3..0aca837 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java
@@ -34,11 +34,11 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
index 4cc31e8..e9168a2 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
@@ -27,12 +27,12 @@ import com.google.protobuf.InvalidProtocolBufferException;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.IsBounded;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ReadPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.IsBounded;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.ReadPayload;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.Source;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java
index ccdd4a7..2e12603 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java
@@ -26,10 +26,10 @@ import com.google.common.cache.LoadingCache;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.WindowingStrategy;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
index 85139e8..c676c97 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
@@ -21,8 +21,8 @@ package org.apache.beam.runners.core.construction;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index c2267dd..0a8ffb6 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -27,10 +27,10 @@ import com.google.common.collect.HashBiMap;
import java.io.IOException;
import java.util.List;
import java.util.Set;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.util.NameUtils;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
index cac7cdc..de4d6bb 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
@@ -28,9 +28,9 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.testing.TestStream;
import org.apache.beam.sdk.transforms.PTransform;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
index b23f686..6b2a182 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
@@ -23,8 +23,8 @@ import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.List;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.transforms.windowing.AfterAll;
import org.apache.beam.sdk.transforms.windowing.AfterEach;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
index 94ef22d..ad6177d 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
@@ -25,10 +25,10 @@ import com.google.protobuf.InvalidProtocolBufferException;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.WindowIntoPayload;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.WindowIntoPayload;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.Window;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
index be8601c..893fbe5 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
@@ -23,15 +23,15 @@ import com.google.protobuf.util.Durations;
import com.google.protobuf.util.Timestamps;
import java.io.IOException;
import java.io.Serializable;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.OutputTime;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.StandardWindowFns;
-import org.apache.beam.sdk.common.runner.v1.StandardWindowFns.FixedWindowsPayload;
-import org.apache.beam.sdk.common.runner.v1.StandardWindowFns.SessionsPayload;
-import org.apache.beam.sdk.common.runner.v1.StandardWindowFns.SlidingWindowsPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.OutputTime;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns.FixedWindowsPayload;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns.SessionsPayload;
+import org.apache.beam.model.pipeline.v1.StandardWindowFns.SlidingWindowsPayload;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
import org.apache.beam.sdk.transforms.windowing.Sessions;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java
index a9601bc..5a49747 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java
@@ -31,12 +31,12 @@ import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
+import org.apache.beam.model.pipeline.v1.RunnerApi.WriteFilesPayload;
import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.WriteFilesPayload;
import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.WriteFiles;
import org.apache.beam.sdk.io.WriteFilesResult;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
index 95766f5..12ff9d6 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
@@ -30,6 +30,8 @@ import java.io.OutputStream;
import java.io.Serializable;
import java.util.HashSet;
import java.util.Set;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.AvroCoder;
import org.apache.beam.sdk.coders.ByteArrayCoder;
@@ -42,8 +44,6 @@ import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.StructuredCoder;
import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow.IntervalWindowCoder;
import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
index 0373fba..8740d7f 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
@@ -23,12 +23,12 @@ import static org.junit.Assert.assertEquals;
import com.google.common.collect.ImmutableList;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.CombinePayload;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.CombinePayload;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy.Node;
import org.apache.beam.sdk.testing.TestPipeline;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java
index 4f57af8..df659a8 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java
@@ -21,8 +21,8 @@ package org.apache.beam.runners.core.construction;
import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
index df02a39..6c641bb 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
@@ -28,12 +28,12 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.Collection;
import java.util.Collections;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.BigEndianLongCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CustomCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
index 0e6ef97..2ec96e8 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
@@ -31,10 +31,10 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
+import org.apache.beam.model.pipeline.v1.RunnerApi.PTransform;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.PTransform;
import org.apache.beam.sdk.io.CountingSource;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.io.Read;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
index 680f940..b79947e 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
@@ -25,14 +25,14 @@ import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.SideInput;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.state.BagState;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java
index 9e6dff4..66fe686 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java
@@ -26,12 +26,12 @@ import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.coders.BigEndianLongCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StructuredCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.runners.TransformHierarchy.Node;
import org.apache.beam.sdk.transforms.Count;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
index 22c79b3..56cf5f3 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
@@ -29,13 +29,13 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ReadPayload;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.ReadPayload;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.CountingSource;
import org.apache.beam.sdk.io.Read;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
index 82840d6..44c42cc 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
@@ -26,6 +26,7 @@ import static org.junit.Assert.assertThat;
import java.io.IOException;
import java.util.Collections;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
@@ -33,7 +34,6 @@ import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.SetCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.testing.TestPipeline;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java
index 893f4b9..3678fc7 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java
@@ -23,13 +23,13 @@ import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi.TestStreamPayload;
import org.apache.beam.runners.core.construction.TestStreamTranslationTest.TestStreamPayloadTranslation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.TestStreamPayload;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.TestStream;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
index 3ba2d6f..b40ccb3 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
@@ -24,9 +24,9 @@ import static org.junit.Assert.assertEquals;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.InvalidProtocolBufferException;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.model.pipeline.v1.RunnerApi.WindowIntoPayload;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.WindowIntoPayload;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy.Node;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
index e57a088..ddf0316 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertThat;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Sessions;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java
index e8eda76..c874828 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java
@@ -24,8 +24,8 @@ import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
import java.util.Objects;
import javax.annotation.Nullable;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
import org.apache.beam.sdk.io.DynamicFileDestinations;
import org.apache.beam.sdk.io.FileBasedSink;
import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java
index a6b38ec..32c9153 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachines.java
@@ -19,7 +19,7 @@ package org.apache.beam.runners.core.triggers;
import java.util.ArrayList;
import java.util.List;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.transforms.windowing.Trigger;
import org.joda.time.Duration;
import org.joda.time.Instant;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
index 5158f50..754110e 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
@@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertThat;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.state.TimeDomain;
import org.joda.time.Duration;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index a944e75..d80e4ff 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.Collection;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.GroupAlsoByWindowsAggregators;
import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly;
import org.apache.beam.runners.core.KeyedWorkItem;
@@ -38,7 +39,6 @@ import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.metrics.Counter;
import org.apache.beam.sdk.metrics.Metrics;
import org.apache.beam.sdk.runners.AppliedPTransform;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
index 7bba7d5..9cfa79f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
@@ -37,6 +37,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessElements;
import org.apache.beam.runners.core.construction.PTransformTranslation;
@@ -44,7 +45,6 @@ import org.apache.beam.runners.core.construction.PTransformTranslation.Transform
import org.apache.beam.runners.core.construction.SdkComponents;
import org.apache.beam.runners.core.construction.TransformPayloadTranslatorRegistrar;
import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DirectTestStream;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index 4d2166c..d1e2d57 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -31,6 +31,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.KeyedWorkItem;
import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
import org.apache.beam.runners.core.SystemReduceFn;
@@ -54,7 +55,6 @@ import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.Combine;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 4cd3db0..e637dd4 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -58,6 +58,7 @@ import java.util.Random;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.CoderTranslation;
import org.apache.beam.runners.core.construction.DeduplicatedFlattenFactory;
import org.apache.beam.runners.core.construction.EmptyFlattenAsCreateFactory;
@@ -89,7 +90,6 @@ import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.FileBasedSink;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
index 9252c64..aaf6ee1 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
@@ -19,11 +19,11 @@
package org.apache.beam.runners.dataflow;
import java.util.List;
+import org.apache.beam.model.pipeline.v1.RunnerApi.DisplayData;
import org.apache.beam.runners.core.construction.ForwardingPTransform;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.DisplayData;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.transforms.DoFn;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 77154b5..091f89b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -18,7 +18,6 @@
package org.apache.beam.runners.dataflow.options;
import java.util.Map;
-
import org.apache.beam.runners.dataflow.DataflowRunner;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 6c74cd9..2e54e43 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -311,6 +311,10 @@
<groupId>org.apache.beam</groupId>
<artifactId>beam-model-pipeline</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-construction</artifactId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
index 4cae4f1..e520780 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
@@ -35,11 +35,11 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.CoderTranslation;
import org.apache.beam.runners.core.construction.RehydratedComponents;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
index 20402f8..e08c422 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
@@ -34,11 +34,11 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.CoderTranslation;
import org.apache.beam.runners.core.construction.RehydratedComponents;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
index c4daa0f..d523365 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
@@ -32,7 +32,7 @@ import org.apache.beam.fn.harness.data.BeamFnDataClient;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.Source.Reader;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
index f0ee319..2c0da7f 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
@@ -48,11 +48,11 @@ import org.apache.beam.fn.harness.state.BeamFnStateClient;
import org.apache.beam.fn.v1.BeamFnApi.StateKey;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.construction.ParDoTranslation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.CombiningState;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
index 5ed93e4..8483f6c 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
@@ -32,7 +32,7 @@ import org.apache.beam.fn.harness.logging.BeamFnLoggingClient;
import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
import org.apache.beam.fn.harness.stream.StreamObserverFactory;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.common.ReflectHelpers;
@@ -45,10 +45,10 @@ import org.slf4j.LoggerFactory;
* <p>This entry point expects the following environment variables:
* <ul>
* <li>LOGGING_API_SERVICE_DESCRIPTOR: A
- * {@link org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor} encoded as text
+ * {@link org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor} encoded as text
* representing the endpoint that is to be connected to for the Beam Fn Logging service.</li>
* <li>CONTROL_API_SERVICE_DESCRIPTOR: A
- * {@link org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor} encoded as text
+ * {@link Endpoints.ApiServiceDescriptor} encoded as text
* representing the endpoint that is to be connected to for the Beam Fn Control service.</li>
* <li>PIPELINE_OPTIONS: A serialized form of {@link PipelineOptions}. See {@link PipelineOptions}
* for further details.</li>
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java
index 4ef56d8..126055a 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PTransformRunnerFactory.java
@@ -26,7 +26,7 @@ import org.apache.beam.fn.harness.data.BeamFnDataClient;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java
index c7e60fd..62e8b44 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/channel/ManagedChannelFactory.java
@@ -27,7 +27,7 @@ import io.netty.channel.epoll.EpollSocketChannel;
import io.netty.channel.unix.DomainSocketAddress;
import java.net.SocketAddress;
import java.util.List;
-import org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
index 8b34f0c..2ab6a41 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
@@ -34,7 +34,7 @@ import java.util.function.Function;
import org.apache.beam.fn.harness.fn.ThrowingFunction;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnControlGrpc;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
index c311c4c..788efec 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
@@ -48,8 +48,8 @@ import org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
-import org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.common.ReflectHelpers;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
index 0e738ac..704f782 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
@@ -26,7 +26,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnApi.RegisterResponse;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
index a3c2f5d..ca158ff 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
@@ -22,7 +22,7 @@ import java.util.concurrent.CompletableFuture;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
index f9aebdf..fe8af62 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
@@ -29,7 +29,7 @@ import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnDataGrpc;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
index a3c3986..da64112 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
@@ -27,14 +27,14 @@ import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.values.KV;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A gRPC multiplexer for a specific {@link
- * org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor}.
+ * Endpoints.ApiServiceDescriptor}.
*
* <p>Multiplexes data for inbound consumers based upon their individual {@link
* org.apache.beam.fn.v1.BeamFnApi.Target}s.
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
index a8f151c..82cdff1 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
@@ -48,7 +48,7 @@ import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptions;
[04/12] beam git commit: Move sdk/common/runner-api protos into
model/pipeline and model/job-management
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
----------------------------------------------------------------------
diff --git a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
deleted file mode 100644
index 74f3897..0000000
--- a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
+++ /dev/null
@@ -1,842 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Runner API, which is the runner-independent,
- * SDK-independent definition of the Beam model.
- */
-
-syntax = "proto3";
-
-package org.apache.beam.runner_api.v1;
-
-option java_package = "org.apache.beam.sdk.common.runner.v1";
-option java_outer_classname = "RunnerApi";
-
-import "google/protobuf/any.proto";
-
-// A set of mappings from id to message. This is included as an optional field
-// on any proto message that may contain references needing resolution.
-message Components {
- // (Required) A map from pipeline-scoped id to PTransform.
- map<string, PTransform> transforms = 1;
-
- // (Required) A map from pipeline-scoped id to PCollection.
- map<string, PCollection> pcollections = 2;
-
- // (Required) A map from pipeline-scoped id to WindowingStrategy.
- map<string, WindowingStrategy> windowing_strategies = 3;
-
- // (Required) A map from pipeline-scoped id to Coder.
- map<string, Coder> coders = 4;
-
- // (Required) A map from pipeline-scoped id to Environment.
- map<string, Environment> environments = 5;
-}
-
-// A disjoint union of all the things that may contain references
-// that require Components to resolve.
-message MessageWithComponents {
-
- // (Optional) The by-reference components of the root message,
- // enabling a standalone message.
- //
- // If this is absent, it is expected that there are no
- // references.
- Components components = 1;
-
- // (Required) The root message that may contain pointers
- // that should be resolved by looking inside components.
- oneof root {
- Coder coder = 2;
- CombinePayload combine_payload = 3;
- SdkFunctionSpec sdk_function_spec = 4;
- ParDoPayload par_do_payload = 6;
- PTransform ptransform = 7;
- PCollection pcollection = 8;
- ReadPayload read_payload = 9;
- SideInput side_input = 11;
- WindowIntoPayload window_into_payload = 12;
- WindowingStrategy windowing_strategy = 13;
- FunctionSpec function_spec = 14;
- }
-}
-
-// A Pipeline is a hierarchical graph of PTransforms, linked
-// by PCollections.
-//
-// This is represented by a number of by-reference maps to nodes,
-// PCollections, SDK environments, UDF, etc., for
-// supporting compact reuse and arbitrary graph structure.
-//
-// All of the keys in the maps here are arbitrary strings that are only
-// required to be internally consistent within this proto message.
-message Pipeline {
-
- // (Required) The coders, UDFs, graph nodes, etc, that make up
- // this pipeline.
- Components components = 1;
-
- // (Required) The ids of all PTransforms that are not contained within another PTransform.
- // These must be in shallow topological order, so that traversing them recursively
- // in this order yields a recursively topological traversal.
- repeated string root_transform_ids = 2;
-
- // (Optional) Static display data for the pipeline. If there is none,
- // it may be omitted.
- DisplayData display_data = 3;
-}
-
-// An applied PTransform! This does not contain the graph data, but only the
-// fields specific to a graph node that is a Runner API transform
-// between PCollections.
-message PTransform {
-
- // (Required) A unique name for the application node.
- //
- // Ideally, this should be stable over multiple evolutions of a pipeline
- // for the purposes of logging and associating pipeline state with a node,
- // etc.
- //
- // If it is not stable, then the runner decides what will happen. But, most
- // importantly, it must always be here and be unique, even if it is
- // autogenerated.
- string unique_name = 5;
-
- // (Optional) A URN and payload that, together, fully defined the semantics
- // of this transform.
- //
- // If absent, this must be an "anonymous" composite transform.
- //
- // For primitive transform in the Runner API, this is required, and the
- // payloads are well-defined messages. When the URN indicates ParDo it
- // is a ParDoPayload, and so on.
- //
- // TODO: document the standardized URNs and payloads
- // TODO: separate standardized payloads into a separate proto file
- //
- // For some special composite transforms, the payload is also officially
- // defined:
- //
- // - when the URN is "urn:beam:transforms:combine" it is a CombinePayload
- //
- FunctionSpec spec = 1;
-
- // (Optional) if this node is a composite, a list of the ids of
- // transforms that it contains.
- repeated string subtransforms = 2;
-
- // (Required) A map from local names of inputs (unique only with this map, and
- // likely embedded in the transform payload and serialized user code) to
- // PCollection ids.
- //
- // The payload for this transform may clarify the relationship of these
- // inputs. For example:
- //
- // - for a Flatten transform they are merged
- // - for a ParDo transform, some may be side inputs
- //
- // All inputs are recorded here so that the topological ordering of
- // the graph is consistent whether or not the payload is understood.
- //
- map<string, string> inputs = 3;
-
- // (Required) A map from local names of outputs (unique only within this map,
- // and likely embedded in the transform payload and serialized user code)
- // to PCollection ids.
- //
- // The URN or payload for this transform node may clarify the type and
- // relationship of these outputs. For example:
- //
- // - for a ParDo transform, these are tags on PCollections, which will be
- // embedded in the DoFn.
- //
- map<string, string> outputs = 4;
-
- // (Optional) Static display data for this PTransform application. If
- // there is none, or it is not relevant (such as use by the Fn API)
- // then it may be omitted.
- DisplayData display_data = 6;
-}
-
-// A PCollection!
-message PCollection {
-
- // (Required) A unique name for the PCollection.
- //
- // Ideally, this should be stable over multiple evolutions of a pipeline
- // for the purposes of logging and associating pipeline state with a node,
- // etc.
- //
- // If it is not stable, then the runner decides what will happen. But, most
- // importantly, it must always be here, even if it is autogenerated.
- string unique_name = 1;
-
- // (Required) The id of the Coder for this PCollection.
- string coder_id = 2;
-
- // (Required) Whether this PCollection is bounded or unbounded
- IsBounded.Enum is_bounded = 3;
-
- // (Required) The id of the windowing strategy for this PCollection.
- string windowing_strategy_id = 4;
-
- // (Optional) Static display data for this PTransform application. If
- // there is none, or it is not relevant (such as use by the Fn API)
- // then it may be omitted.
- DisplayData display_data = 5;
-}
-
-// The payload for the primitive ParDo transform.
-message ParDoPayload {
-
- // (Required) The SdkFunctionSpec of the DoFn.
- SdkFunctionSpec do_fn = 1;
-
- // (Required) Additional pieces of context the DoFn may require that
- // are not otherwise represented in the payload.
- // (may force runners to execute the ParDo differently)
- repeated Parameter parameters = 2;
-
- // (Optional) A mapping of local input names to side inputs, describing
- // the expected access pattern.
- map<string, SideInput> side_inputs = 3;
-
- // (Optional) A mapping of local state names to state specifications.
- map<string, StateSpec> state_specs = 4;
-
- // (Optional) A mapping of local timer names to timer specifications.
- map<string, TimerSpec> timer_specs = 5;
-
- // Whether the DoFn is splittable
- bool splittable = 6;
-}
-
-// Parameters that a UDF might require.
-//
-// The details of how a runner sends these parameters to the SDK harness
-// are the subject of the Fn API.
-//
-// The details of how an SDK harness delivers them to the UDF is entirely
-// up to the SDK. (for some SDKs there may be parameters that are not
-// represented here if the runner doesn't need to do anything)
-//
-// Here, the parameters are simply indicators to the runner that they
-// need to run the function a particular way.
-//
-// TODO: the evolution of the Fn API will influence what needs explicit
-// representation here
-message Parameter {
- Type.Enum type = 1;
-
- message Type {
- enum Enum {
- UNSPECIFIED = 0;
- WINDOW = 1;
- PIPELINE_OPTIONS = 2;
- RESTRICTION_TRACKER = 3;
- }
- }
-}
-
-message StateSpec {
- oneof spec {
- ValueStateSpec value_spec = 1;
- BagStateSpec bag_spec = 2;
- CombiningStateSpec combining_spec = 3;
- MapStateSpec map_spec = 4;
- SetStateSpec set_spec = 5;
- }
-}
-
-message ValueStateSpec {
- string coder_id = 1;
-}
-
-message BagStateSpec {
- string element_coder_id = 1;
-}
-
-message CombiningStateSpec {
- string accumulator_coder_id = 1;
- SdkFunctionSpec combine_fn = 2;
-}
-
-message MapStateSpec {
- string key_coder_id = 1;
- string value_coder_id = 2;
-}
-
-message SetStateSpec {
- string element_coder_id = 1;
-}
-
-message TimerSpec {
- TimeDomain.Enum time_domain = 1;
-}
-
-message IsBounded {
- enum Enum {
- UNSPECIFIED = 0;
- UNBOUNDED = 1;
- BOUNDED = 2;
- }
-}
-
-// The payload for the primitive Read transform.
-message ReadPayload {
-
- // (Required) The SdkFunctionSpec of the source for this Read.
- SdkFunctionSpec source = 1;
-
- // (Required) Whether the source is bounded or unbounded
- IsBounded.Enum is_bounded = 2;
-
- // TODO: full audit of fields required by runners as opposed to SDK harness
-}
-
-// The payload for the WindowInto transform.
-message WindowIntoPayload {
-
- // (Required) The SdkFunctionSpec of the WindowFn.
- SdkFunctionSpec window_fn = 1;
-}
-
-// The payload for the special-but-not-primitive Combine transform.
-message CombinePayload {
-
- // (Required) The SdkFunctionSpec of the CombineFn.
- SdkFunctionSpec combine_fn = 1;
-
- // (Required) A reference to the Coder to use for accumulators of the CombineFn
- string accumulator_coder_id = 2;
-
- // (Required) Additional pieces of context the DoFn may require that
- // are not otherwise represented in the payload.
- // (may force runners to execute the ParDo differently)
- repeated Parameter parameters = 3;
-
- // (Optional) A mapping of local input names to side inputs, describing
- // the expected access pattern.
- map<string, SideInput> side_inputs = 4;
-}
-
-// The payload for the test-only primitive TestStream
-message TestStreamPayload {
-
- // (Required) the coder for elements in the TestStream events
- string coder_id = 1;
-
- repeated Event events = 2;
-
- message Event {
- oneof event {
- AdvanceWatermark watermark_event = 1;
- AdvanceProcessingTime processing_time_event = 2;
- AddElements element_event = 3;
- }
-
- message AdvanceWatermark {
- int64 new_watermark = 1;
- }
-
- message AdvanceProcessingTime {
- int64 advance_duration = 1;
- }
-
- message AddElements {
- repeated TimestampedElement elements = 1;
- }
- }
-
- message TimestampedElement {
- bytes encoded_element = 1;
- int64 timestamp = 2;
- }
-}
-// The payload for the special-but-not-primitive WriteFiles transform.
-message WriteFilesPayload {
-
- // (Required) The SdkFunctionSpec of the FileBasedSink.
- SdkFunctionSpec sink = 1;
-
- // (Required) The format function.
- SdkFunctionSpec format_function = 2;
-
- bool windowed_writes = 3;
-
- bool runner_determined_sharding = 4;
-
- map<string, SideInput> side_inputs = 5;
-}
-
-// A coder, the binary format for serialization and deserialization of data in
-// a pipeline.
-message Coder {
-
- // (Required) A specification for the coder, as a URN plus parameters. This
- // may be a cross-language agreed-upon format, or it may be a "custom coder"
- // that can only be used by a particular SDK. It does not include component
- // coders, as it is beneficial for these to be comprehensible to a runner
- // regardless of whether the binary format is agree-upon.
- SdkFunctionSpec spec = 1;
-
- // (Optional) If this coder is parametric, such as ListCoder(VarIntCoder),
- // this is a list of the components. In order for encodings to be identical,
- // the SdkFunctionSpec and all components must be identical, recursively.
- repeated string component_coder_ids = 2;
-}
-
-// A windowing strategy describes the window function, triggering, allowed
-// lateness, and accumulation mode for a PCollection.
-//
-// TODO: consider inlining field on PCollection
-message WindowingStrategy {
-
- // (Required) The SdkFunctionSpec of the UDF that assigns windows,
- // merges windows, and shifts timestamps before they are
- // combined according to the OutputTime.
- SdkFunctionSpec window_fn = 1;
-
- // (Required) Whether or not the window fn is merging.
- //
- // This knowledge is required for many optimizations.
- MergeStatus.Enum merge_status = 2;
-
- // (Required) The coder for the windows of this PCollection.
- string window_coder_id = 3;
-
- // (Required) The trigger to use when grouping this PCollection.
- Trigger trigger = 4;
-
- // (Required) The accumulation mode indicates whether new panes are a full
- // replacement for prior panes or whether they are deltas to be combined
- // with other panes (the combine should correspond to whatever the upstream
- // grouping transform is).
- AccumulationMode.Enum accumulation_mode = 5;
-
- // (Required) The OutputTime specifies, for a grouping transform, how to
- // compute the aggregate timestamp. The window_fn will first possibly shift
- // it later, then the OutputTime takes the max, min, or ignores it and takes
- // the end of window.
- //
- // This is actually only for input to grouping transforms, but since they
- // may be introduced in runner-specific ways, it is carried along with the
- // windowing strategy.
- OutputTime.Enum output_time = 6;
-
- // (Required) Indicate when output should be omitted upon window expiration.
- ClosingBehavior.Enum closing_behavior = 7;
-
- // (Required) The duration, in milliseconds, beyond the end of a window at
- // which the window becomes droppable.
- int64 allowed_lateness = 8;
-
- // (Required) Indicate whether empty on-time panes should be omitted.
- OnTimeBehavior.Enum OnTimeBehavior = 9;
-
- // (Required) Whether or not the window fn assigns inputs to exactly one window
- //
- // This knowledge is required for some optimizations
- bool assigns_to_one_window = 10;
-}
-
-// Whether or not a PCollection's WindowFn is non-merging, merging, or
-// merging-but-already-merged, in which case a subsequent GroupByKey is almost
-// always going to do something the user does not want
-message MergeStatus {
- enum Enum {
- UNSPECIFIED = 0;
-
- // The WindowFn does not require merging.
- // Examples: global window, FixedWindows, SlidingWindows
- NON_MERGING = 1;
-
- // The WindowFn is merging and the PCollection has not had merging
- // performed.
- // Example: Sessions prior to a GroupByKey
- NEEDS_MERGE = 2;
-
- // The WindowFn is merging and the PCollection has had merging occur
- // already.
- // Example: Sessions after a GroupByKey
- ALREADY_MERGED = 3;
- }
-}
-
-// Whether or not subsequent outputs of aggregations should be entire
-// replacement values or just the aggregation of inputs received since
-// the prior output.
-message AccumulationMode {
- enum Enum {
- UNSPECIFIED = 0;
-
- // The aggregation is discarded when it is output
- DISCARDING = 1;
-
- // The aggregation is accumulated across outputs
- ACCUMULATING = 2;
- }
-}
-
-// Controls whether or not an aggregating transform should output data
-// when a window expires.
-message ClosingBehavior {
- enum Enum {
- UNSPECIFIED = 0;
-
- // Emit output when a window expires, whether or not there has been
- // any new data since the last output.
- EMIT_ALWAYS = 1;
-
- // Only emit output when new data has arrives since the last output
- EMIT_IF_NONEMPTY = 2;
- }
-}
-
-// Controls whether or not an aggregating transform should output data
-// when an on-time pane is empty.
-message OnTimeBehavior {
- enum Enum {
- UNSPECIFIED = 0;
-
- // Always fire the on-time pane. Even if there is no new data since
- // the previous firing, an element will be produced.
- FIRE_ALWAYS = 1;
-
- // Only fire the on-time pane if there is new data since the previous firing.
- FIRE_IF_NONEMPTY = 2;
- }
-}
-
-// When a number of windowed, timestamped inputs are aggregated, the timestamp
-// for the resulting output.
-message OutputTime {
- enum Enum {
- UNSPECIFIED = 0;
-
- // The output has the timestamp of the end of the window.
- END_OF_WINDOW = 1;
-
- // The output has the latest timestamp of the input elements since
- // the last output.
- LATEST_IN_PANE = 2;
-
- // The output has the earliest timestamp of the input elements since
- // the last output.
- EARLIEST_IN_PANE = 3;
- }
-}
-
-// The different time domains in the Beam model.
-message TimeDomain {
- enum Enum {
- UNSPECIFIED = 0;
-
- // Event time is time from the perspective of the data
- EVENT_TIME = 1;
-
- // Processing time is time from the perspective of the
- // execution of your pipeline
- PROCESSING_TIME = 2;
-
- // Synchronized processing time is the minimum of the
- // processing time of all pending elements.
- //
- // The "processing time" of an element refers to
- // the local processing time at which it was emitted
- SYNCHRONIZED_PROCESSING_TIME = 3;
- }
-}
-
-// A small DSL for expressing when to emit new aggregations
-// from a GroupByKey or CombinePerKey
-//
-// A trigger is described in terms of when it is _ready_ to permit output.
-message Trigger {
-
- // Ready when all subtriggers are ready.
- message AfterAll {
- repeated Trigger subtriggers = 1;
- }
-
- // Ready when any subtrigger is ready.
- message AfterAny {
- repeated Trigger subtriggers = 1;
- }
-
- // Starting with the first subtrigger, ready when the _current_ subtrigger
- // is ready. After output, advances the current trigger by one.
- message AfterEach {
- repeated Trigger subtriggers = 1;
- }
-
- // Ready after the input watermark is past the end of the window.
- //
- // May have implicitly-repeated subtriggers for early and late firings.
- // When the end of the window is reached, the trigger transitions between
- // the subtriggers.
- message AfterEndOfWindow {
-
- // (Optional) A trigger governing output prior to the end of the window.
- Trigger early_firings = 1;
-
- // (Optional) A trigger governing output after the end of the window.
- Trigger late_firings = 2;
- }
-
- // After input arrives, ready when the specified delay has passed.
- message AfterProcessingTime {
-
- // (Required) The transforms to apply to an arriving element's timestamp,
- // in order
- repeated TimestampTransform timestamp_transforms = 1;
- }
-
- // Ready whenever upstream processing time has all caught up with
- // the arrival time of an input element
- message AfterSynchronizedProcessingTime {
- }
-
- // The default trigger. Equivalent to Repeat { AfterEndOfWindow } but
- // specially denoted to indicate the user did not alter the triggering.
- message Default {
- }
-
- // Ready whenever the requisite number of input elements have arrived
- message ElementCount {
- int32 element_count = 1;
- }
-
- // Never ready. There will only be an ON_TIME output and a final
- // output at window expiration.
- message Never {
- }
-
- // Always ready. This can also be expressed as ElementCount(1) but
- // is more explicit.
- message Always {
- }
-
- // Ready whenever either of its subtriggers are ready, but finishes output
- // when the finally subtrigger fires.
- message OrFinally {
-
- // (Required) Trigger governing main output; may fire repeatedly.
- Trigger main = 1;
-
- // (Required) Trigger governing termination of output.
- Trigger finally = 2;
- }
-
- // Ready whenever the subtrigger is ready; resets state when the subtrigger
- // completes.
- message Repeat {
- // (Require) Trigger that is run repeatedly.
- Trigger subtrigger = 1;
- }
-
- // The full disjoint union of possible triggers.
- oneof trigger {
- AfterAll after_all = 1;
- AfterAny after_any = 2;
- AfterEach after_each = 3;
- AfterEndOfWindow after_end_of_window = 4;
- AfterProcessingTime after_processing_time = 5;
- AfterSynchronizedProcessingTime after_synchronized_processing_time = 6;
- Always always = 12;
- Default default = 7;
- ElementCount element_count = 8;
- Never never = 9;
- OrFinally or_finally = 10;
- Repeat repeat = 11;
- }
-}
-
-// A specification for a transformation on a timestamp.
-//
-// Primarily used by AfterProcessingTime triggers to transform
-// the arrival time of input to a target time for firing.
-message TimestampTransform {
- oneof timestamp_transform {
- Delay delay = 1;
- AlignTo align_to = 2;
- }
-
- message Delay {
- // (Required) The delay, in milliseconds.
- int64 delay_millis = 1;
- }
-
- message AlignTo {
- // (Required) A duration to which delays should be quantized
- // in milliseconds.
- int64 period = 3;
-
- // (Required) An offset from 0 for the quantization specified by
- // alignment_size, in milliseconds
- int64 offset = 4;
- }
-}
-
-// A specification for how to "side input" a PCollection.
-message SideInput {
- // (Required) URN of the access pattern required by the `view_fn` to present
- // the desired SDK-specific interface to a UDF.
- //
- // This access pattern defines the SDK harness <-> Runner Harness RPC
- // interface for accessing a side input.
- //
- // The only access pattern intended for Beam, because of its superior
- // performance possibilities, is "urn:beam:sideinput:multimap" (or some such
- // URN)
- FunctionSpec access_pattern = 1;
-
- // (Required) The SdkFunctionSpec of the UDF that adapts a particular
- // access_pattern to a user-facing view type.
- //
- // For example, View.asSingleton() may include a `view_fn` that adapts a
- // specially-designed multimap to a single value per window.
- SdkFunctionSpec view_fn = 2;
-
- // (Required) The SdkFunctionSpec of the UDF that maps a main input window
- // to a side input window.
- //
- // For example, when the main input is in fixed windows of one hour, this
- // can specify that the side input should be accessed according to the day
- // in which that hour falls.
- SdkFunctionSpec window_mapping_fn = 3;
-}
-
-// An environment for executing UDFs. Generally an SDK container URL, but
-// there can be many for a single SDK, for example to provide dependency
-// isolation.
-message Environment {
-
- // (Required) The URL of a container
- //
- // TODO: reconcile with Fn API's DockerContainer structure by
- // adding adequate metadata to know how to interpret the container
- string url = 1;
-}
-
-// A specification of a user defined function.
-//
-message SdkFunctionSpec {
-
- // (Required) A full specification of this function.
- FunctionSpec spec = 1;
-
- // (Required) Reference to an execution environment capable of
- // invoking this function.
- string environment_id = 2;
-}
-
-// A URN along with a parameter object whose schema is determined by the
-// URN.
-//
-// This structure is reused in two distinct, but compatible, ways:
-//
-// 1. This can be a specification of the function over PCollections
-// that a PTransform computes.
-// 2. This can be a specification of a user-defined function, possibly
-// SDK-specific. (external to this message must be adequate context
-// to indicate the environment in which the UDF can be understood).
-//
-// Though not explicit in this proto, there are two possibilities
-// for the relationship of a runner to this specification that
-// one should bear in mind:
-//
-// 1. The runner understands the URN. For example, it might be
-// a well-known URN like "urn:beam:transform:Top" or
-// "urn:beam:windowfn:FixedWindows" with
-// an agreed-upon payload (e.g. a number or duration,
-// respectively).
-// 2. The runner does not understand the URN. It might be an
-// SDK specific URN such as "urn:beam:dofn:javasdk:1.0"
-// that indicates to the SDK what the payload is,
-// such as a serialized Java DoFn from a particular
-// version of the Beam Java SDK. The payload will often
-// then be an opaque message such as bytes in a
-// language-specific serialization format.
-message FunctionSpec {
-
- // (Required) A URN that describes the accompanying payload.
- // For any URN that is not recognized (by whomever is inspecting
- // it) the parameter payload should be treated as opaque and
- // passed as-is.
- string urn = 1;
-
- // (Optional) The data specifying any parameters to the URN. If
- // the URN does not require any arguments, this may be omitted.
- bytes payload = 3;
-}
-
-// TODO: transfer javadoc here
-message DisplayData {
-
- // (Required) The list of display data.
- repeated Item items = 1;
-
- // A complete identifier for a DisplayData.Item
- message Identifier {
-
- // (Required) The transform originating this display data.
- string transform_id = 1;
-
- // (Optional) The URN indicating the type of the originating transform,
- // if there is one.
- string transform_urn = 2;
-
- string key = 3;
- }
-
- // A single item of display data.
- message Item {
- // (Required)
- Identifier id = 1;
-
- // (Required)
- Type.Enum type = 2;
-
- // (Required)
- google.protobuf.Any value = 3;
-
- // (Optional)
- google.protobuf.Any short_value = 4;
-
- // (Optional)
- string label = 5;
-
- // (Optional)
- string link_url = 6;
- }
-
- message Type {
- enum Enum {
- UNSPECIFIED = 0;
- STRING = 1;
- INTEGER = 2;
- FLOAT = 3;
- BOOLEAN = 4;
- TIMESTAMP = 5;
- DURATION = 6;
- JAVA_CLASS = 7;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/runner-api/src/main/proto/endpoints.proto
----------------------------------------------------------------------
diff --git a/sdks/common/runner-api/src/main/proto/endpoints.proto b/sdks/common/runner-api/src/main/proto/endpoints.proto
deleted file mode 100644
index a642e63..0000000
--- a/sdks/common/runner-api/src/main/proto/endpoints.proto
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing endpoints containing a service.
- */
-
-syntax = "proto3";
-
-package org.apache.beam.portability.v1;
-
-option java_package = "org.apache.beam.portability.v1";
-option java_outer_classname = "Endpoints";
-
-message ApiServiceDescriptor {
- // (Required) The URL to connect to.
- string url = 2;
-
- // (Optional) The method for authentication. If unspecified, access to the
- // url is already being performed in a trusted context (e.g. localhost,
- // private network).
- oneof authentication {
- OAuth2ClientCredentialsGrant oauth2_client_credentials_grant = 3;
- }
-}
-
-message OAuth2ClientCredentialsGrant {
- // (Required) The URL to submit a "client_credentials" grant type request for
- // an OAuth access token which will be used as a bearer token for requests.
- string url = 1;
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/runner-api/src/main/proto/standard_window_fns.proto
----------------------------------------------------------------------
diff --git a/sdks/common/runner-api/src/main/proto/standard_window_fns.proto b/sdks/common/runner-api/src/main/proto/standard_window_fns.proto
deleted file mode 100644
index 0682044..0000000
--- a/sdks/common/runner-api/src/main/proto/standard_window_fns.proto
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Runner API, which is the runner-independent,
- * SDK-independent definition of the Beam model.
- */
-
-syntax = "proto3";
-
-package org.apache.beam.runner_api.v1;
-
-option java_package = "org.apache.beam.sdk.common.runner.v1";
-option java_outer_classname = "StandardWindowFns";
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/timestamp.proto";
-
-// beam:windowfn:global_windows:v0.1
-// empty payload
-
-// beam:windowfn:fixed_windows:v0.1
-message FixedWindowsPayload {
- google.protobuf.Duration size = 1;
- google.protobuf.Timestamp offset = 2;
-}
-
-// beam:windowfn:sliding_windows:v0.1
-message SlidingWindowsPayload {
- google.protobuf.Duration size = 1;
- google.protobuf.Timestamp offset = 2;
- google.protobuf.Duration period = 3;
-}
-
-// beam:windowfn:session_windows:v0.1
-message SessionsPayload {
- google.protobuf.Duration gap_size = 1;
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index cc01c83..36271eb 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -309,7 +309,7 @@
<!-- We only rely on the standard_coders.yaml -->
<exclusion>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/java/harness/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/harness/pom.xml b/sdks/java/harness/pom.xml
index e0449ab..7f9156c 100644
--- a/sdks/java/harness/pom.xml
+++ b/sdks/java/harness/pom.xml
@@ -50,10 +50,10 @@
<includes>
<include>com.google.guava:guava</include>
<!-- java harness dependencies that are not staged -->
+ <include>org.apache.beam:beam-model-pipeline</include>
<include>org.apache.beam:beam-runners-core-construction-java</include>
<include>org.apache.beam:beam-runners-core-java</include>
<include>org.apache.beam:beam-runners-google-cloud-dataflow-java</include>
- <include>org.apache.beam:beam-sdks-common-runner-api</include>
<include>org.apache.beam:beam-sdks-common-fn-api</include>
<include>io.netty:netty-transport-native-epoll</include>
</includes>
@@ -126,6 +126,11 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
@@ -160,11 +165,6 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-common-fn-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/python/gen_protos.py
----------------------------------------------------------------------
diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
index f5bb197..4ca3561 100644
--- a/sdks/python/gen_protos.py
+++ b/sdks/python/gen_protos.py
@@ -35,7 +35,6 @@ BEAM_PROTO_PATHS = [
os.path.join('..', '..', 'model', 'pipeline', 'src', 'main', 'proto'),
os.path.join('..', '..', 'model', 'job-management', 'src', 'main', 'proto'),
os.path.join('..', '..', 'model', 'fn-execution', 'src', 'main', 'proto'),
- os.path.join('..', 'common', 'runner-api', 'src', 'main', 'proto'),
os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto')
]
[02/12] beam git commit: Remove sdks/common
Posted by ke...@apache.org.
Remove sdks/common
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3a9a435f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3a9a435f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3a9a435f
Branch: refs/heads/master
Commit: 3a9a435fe2d745895bb3da32d2ff72d1d0b2e599
Parents: e233af9
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Oct 5 15:07:01 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:53:37 2017 -0700
----------------------------------------------------------------------
sdks/common/pom.xml | 37 -------------------------------------
sdks/pom.xml | 1 -
2 files changed, 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3a9a435f/sdks/common/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/common/pom.xml b/sdks/common/pom.xml
deleted file mode 100644
index d4ecad0..0000000
--- a/sdks/common/pom.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-parent</artifactId>
- <version>2.3.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>beam-sdks-common-parent</artifactId>
-
- <packaging>pom</packaging>
-
- <name>Apache Beam :: SDKs :: Common</name>
-
- <modules>
- </modules>
-</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/3a9a435f/sdks/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/pom.xml b/sdks/pom.xml
index 4c337f6..7c85489 100644
--- a/sdks/pom.xml
+++ b/sdks/pom.xml
@@ -33,7 +33,6 @@
<name>Apache Beam :: SDKs</name>
<modules>
- <module>common</module>
<module>go</module>
<module>java</module>
<module>python</module>
[09/12] beam git commit: Improve error message when
standard_coders.yaml not found
Posted by ke...@apache.org.
Improve error message when standard_coders.yaml not found
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7fa37fd9
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7fa37fd9
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7fa37fd9
Branch: refs/heads/master
Commit: 7fa37fd9b5225d83d8df8ee26043d78aba9bb1b5
Parents: 80c6f4e
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Oct 9 19:33:21 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:56:39 2017 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/7fa37fd9/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
index 8adc08c..d50bc0a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
@@ -130,7 +130,7 @@ public class CommonCoderTest {
private static List<OneCoderTestSpec> loadStandardCodersSuite() throws IOException {
InputStream stream = CommonCoderTest.class.getResourceAsStream(STANDARD_CODERS_YAML_PATH);
if (stream == null) {
- fail("null stream");
+ fail("Could not load standard coder specs as resource:" + STANDARD_CODERS_YAML_PATH);
}
// Would like to use the InputStream directly with Jackson, but Jackson does not seem to
[05/12] beam git commit: Move sdk/common/runner-api protos into
model/pipeline and model/job-management
Posted by ke...@apache.org.
Move sdk/common/runner-api protos into model/pipeline and model/job-management
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2f3af318
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2f3af318
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2f3af318
Branch: refs/heads/master
Commit: 2f3af3188193e690d36ab892b3138f2fccad93ec
Parents: fde0b93
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Oct 5 13:57:49 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:53:37 2017 -0700
----------------------------------------------------------------------
model/job-management/pom.xml | 32 +
.../src/main/proto/beam_artifact_api.proto | 133 +++
.../src/main/proto/beam_job_api.proto | 173 ++++
model/pipeline/pom.xml | 7 +
.../src/main/proto/beam_runner_api.proto | 842 +++++++++++++++++++
model/pipeline/src/main/proto/endpoints.proto | 46 +
.../src/main/proto/standard_window_fns.proto | 53 ++
pom.xml | 6 -
runners/apex/pom.xml | 2 +-
runners/core-construction-java/pom.xml | 7 +-
runners/core-java/pom.xml | 4 +-
runners/direct-java/pom.xml | 6 +-
runners/flink/pom.xml | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 6 +-
runners/local-artifact-service-java/pom.xml | 2 +-
runners/reference/job-server/pom.xml | 2 +-
runners/reference/pom.xml | 1 -
runners/spark/pom.xml | 2 +-
sdks/common/fn-api/pom.xml | 2 +-
sdks/common/pom.xml | 1 -
sdks/common/runner-api/pom.xml | 109 ---
.../src/main/proto/beam_artifact_api.proto | 133 ---
.../src/main/proto/beam_job_api.proto | 173 ----
.../src/main/proto/beam_runner_api.proto | 842 -------------------
.../runner-api/src/main/proto/endpoints.proto | 46 -
.../src/main/proto/standard_window_fns.proto | 53 --
sdks/java/core/pom.xml | 2 +-
sdks/java/harness/pom.xml | 12 +-
sdks/python/gen_protos.py | 1 -
29 files changed, 1313 insertions(+), 1387 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/model/job-management/pom.xml
----------------------------------------------------------------------
diff --git a/model/job-management/pom.xml b/model/job-management/pom.xml
index 3f7fd18..580188c 100644
--- a/model/job-management/pom.xml
+++ b/model/job-management/pom.xml
@@ -79,4 +79,36 @@
</plugin>
</plugins>
</build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/model/job-management/src/main/proto/beam_artifact_api.proto
----------------------------------------------------------------------
diff --git a/model/job-management/src/main/proto/beam_artifact_api.proto b/model/job-management/src/main/proto/beam_artifact_api.proto
new file mode 100644
index 0000000..e11551c
--- /dev/null
+++ b/model/job-management/src/main/proto/beam_artifact_api.proto
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Artifact API, for communicating with a runner
+ * for artifact staging and retrieval over GRPC.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.runner_api.v1;
+
+option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_outer_classname = "ArtifactApi";
+
+// A service to stage artifacts for use in a Job.
+//
+// RPCs made to an ArtifactStagingService endpoint should include some form of identification for
+// the job as a header.
+service ArtifactStagingService {
+ // Stage an artifact to be available during job execution. The first request must contain the
+ // name of the artifact. All future requests must contain sequential chunks of the content of
+ // the artifact.
+ rpc PutArtifact(stream PutArtifactRequest) returns (PutArtifactResponse);
+
+ // Commit the manifest for a Job. All artifacts must have been successfully uploaded
+ // before this call is made.
+ //
+ // Throws error INVALID_ARGUMENT if not all of the members of the manifest are present
+ rpc CommitManifest(CommitManifestRequest) returns (CommitManifestResponse);
+}
+
+// A service to retrieve artifacts for use in a Job.
+service ArtifactRetrievalService {
+ // Get the manifest for the job
+ rpc GetManifest(GetManifestRequest) returns (GetManifestResponse);
+
+ // Get an artifact staged for the job. The requested artifact must be within the manifest
+ rpc GetArtifact(GetArtifactRequest) returns (stream ArtifactChunk);
+}
+
+// An artifact identifier and associated metadata.
+message ArtifactMetadata {
+ // (Required) The name of the artifact.
+ string name = 1;
+
+ // (Optional) The Unix-like permissions of the artifact
+ uint32 permissions = 2;
+
+ // (Optional) The base64-encoded md5 checksum of the artifact. Used, among other things, by
+ // harness boot code to validate the integrity of the artifact.
+ string md5 = 3;
+}
+
+// A collection of artifacts.
+message Manifest {
+ repeated ArtifactMetadata artifact = 1;
+}
+
+// A manifest with location information.
+message ProxyManifest {
+ Manifest manifest = 1;
+ message Location {
+ string name = 1;
+ string uri = 2;
+ }
+ repeated Location location = 2;
+}
+
+// A request to get the manifest of a Job.
+message GetManifestRequest {}
+
+// A response containing a job manifest.
+message GetManifestResponse {
+ Manifest manifest = 1;
+}
+
+// A request to get an artifact. The artifact must be present in the manifest for the job.
+message GetArtifactRequest {
+ // (Required) The name of the artifact to retrieve.
+ string name = 1;
+}
+
+// Part of an artifact.
+message ArtifactChunk {
+ bytes data = 1;
+}
+
+// A request to stage an artifact.
+message PutArtifactRequest {
+ // (Required)
+ oneof content {
+ // The Artifact metadata. The first message in a PutArtifact call must contain the name
+ // of the artifact.
+ ArtifactMetadata metadata = 1;
+
+ // A chunk of the artifact. All messages after the first in a PutArtifact call must contain a
+ // chunk.
+ ArtifactChunk data = 2;
+ }
+}
+
+message PutArtifactResponse {
+}
+
+// A request to commit the manifest for a Job. All artifacts must have been successfully uploaded
+// before this call is made.
+message CommitManifestRequest {
+ // (Required) The manifest to commit.
+ Manifest manifest = 1;
+}
+
+// The result of committing a manifest.
+message CommitManifestResponse {
+ // (Required) An opaque token representing the entirety of the staged artifacts.
+ string staging_token = 1;
+}
+
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/model/job-management/src/main/proto/beam_job_api.proto
----------------------------------------------------------------------
diff --git a/model/job-management/src/main/proto/beam_job_api.proto b/model/job-management/src/main/proto/beam_job_api.proto
new file mode 100644
index 0000000..8f2ed0e
--- /dev/null
+++ b/model/job-management/src/main/proto/beam_job_api.proto
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Job API, api for communicating with a runner
+ * for job submission over GRPC.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.runner_api.v1;
+
+option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_outer_classname = "JobApi";
+
+import "beam_runner_api.proto";
+import "endpoints.proto";
+import "google/protobuf/struct.proto";
+
+
+// Job Service for running RunnerAPI pipelines
+service JobService {
+ // Prepare a job for execution. The job will not be executed until a call is made to run with the
+ // returned preparationId.
+ rpc Prepare (PrepareJobRequest) returns (PrepareJobResponse);
+
+ // Submit the job for execution
+ rpc Run (RunJobRequest) returns (RunJobResponse);
+
+ // Get the current state of the job
+ rpc GetState (GetJobStateRequest) returns (GetJobStateResponse);
+
+ // Cancel the job
+ rpc Cancel (CancelJobRequest) returns (CancelJobResponse);
+
+ // Subscribe to a stream of state changes of the job, will immediately return the current state of the job as the first response.
+ rpc GetStateStream (GetJobStateRequest) returns (stream GetJobStateResponse);
+
+ // Subscribe to a stream of state changes and messages from the job
+ rpc GetMessageStream (JobMessagesRequest) returns (stream JobMessagesResponse);
+}
+
+
+// Prepare is a synchronous request that returns a preparationId back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error ALREADY_EXISTS if the jobName is reused. Runners are permitted to deduplicate based on the name of the job.
+// Throws error UNKNOWN for all other issues
+message PrepareJobRequest {
+ org.apache.beam.runner_api.v1.Pipeline pipeline = 1; // (required)
+ google.protobuf.Struct pipeline_options = 2; // (required)
+ string job_name = 3; // (required)
+}
+
+message PrepareJobResponse {
+ // (required) The ID used to associate calls made while preparing the job. preparationId is used
+ // to run the job, as well as in other pre-execution APIs such as Artifact staging.
+ string preparation_id = 1;
+
+ // An endpoint which exposes the Beam Artifact Staging API. Artifacts used by the job should be
+ // staged to this endpoint, and will be available during job execution.
+ org.apache.beam.portability.v1.ApiServiceDescriptor artifact_staging_endpoint = 2;
+}
+
+
+// Run is a synchronous request that returns a jobId back.
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the preparation ID does not exist
+// Throws error UNKNOWN for all other issues
+message RunJobRequest {
+ // (required) The ID provided by an earlier call to prepare. Runs the job. All prerequisite tasks
+ // must have been completed.
+ string preparation_id = 1;
+ // (optional) If any artifacts have been staged for this job, contains the staging_token returned
+ // from the CommitManifestResponse.
+ string staging_token = 2;
+}
+
+
+message RunJobResponse {
+ string job_id = 1; // (required) The ID for the executing job
+}
+
+
+// Cancel is a synchronus request that returns a job state back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the jobId is not found
+message CancelJobRequest {
+ string job_id = 1; // (required)
+
+}
+
+// Valid responses include any terminal state or CANCELLING
+message CancelJobResponse {
+ JobState.Enum state = 1; // (required)
+}
+
+
+// GetState is a synchronus request that returns a job state back
+// Throws error GRPC_STATUS_UNAVAILABLE if server is down
+// Throws error NOT_FOUND if the jobId is not found
+message GetJobStateRequest {
+ string job_id = 1; // (required)
+
+}
+
+message GetJobStateResponse {
+ JobState.Enum state = 1; // (required)
+}
+
+
+// GetJobMessages is a streaming api for streaming job messages from the service
+// One request will connect you to the job and you'll get a stream of job state
+// and job messages back; one is used for logging and the other for detecting
+// the job ended.
+message JobMessagesRequest {
+ string job_id = 1; // (required)
+
+}
+
+message JobMessage {
+ string message_id = 1;
+ string time = 2;
+ MessageImportance importance = 3;
+ string message_text = 4;
+
+ enum MessageImportance {
+ MESSAGE_IMPORTANCE_UNSPECIFIED = 0;
+ JOB_MESSAGE_DEBUG = 1;
+ JOB_MESSAGE_DETAILED = 2;
+ JOB_MESSAGE_BASIC = 3;
+ JOB_MESSAGE_WARNING = 4;
+ JOB_MESSAGE_ERROR = 5;
+ }
+}
+
+message JobMessagesResponse {
+ oneof response {
+ JobMessage message_response = 1;
+ GetJobStateResponse state_response = 2;
+ }
+}
+
+// Enumeration of all JobStates
+message JobState {
+ enum Enum {
+ UNSPECIFIED = 0;
+ STOPPED = 1;
+ RUNNING = 2;
+ DONE = 3;
+ FAILED = 4;
+ CANCELLED = 5;
+ UPDATED = 6;
+ DRAINING = 7;
+ DRAINED = 8;
+ STARTING = 9;
+ CANCELLING = 10;
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/model/pipeline/pom.xml
----------------------------------------------------------------------
diff --git a/model/pipeline/pom.xml b/model/pipeline/pom.xml
index ed786c0..21d97a2 100644
--- a/model/pipeline/pom.xml
+++ b/model/pipeline/pom.xml
@@ -79,4 +79,11 @@
</plugin>
</plugins>
</build>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/model/pipeline/src/main/proto/beam_runner_api.proto
----------------------------------------------------------------------
diff --git a/model/pipeline/src/main/proto/beam_runner_api.proto b/model/pipeline/src/main/proto/beam_runner_api.proto
new file mode 100644
index 0000000..74f3897
--- /dev/null
+++ b/model/pipeline/src/main/proto/beam_runner_api.proto
@@ -0,0 +1,842 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Runner API, which is the runner-independent,
+ * SDK-independent definition of the Beam model.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.runner_api.v1;
+
+option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_outer_classname = "RunnerApi";
+
+import "google/protobuf/any.proto";
+
+// A set of mappings from id to message. This is included as an optional field
+// on any proto message that may contain references needing resolution.
+message Components {
+ // (Required) A map from pipeline-scoped id to PTransform.
+ map<string, PTransform> transforms = 1;
+
+ // (Required) A map from pipeline-scoped id to PCollection.
+ map<string, PCollection> pcollections = 2;
+
+ // (Required) A map from pipeline-scoped id to WindowingStrategy.
+ map<string, WindowingStrategy> windowing_strategies = 3;
+
+ // (Required) A map from pipeline-scoped id to Coder.
+ map<string, Coder> coders = 4;
+
+ // (Required) A map from pipeline-scoped id to Environment.
+ map<string, Environment> environments = 5;
+}
+
+// A disjoint union of all the things that may contain references
+// that require Components to resolve.
+message MessageWithComponents {
+
+ // (Optional) The by-reference components of the root message,
+ // enabling a standalone message.
+ //
+ // If this is absent, it is expected that there are no
+ // references.
+ Components components = 1;
+
+ // (Required) The root message that may contain pointers
+ // that should be resolved by looking inside components.
+ oneof root {
+ Coder coder = 2;
+ CombinePayload combine_payload = 3;
+ SdkFunctionSpec sdk_function_spec = 4;
+ ParDoPayload par_do_payload = 6;
+ PTransform ptransform = 7;
+ PCollection pcollection = 8;
+ ReadPayload read_payload = 9;
+ SideInput side_input = 11;
+ WindowIntoPayload window_into_payload = 12;
+ WindowingStrategy windowing_strategy = 13;
+ FunctionSpec function_spec = 14;
+ }
+}
+
+// A Pipeline is a hierarchical graph of PTransforms, linked
+// by PCollections.
+//
+// This is represented by a number of by-reference maps to nodes,
+// PCollections, SDK environments, UDF, etc., for
+// supporting compact reuse and arbitrary graph structure.
+//
+// All of the keys in the maps here are arbitrary strings that are only
+// required to be internally consistent within this proto message.
+message Pipeline {
+
+ // (Required) The coders, UDFs, graph nodes, etc, that make up
+ // this pipeline.
+ Components components = 1;
+
+ // (Required) The ids of all PTransforms that are not contained within another PTransform.
+ // These must be in shallow topological order, so that traversing them recursively
+ // in this order yields a recursively topological traversal.
+ repeated string root_transform_ids = 2;
+
+ // (Optional) Static display data for the pipeline. If there is none,
+ // it may be omitted.
+ DisplayData display_data = 3;
+}
+
+// An applied PTransform! This does not contain the graph data, but only the
+// fields specific to a graph node that is a Runner API transform
+// between PCollections.
+message PTransform {
+
+ // (Required) A unique name for the application node.
+ //
+ // Ideally, this should be stable over multiple evolutions of a pipeline
+ // for the purposes of logging and associating pipeline state with a node,
+ // etc.
+ //
+ // If it is not stable, then the runner decides what will happen. But, most
+ // importantly, it must always be here and be unique, even if it is
+ // autogenerated.
+ string unique_name = 5;
+
+ // (Optional) A URN and payload that, together, fully defined the semantics
+ // of this transform.
+ //
+ // If absent, this must be an "anonymous" composite transform.
+ //
+ // For primitive transform in the Runner API, this is required, and the
+ // payloads are well-defined messages. When the URN indicates ParDo it
+ // is a ParDoPayload, and so on.
+ //
+ // TODO: document the standardized URNs and payloads
+ // TODO: separate standardized payloads into a separate proto file
+ //
+ // For some special composite transforms, the payload is also officially
+ // defined:
+ //
+ // - when the URN is "urn:beam:transforms:combine" it is a CombinePayload
+ //
+ FunctionSpec spec = 1;
+
+ // (Optional) if this node is a composite, a list of the ids of
+ // transforms that it contains.
+ repeated string subtransforms = 2;
+
+ // (Required) A map from local names of inputs (unique only with this map, and
+ // likely embedded in the transform payload and serialized user code) to
+ // PCollection ids.
+ //
+ // The payload for this transform may clarify the relationship of these
+ // inputs. For example:
+ //
+ // - for a Flatten transform they are merged
+ // - for a ParDo transform, some may be side inputs
+ //
+ // All inputs are recorded here so that the topological ordering of
+ // the graph is consistent whether or not the payload is understood.
+ //
+ map<string, string> inputs = 3;
+
+ // (Required) A map from local names of outputs (unique only within this map,
+ // and likely embedded in the transform payload and serialized user code)
+ // to PCollection ids.
+ //
+ // The URN or payload for this transform node may clarify the type and
+ // relationship of these outputs. For example:
+ //
+ // - for a ParDo transform, these are tags on PCollections, which will be
+ // embedded in the DoFn.
+ //
+ map<string, string> outputs = 4;
+
+ // (Optional) Static display data for this PTransform application. If
+ // there is none, or it is not relevant (such as use by the Fn API)
+ // then it may be omitted.
+ DisplayData display_data = 6;
+}
+
+// A PCollection!
+message PCollection {
+
+ // (Required) A unique name for the PCollection.
+ //
+ // Ideally, this should be stable over multiple evolutions of a pipeline
+ // for the purposes of logging and associating pipeline state with a node,
+ // etc.
+ //
+ // If it is not stable, then the runner decides what will happen. But, most
+ // importantly, it must always be here, even if it is autogenerated.
+ string unique_name = 1;
+
+ // (Required) The id of the Coder for this PCollection.
+ string coder_id = 2;
+
+ // (Required) Whether this PCollection is bounded or unbounded
+ IsBounded.Enum is_bounded = 3;
+
+ // (Required) The id of the windowing strategy for this PCollection.
+ string windowing_strategy_id = 4;
+
+ // (Optional) Static display data for this PTransform application. If
+ // there is none, or it is not relevant (such as use by the Fn API)
+ // then it may be omitted.
+ DisplayData display_data = 5;
+}
+
+// The payload for the primitive ParDo transform.
+message ParDoPayload {
+
+ // (Required) The SdkFunctionSpec of the DoFn.
+ SdkFunctionSpec do_fn = 1;
+
+ // (Required) Additional pieces of context the DoFn may require that
+ // are not otherwise represented in the payload.
+ // (may force runners to execute the ParDo differently)
+ repeated Parameter parameters = 2;
+
+ // (Optional) A mapping of local input names to side inputs, describing
+ // the expected access pattern.
+ map<string, SideInput> side_inputs = 3;
+
+ // (Optional) A mapping of local state names to state specifications.
+ map<string, StateSpec> state_specs = 4;
+
+ // (Optional) A mapping of local timer names to timer specifications.
+ map<string, TimerSpec> timer_specs = 5;
+
+ // Whether the DoFn is splittable
+ bool splittable = 6;
+}
+
+// Parameters that a UDF might require.
+//
+// The details of how a runner sends these parameters to the SDK harness
+// are the subject of the Fn API.
+//
+// The details of how an SDK harness delivers them to the UDF is entirely
+// up to the SDK. (for some SDKs there may be parameters that are not
+// represented here if the runner doesn't need to do anything)
+//
+// Here, the parameters are simply indicators to the runner that they
+// need to run the function a particular way.
+//
+// TODO: the evolution of the Fn API will influence what needs explicit
+// representation here
+message Parameter {
+ Type.Enum type = 1;
+
+ message Type {
+ enum Enum {
+ UNSPECIFIED = 0;
+ WINDOW = 1;
+ PIPELINE_OPTIONS = 2;
+ RESTRICTION_TRACKER = 3;
+ }
+ }
+}
+
+message StateSpec {
+ oneof spec {
+ ValueStateSpec value_spec = 1;
+ BagStateSpec bag_spec = 2;
+ CombiningStateSpec combining_spec = 3;
+ MapStateSpec map_spec = 4;
+ SetStateSpec set_spec = 5;
+ }
+}
+
+message ValueStateSpec {
+ string coder_id = 1;
+}
+
+message BagStateSpec {
+ string element_coder_id = 1;
+}
+
+message CombiningStateSpec {
+ string accumulator_coder_id = 1;
+ SdkFunctionSpec combine_fn = 2;
+}
+
+message MapStateSpec {
+ string key_coder_id = 1;
+ string value_coder_id = 2;
+}
+
+message SetStateSpec {
+ string element_coder_id = 1;
+}
+
+message TimerSpec {
+ TimeDomain.Enum time_domain = 1;
+}
+
+message IsBounded {
+ enum Enum {
+ UNSPECIFIED = 0;
+ UNBOUNDED = 1;
+ BOUNDED = 2;
+ }
+}
+
+// The payload for the primitive Read transform.
+message ReadPayload {
+
+ // (Required) The SdkFunctionSpec of the source for this Read.
+ SdkFunctionSpec source = 1;
+
+ // (Required) Whether the source is bounded or unbounded
+ IsBounded.Enum is_bounded = 2;
+
+ // TODO: full audit of fields required by runners as opposed to SDK harness
+}
+
+// The payload for the WindowInto transform.
+message WindowIntoPayload {
+
+ // (Required) The SdkFunctionSpec of the WindowFn.
+ SdkFunctionSpec window_fn = 1;
+}
+
+// The payload for the special-but-not-primitive Combine transform.
+message CombinePayload {
+
+ // (Required) The SdkFunctionSpec of the CombineFn.
+ SdkFunctionSpec combine_fn = 1;
+
+ // (Required) A reference to the Coder to use for accumulators of the CombineFn
+ string accumulator_coder_id = 2;
+
+ // (Required) Additional pieces of context the DoFn may require that
+ // are not otherwise represented in the payload.
+ // (may force runners to execute the ParDo differently)
+ repeated Parameter parameters = 3;
+
+ // (Optional) A mapping of local input names to side inputs, describing
+ // the expected access pattern.
+ map<string, SideInput> side_inputs = 4;
+}
+
+// The payload for the test-only primitive TestStream
+message TestStreamPayload {
+
+ // (Required) the coder for elements in the TestStream events
+ string coder_id = 1;
+
+ repeated Event events = 2;
+
+ message Event {
+ oneof event {
+ AdvanceWatermark watermark_event = 1;
+ AdvanceProcessingTime processing_time_event = 2;
+ AddElements element_event = 3;
+ }
+
+ message AdvanceWatermark {
+ int64 new_watermark = 1;
+ }
+
+ message AdvanceProcessingTime {
+ int64 advance_duration = 1;
+ }
+
+ message AddElements {
+ repeated TimestampedElement elements = 1;
+ }
+ }
+
+ message TimestampedElement {
+ bytes encoded_element = 1;
+ int64 timestamp = 2;
+ }
+}
+// The payload for the special-but-not-primitive WriteFiles transform.
+message WriteFilesPayload {
+
+ // (Required) The SdkFunctionSpec of the FileBasedSink.
+ SdkFunctionSpec sink = 1;
+
+ // (Required) The format function.
+ SdkFunctionSpec format_function = 2;
+
+ bool windowed_writes = 3;
+
+ bool runner_determined_sharding = 4;
+
+ map<string, SideInput> side_inputs = 5;
+}
+
+// A coder, the binary format for serialization and deserialization of data in
+// a pipeline.
+message Coder {
+
+ // (Required) A specification for the coder, as a URN plus parameters. This
+ // may be a cross-language agreed-upon format, or it may be a "custom coder"
+ // that can only be used by a particular SDK. It does not include component
+ // coders, as it is beneficial for these to be comprehensible to a runner
+ // regardless of whether the binary format is agree-upon.
+ SdkFunctionSpec spec = 1;
+
+ // (Optional) If this coder is parametric, such as ListCoder(VarIntCoder),
+ // this is a list of the components. In order for encodings to be identical,
+ // the SdkFunctionSpec and all components must be identical, recursively.
+ repeated string component_coder_ids = 2;
+}
+
+// A windowing strategy describes the window function, triggering, allowed
+// lateness, and accumulation mode for a PCollection.
+//
+// TODO: consider inlining field on PCollection
+message WindowingStrategy {
+
+ // (Required) The SdkFunctionSpec of the UDF that assigns windows,
+ // merges windows, and shifts timestamps before they are
+ // combined according to the OutputTime.
+ SdkFunctionSpec window_fn = 1;
+
+ // (Required) Whether or not the window fn is merging.
+ //
+ // This knowledge is required for many optimizations.
+ MergeStatus.Enum merge_status = 2;
+
+ // (Required) The coder for the windows of this PCollection.
+ string window_coder_id = 3;
+
+ // (Required) The trigger to use when grouping this PCollection.
+ Trigger trigger = 4;
+
+ // (Required) The accumulation mode indicates whether new panes are a full
+ // replacement for prior panes or whether they are deltas to be combined
+ // with other panes (the combine should correspond to whatever the upstream
+ // grouping transform is).
+ AccumulationMode.Enum accumulation_mode = 5;
+
+ // (Required) The OutputTime specifies, for a grouping transform, how to
+ // compute the aggregate timestamp. The window_fn will first possibly shift
+ // it later, then the OutputTime takes the max, min, or ignores it and takes
+ // the end of window.
+ //
+ // This is actually only for input to grouping transforms, but since they
+ // may be introduced in runner-specific ways, it is carried along with the
+ // windowing strategy.
+ OutputTime.Enum output_time = 6;
+
+ // (Required) Indicate when output should be omitted upon window expiration.
+ ClosingBehavior.Enum closing_behavior = 7;
+
+ // (Required) The duration, in milliseconds, beyond the end of a window at
+ // which the window becomes droppable.
+ int64 allowed_lateness = 8;
+
+ // (Required) Indicate whether empty on-time panes should be omitted.
+ OnTimeBehavior.Enum OnTimeBehavior = 9;
+
+ // (Required) Whether or not the window fn assigns inputs to exactly one window
+ //
+ // This knowledge is required for some optimizations
+ bool assigns_to_one_window = 10;
+}
+
+// Whether or not a PCollection's WindowFn is non-merging, merging, or
+// merging-but-already-merged, in which case a subsequent GroupByKey is almost
+// always going to do something the user does not want
+message MergeStatus {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // The WindowFn does not require merging.
+ // Examples: global window, FixedWindows, SlidingWindows
+ NON_MERGING = 1;
+
+ // The WindowFn is merging and the PCollection has not had merging
+ // performed.
+ // Example: Sessions prior to a GroupByKey
+ NEEDS_MERGE = 2;
+
+ // The WindowFn is merging and the PCollection has had merging occur
+ // already.
+ // Example: Sessions after a GroupByKey
+ ALREADY_MERGED = 3;
+ }
+}
+
+// Whether or not subsequent outputs of aggregations should be entire
+// replacement values or just the aggregation of inputs received since
+// the prior output.
+message AccumulationMode {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // The aggregation is discarded when it is output
+ DISCARDING = 1;
+
+ // The aggregation is accumulated across outputs
+ ACCUMULATING = 2;
+ }
+}
+
+// Controls whether or not an aggregating transform should output data
+// when a window expires.
+message ClosingBehavior {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // Emit output when a window expires, whether or not there has been
+ // any new data since the last output.
+ EMIT_ALWAYS = 1;
+
+ // Only emit output when new data has arrives since the last output
+ EMIT_IF_NONEMPTY = 2;
+ }
+}
+
+// Controls whether or not an aggregating transform should output data
+// when an on-time pane is empty.
+message OnTimeBehavior {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // Always fire the on-time pane. Even if there is no new data since
+ // the previous firing, an element will be produced.
+ FIRE_ALWAYS = 1;
+
+ // Only fire the on-time pane if there is new data since the previous firing.
+ FIRE_IF_NONEMPTY = 2;
+ }
+}
+
+// When a number of windowed, timestamped inputs are aggregated, the timestamp
+// for the resulting output.
+message OutputTime {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // The output has the timestamp of the end of the window.
+ END_OF_WINDOW = 1;
+
+ // The output has the latest timestamp of the input elements since
+ // the last output.
+ LATEST_IN_PANE = 2;
+
+ // The output has the earliest timestamp of the input elements since
+ // the last output.
+ EARLIEST_IN_PANE = 3;
+ }
+}
+
+// The different time domains in the Beam model.
+message TimeDomain {
+ enum Enum {
+ UNSPECIFIED = 0;
+
+ // Event time is time from the perspective of the data
+ EVENT_TIME = 1;
+
+ // Processing time is time from the perspective of the
+ // execution of your pipeline
+ PROCESSING_TIME = 2;
+
+ // Synchronized processing time is the minimum of the
+ // processing time of all pending elements.
+ //
+ // The "processing time" of an element refers to
+ // the local processing time at which it was emitted
+ SYNCHRONIZED_PROCESSING_TIME = 3;
+ }
+}
+
+// A small DSL for expressing when to emit new aggregations
+// from a GroupByKey or CombinePerKey
+//
+// A trigger is described in terms of when it is _ready_ to permit output.
+message Trigger {
+
+ // Ready when all subtriggers are ready.
+ message AfterAll {
+ repeated Trigger subtriggers = 1;
+ }
+
+ // Ready when any subtrigger is ready.
+ message AfterAny {
+ repeated Trigger subtriggers = 1;
+ }
+
+ // Starting with the first subtrigger, ready when the _current_ subtrigger
+ // is ready. After output, advances the current trigger by one.
+ message AfterEach {
+ repeated Trigger subtriggers = 1;
+ }
+
+ // Ready after the input watermark is past the end of the window.
+ //
+ // May have implicitly-repeated subtriggers for early and late firings.
+ // When the end of the window is reached, the trigger transitions between
+ // the subtriggers.
+ message AfterEndOfWindow {
+
+ // (Optional) A trigger governing output prior to the end of the window.
+ Trigger early_firings = 1;
+
+ // (Optional) A trigger governing output after the end of the window.
+ Trigger late_firings = 2;
+ }
+
+ // After input arrives, ready when the specified delay has passed.
+ message AfterProcessingTime {
+
+ // (Required) The transforms to apply to an arriving element's timestamp,
+ // in order
+ repeated TimestampTransform timestamp_transforms = 1;
+ }
+
+ // Ready whenever upstream processing time has all caught up with
+ // the arrival time of an input element
+ message AfterSynchronizedProcessingTime {
+ }
+
+ // The default trigger. Equivalent to Repeat { AfterEndOfWindow } but
+ // specially denoted to indicate the user did not alter the triggering.
+ message Default {
+ }
+
+ // Ready whenever the requisite number of input elements have arrived
+ message ElementCount {
+ int32 element_count = 1;
+ }
+
+ // Never ready. There will only be an ON_TIME output and a final
+ // output at window expiration.
+ message Never {
+ }
+
+ // Always ready. This can also be expressed as ElementCount(1) but
+ // is more explicit.
+ message Always {
+ }
+
+ // Ready whenever either of its subtriggers are ready, but finishes output
+ // when the finally subtrigger fires.
+ message OrFinally {
+
+ // (Required) Trigger governing main output; may fire repeatedly.
+ Trigger main = 1;
+
+ // (Required) Trigger governing termination of output.
+ Trigger finally = 2;
+ }
+
+ // Ready whenever the subtrigger is ready; resets state when the subtrigger
+ // completes.
+ message Repeat {
+ // (Require) Trigger that is run repeatedly.
+ Trigger subtrigger = 1;
+ }
+
+ // The full disjoint union of possible triggers.
+ oneof trigger {
+ AfterAll after_all = 1;
+ AfterAny after_any = 2;
+ AfterEach after_each = 3;
+ AfterEndOfWindow after_end_of_window = 4;
+ AfterProcessingTime after_processing_time = 5;
+ AfterSynchronizedProcessingTime after_synchronized_processing_time = 6;
+ Always always = 12;
+ Default default = 7;
+ ElementCount element_count = 8;
+ Never never = 9;
+ OrFinally or_finally = 10;
+ Repeat repeat = 11;
+ }
+}
+
+// A specification for a transformation on a timestamp.
+//
+// Primarily used by AfterProcessingTime triggers to transform
+// the arrival time of input to a target time for firing.
+message TimestampTransform {
+ oneof timestamp_transform {
+ Delay delay = 1;
+ AlignTo align_to = 2;
+ }
+
+ message Delay {
+ // (Required) The delay, in milliseconds.
+ int64 delay_millis = 1;
+ }
+
+ message AlignTo {
+ // (Required) A duration to which delays should be quantized
+ // in milliseconds.
+ int64 period = 3;
+
+ // (Required) An offset from 0 for the quantization specified by
+ // alignment_size, in milliseconds
+ int64 offset = 4;
+ }
+}
+
+// A specification for how to "side input" a PCollection.
+message SideInput {
+ // (Required) URN of the access pattern required by the `view_fn` to present
+ // the desired SDK-specific interface to a UDF.
+ //
+ // This access pattern defines the SDK harness <-> Runner Harness RPC
+ // interface for accessing a side input.
+ //
+ // The only access pattern intended for Beam, because of its superior
+ // performance possibilities, is "urn:beam:sideinput:multimap" (or some such
+ // URN)
+ FunctionSpec access_pattern = 1;
+
+ // (Required) The SdkFunctionSpec of the UDF that adapts a particular
+ // access_pattern to a user-facing view type.
+ //
+ // For example, View.asSingleton() may include a `view_fn` that adapts a
+ // specially-designed multimap to a single value per window.
+ SdkFunctionSpec view_fn = 2;
+
+ // (Required) The SdkFunctionSpec of the UDF that maps a main input window
+ // to a side input window.
+ //
+ // For example, when the main input is in fixed windows of one hour, this
+ // can specify that the side input should be accessed according to the day
+ // in which that hour falls.
+ SdkFunctionSpec window_mapping_fn = 3;
+}
+
+// An environment for executing UDFs. Generally an SDK container URL, but
+// there can be many for a single SDK, for example to provide dependency
+// isolation.
+message Environment {
+
+ // (Required) The URL of a container
+ //
+ // TODO: reconcile with Fn API's DockerContainer structure by
+ // adding adequate metadata to know how to interpret the container
+ string url = 1;
+}
+
+// A specification of a user defined function.
+//
+message SdkFunctionSpec {
+
+ // (Required) A full specification of this function.
+ FunctionSpec spec = 1;
+
+ // (Required) Reference to an execution environment capable of
+ // invoking this function.
+ string environment_id = 2;
+}
+
+// A URN along with a parameter object whose schema is determined by the
+// URN.
+//
+// This structure is reused in two distinct, but compatible, ways:
+//
+// 1. This can be a specification of the function over PCollections
+// that a PTransform computes.
+// 2. This can be a specification of a user-defined function, possibly
+// SDK-specific. (external to this message must be adequate context
+// to indicate the environment in which the UDF can be understood).
+//
+// Though not explicit in this proto, there are two possibilities
+// for the relationship of a runner to this specification that
+// one should bear in mind:
+//
+// 1. The runner understands the URN. For example, it might be
+// a well-known URN like "urn:beam:transform:Top" or
+// "urn:beam:windowfn:FixedWindows" with
+// an agreed-upon payload (e.g. a number or duration,
+// respectively).
+// 2. The runner does not understand the URN. It might be an
+// SDK specific URN such as "urn:beam:dofn:javasdk:1.0"
+// that indicates to the SDK what the payload is,
+// such as a serialized Java DoFn from a particular
+// version of the Beam Java SDK. The payload will often
+// then be an opaque message such as bytes in a
+// language-specific serialization format.
+message FunctionSpec {
+
+ // (Required) A URN that describes the accompanying payload.
+ // For any URN that is not recognized (by whomever is inspecting
+ // it) the parameter payload should be treated as opaque and
+ // passed as-is.
+ string urn = 1;
+
+ // (Optional) The data specifying any parameters to the URN. If
+ // the URN does not require any arguments, this may be omitted.
+ bytes payload = 3;
+}
+
+// TODO: transfer javadoc here
+message DisplayData {
+
+ // (Required) The list of display data.
+ repeated Item items = 1;
+
+ // A complete identifier for a DisplayData.Item
+ message Identifier {
+
+ // (Required) The transform originating this display data.
+ string transform_id = 1;
+
+ // (Optional) The URN indicating the type of the originating transform,
+ // if there is one.
+ string transform_urn = 2;
+
+ string key = 3;
+ }
+
+ // A single item of display data.
+ message Item {
+ // (Required)
+ Identifier id = 1;
+
+ // (Required)
+ Type.Enum type = 2;
+
+ // (Required)
+ google.protobuf.Any value = 3;
+
+ // (Optional)
+ google.protobuf.Any short_value = 4;
+
+ // (Optional)
+ string label = 5;
+
+ // (Optional)
+ string link_url = 6;
+ }
+
+ message Type {
+ enum Enum {
+ UNSPECIFIED = 0;
+ STRING = 1;
+ INTEGER = 2;
+ FLOAT = 3;
+ BOOLEAN = 4;
+ TIMESTAMP = 5;
+ DURATION = 6;
+ JAVA_CLASS = 7;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/model/pipeline/src/main/proto/endpoints.proto
----------------------------------------------------------------------
diff --git a/model/pipeline/src/main/proto/endpoints.proto b/model/pipeline/src/main/proto/endpoints.proto
new file mode 100644
index 0000000..a642e63
--- /dev/null
+++ b/model/pipeline/src/main/proto/endpoints.proto
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing endpoints containing a service.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.portability.v1;
+
+option java_package = "org.apache.beam.portability.v1";
+option java_outer_classname = "Endpoints";
+
+message ApiServiceDescriptor {
+ // (Required) The URL to connect to.
+ string url = 2;
+
+ // (Optional) The method for authentication. If unspecified, access to the
+ // url is already being performed in a trusted context (e.g. localhost,
+ // private network).
+ oneof authentication {
+ OAuth2ClientCredentialsGrant oauth2_client_credentials_grant = 3;
+ }
+}
+
+message OAuth2ClientCredentialsGrant {
+ // (Required) The URL to submit a "client_credentials" grant type request for
+ // an OAuth access token which will be used as a bearer token for requests.
+ string url = 1;
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/model/pipeline/src/main/proto/standard_window_fns.proto
----------------------------------------------------------------------
diff --git a/model/pipeline/src/main/proto/standard_window_fns.proto b/model/pipeline/src/main/proto/standard_window_fns.proto
new file mode 100644
index 0000000..0682044
--- /dev/null
+++ b/model/pipeline/src/main/proto/standard_window_fns.proto
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Runner API, which is the runner-independent,
+ * SDK-independent definition of the Beam model.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.runner_api.v1;
+
+option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_outer_classname = "StandardWindowFns";
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+// beam:windowfn:global_windows:v0.1
+// empty payload
+
+// beam:windowfn:fixed_windows:v0.1
+message FixedWindowsPayload {
+ google.protobuf.Duration size = 1;
+ google.protobuf.Timestamp offset = 2;
+}
+
+// beam:windowfn:sliding_windows:v0.1
+message SlidingWindowsPayload {
+ google.protobuf.Duration size = 1;
+ google.protobuf.Timestamp offset = 2;
+ google.protobuf.Duration period = 3;
+}
+
+// beam:windowfn:session_windows:v0.1
+message SessionsPayload {
+ google.protobuf.Duration gap_size = 1;
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 72e2eff..212d703 100644
--- a/pom.xml
+++ b/pom.xml
@@ -441,12 +441,6 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/apex/pom.xml
----------------------------------------------------------------------
diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index bcba1bc..8ade583 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -79,7 +79,7 @@
<!-- Beam -->
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/core-construction-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/pom.xml b/runners/core-construction-java/pom.xml
index aed8b8d..9f71959 100644
--- a/runners/core-construction-java/pom.xml
+++ b/runners/core-construction-java/pom.xml
@@ -56,7 +56,12 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-job-management</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/core-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index a7a76ba..668c08c 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -59,12 +59,12 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-core</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-sdks-java-core</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/direct-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 2cb3179..4e28438 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -58,9 +58,9 @@
<includes>
<include>com.google.guava:guava</include>
<include>com.google.protobuf:protobuf-java</include>
+ <include>org.apache.beam:beam-model-pipeline</include>
<include>org.apache.beam:beam-runners-core-construction-java</include>
<include>org.apache.beam:beam-runners-core-java</include>
- <include>org.apache.beam:beam-sdks-common-runner-api</include>
<include>com.google.code.findbugs:jsr305</include>
</includes>
</artifactSet>
@@ -191,12 +191,12 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-core</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-sdks-java-core</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/flink/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index 58a01e4..effbdb4 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -219,7 +219,7 @@
<!-- Beam -->
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 3d35fa0..873bb2e 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -270,17 +270,17 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-core</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
+ <artifactId>beam-sdks-java-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/local-artifact-service-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/local-artifact-service-java/pom.xml b/runners/local-artifact-service-java/pom.xml
index 8d7dfec..72490cc 100644
--- a/runners/local-artifact-service-java/pom.xml
+++ b/runners/local-artifact-service-java/pom.xml
@@ -53,7 +53,7 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-job-management</artifactId>
</dependency>
<!-- build dependencies -->
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/reference/job-server/pom.xml
----------------------------------------------------------------------
diff --git a/runners/reference/job-server/pom.xml b/runners/reference/job-server/pom.xml
index aed03e9..fb0f170 100644
--- a/runners/reference/job-server/pom.xml
+++ b/runners/reference/job-server/pom.xml
@@ -44,7 +44,7 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-job-management</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/reference/pom.xml
----------------------------------------------------------------------
diff --git a/runners/reference/pom.xml b/runners/reference/pom.xml
index d5f1993..0c7f939 100644
--- a/runners/reference/pom.xml
+++ b/runners/reference/pom.xml
@@ -33,7 +33,6 @@
Beam portability framework to execute an arbitrary Pipeline.</description>
<packaging>pom</packaging>
-
<modules>
<module>job-server</module>
</modules>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index e3e2439..3e9095a 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -232,7 +232,7 @@
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/fn-api/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/common/fn-api/pom.xml b/sdks/common/fn-api/pom.xml
index f6c8632..498f83a 100644
--- a/sdks/common/fn-api/pom.xml
+++ b/sdks/common/fn-api/pom.xml
@@ -83,7 +83,7 @@
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-runner-api</artifactId>
+ <artifactId>beam-model-pipeline</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/common/pom.xml b/sdks/common/pom.xml
index d299ca6..d950759 100644
--- a/sdks/common/pom.xml
+++ b/sdks/common/pom.xml
@@ -34,6 +34,5 @@
<modules>
<module>fn-api</module>
- <module>runner-api</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/runner-api/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/common/runner-api/pom.xml b/sdks/common/runner-api/pom.xml
deleted file mode 100644
index 66914a6..0000000
--- a/sdks/common/runner-api/pom.xml
+++ /dev/null
@@ -1,109 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <packaging>jar</packaging>
- <parent>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-parent</artifactId>
- <version>2.3.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>beam-sdks-common-runner-api</artifactId>
- <name>Apache Beam :: SDKs :: Common :: Runner API</name>
- <description>This artifact generates the stub bindings.</description>
-
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- <resource>
- <directory>${project.build.directory}/original_sources_to_package</directory>
- </resource>
- </resources>
-
- <plugins>
- <!-- Skip the checkstyle plugin on generated code -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <!-- Skip the findbugs plugin on generated code -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <configuration>
- <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <pluginId>grpc-java</pluginId>
- <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>compile-custom</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-stub</artifactId>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/runner-api/src/main/proto/beam_artifact_api.proto
----------------------------------------------------------------------
diff --git a/sdks/common/runner-api/src/main/proto/beam_artifact_api.proto b/sdks/common/runner-api/src/main/proto/beam_artifact_api.proto
deleted file mode 100644
index e11551c..0000000
--- a/sdks/common/runner-api/src/main/proto/beam_artifact_api.proto
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Artifact API, for communicating with a runner
- * for artifact staging and retrieval over GRPC.
- */
-
-syntax = "proto3";
-
-package org.apache.beam.runner_api.v1;
-
-option java_package = "org.apache.beam.sdk.common.runner.v1";
-option java_outer_classname = "ArtifactApi";
-
-// A service to stage artifacts for use in a Job.
-//
-// RPCs made to an ArtifactStagingService endpoint should include some form of identification for
-// the job as a header.
-service ArtifactStagingService {
- // Stage an artifact to be available during job execution. The first request must contain the
- // name of the artifact. All future requests must contain sequential chunks of the content of
- // the artifact.
- rpc PutArtifact(stream PutArtifactRequest) returns (PutArtifactResponse);
-
- // Commit the manifest for a Job. All artifacts must have been successfully uploaded
- // before this call is made.
- //
- // Throws error INVALID_ARGUMENT if not all of the members of the manifest are present
- rpc CommitManifest(CommitManifestRequest) returns (CommitManifestResponse);
-}
-
-// A service to retrieve artifacts for use in a Job.
-service ArtifactRetrievalService {
- // Get the manifest for the job
- rpc GetManifest(GetManifestRequest) returns (GetManifestResponse);
-
- // Get an artifact staged for the job. The requested artifact must be within the manifest
- rpc GetArtifact(GetArtifactRequest) returns (stream ArtifactChunk);
-}
-
-// An artifact identifier and associated metadata.
-message ArtifactMetadata {
- // (Required) The name of the artifact.
- string name = 1;
-
- // (Optional) The Unix-like permissions of the artifact
- uint32 permissions = 2;
-
- // (Optional) The base64-encoded md5 checksum of the artifact. Used, among other things, by
- // harness boot code to validate the integrity of the artifact.
- string md5 = 3;
-}
-
-// A collection of artifacts.
-message Manifest {
- repeated ArtifactMetadata artifact = 1;
-}
-
-// A manifest with location information.
-message ProxyManifest {
- Manifest manifest = 1;
- message Location {
- string name = 1;
- string uri = 2;
- }
- repeated Location location = 2;
-}
-
-// A request to get the manifest of a Job.
-message GetManifestRequest {}
-
-// A response containing a job manifest.
-message GetManifestResponse {
- Manifest manifest = 1;
-}
-
-// A request to get an artifact. The artifact must be present in the manifest for the job.
-message GetArtifactRequest {
- // (Required) The name of the artifact to retrieve.
- string name = 1;
-}
-
-// Part of an artifact.
-message ArtifactChunk {
- bytes data = 1;
-}
-
-// A request to stage an artifact.
-message PutArtifactRequest {
- // (Required)
- oneof content {
- // The Artifact metadata. The first message in a PutArtifact call must contain the name
- // of the artifact.
- ArtifactMetadata metadata = 1;
-
- // A chunk of the artifact. All messages after the first in a PutArtifact call must contain a
- // chunk.
- ArtifactChunk data = 2;
- }
-}
-
-message PutArtifactResponse {
-}
-
-// A request to commit the manifest for a Job. All artifacts must have been successfully uploaded
-// before this call is made.
-message CommitManifestRequest {
- // (Required) The manifest to commit.
- Manifest manifest = 1;
-}
-
-// The result of committing a manifest.
-message CommitManifestResponse {
- // (Required) An opaque token representing the entirety of the staged artifacts.
- string staging_token = 1;
-}
-
http://git-wip-us.apache.org/repos/asf/beam/blob/2f3af318/sdks/common/runner-api/src/main/proto/beam_job_api.proto
----------------------------------------------------------------------
diff --git a/sdks/common/runner-api/src/main/proto/beam_job_api.proto b/sdks/common/runner-api/src/main/proto/beam_job_api.proto
deleted file mode 100644
index 8f2ed0e..0000000
--- a/sdks/common/runner-api/src/main/proto/beam_job_api.proto
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Job API, api for communicating with a runner
- * for job submission over GRPC.
- */
-
-syntax = "proto3";
-
-package org.apache.beam.runner_api.v1;
-
-option java_package = "org.apache.beam.sdk.common.runner.v1";
-option java_outer_classname = "JobApi";
-
-import "beam_runner_api.proto";
-import "endpoints.proto";
-import "google/protobuf/struct.proto";
-
-
-// Job Service for running RunnerAPI pipelines
-service JobService {
- // Prepare a job for execution. The job will not be executed until a call is made to run with the
- // returned preparationId.
- rpc Prepare (PrepareJobRequest) returns (PrepareJobResponse);
-
- // Submit the job for execution
- rpc Run (RunJobRequest) returns (RunJobResponse);
-
- // Get the current state of the job
- rpc GetState (GetJobStateRequest) returns (GetJobStateResponse);
-
- // Cancel the job
- rpc Cancel (CancelJobRequest) returns (CancelJobResponse);
-
- // Subscribe to a stream of state changes of the job, will immediately return the current state of the job as the first response.
- rpc GetStateStream (GetJobStateRequest) returns (stream GetJobStateResponse);
-
- // Subscribe to a stream of state changes and messages from the job
- rpc GetMessageStream (JobMessagesRequest) returns (stream JobMessagesResponse);
-}
-
-
-// Prepare is a synchronous request that returns a preparationId back
-// Throws error GRPC_STATUS_UNAVAILABLE if server is down
-// Throws error ALREADY_EXISTS if the jobName is reused. Runners are permitted to deduplicate based on the name of the job.
-// Throws error UNKNOWN for all other issues
-message PrepareJobRequest {
- org.apache.beam.runner_api.v1.Pipeline pipeline = 1; // (required)
- google.protobuf.Struct pipeline_options = 2; // (required)
- string job_name = 3; // (required)
-}
-
-message PrepareJobResponse {
- // (required) The ID used to associate calls made while preparing the job. preparationId is used
- // to run the job, as well as in other pre-execution APIs such as Artifact staging.
- string preparation_id = 1;
-
- // An endpoint which exposes the Beam Artifact Staging API. Artifacts used by the job should be
- // staged to this endpoint, and will be available during job execution.
- org.apache.beam.portability.v1.ApiServiceDescriptor artifact_staging_endpoint = 2;
-}
-
-
-// Run is a synchronous request that returns a jobId back.
-// Throws error GRPC_STATUS_UNAVAILABLE if server is down
-// Throws error NOT_FOUND if the preparation ID does not exist
-// Throws error UNKNOWN for all other issues
-message RunJobRequest {
- // (required) The ID provided by an earlier call to prepare. Runs the job. All prerequisite tasks
- // must have been completed.
- string preparation_id = 1;
- // (optional) If any artifacts have been staged for this job, contains the staging_token returned
- // from the CommitManifestResponse.
- string staging_token = 2;
-}
-
-
-message RunJobResponse {
- string job_id = 1; // (required) The ID for the executing job
-}
-
-
-// Cancel is a synchronus request that returns a job state back
-// Throws error GRPC_STATUS_UNAVAILABLE if server is down
-// Throws error NOT_FOUND if the jobId is not found
-message CancelJobRequest {
- string job_id = 1; // (required)
-
-}
-
-// Valid responses include any terminal state or CANCELLING
-message CancelJobResponse {
- JobState.Enum state = 1; // (required)
-}
-
-
-// GetState is a synchronus request that returns a job state back
-// Throws error GRPC_STATUS_UNAVAILABLE if server is down
-// Throws error NOT_FOUND if the jobId is not found
-message GetJobStateRequest {
- string job_id = 1; // (required)
-
-}
-
-message GetJobStateResponse {
- JobState.Enum state = 1; // (required)
-}
-
-
-// GetJobMessages is a streaming api for streaming job messages from the service
-// One request will connect you to the job and you'll get a stream of job state
-// and job messages back; one is used for logging and the other for detecting
-// the job ended.
-message JobMessagesRequest {
- string job_id = 1; // (required)
-
-}
-
-message JobMessage {
- string message_id = 1;
- string time = 2;
- MessageImportance importance = 3;
- string message_text = 4;
-
- enum MessageImportance {
- MESSAGE_IMPORTANCE_UNSPECIFIED = 0;
- JOB_MESSAGE_DEBUG = 1;
- JOB_MESSAGE_DETAILED = 2;
- JOB_MESSAGE_BASIC = 3;
- JOB_MESSAGE_WARNING = 4;
- JOB_MESSAGE_ERROR = 5;
- }
-}
-
-message JobMessagesResponse {
- oneof response {
- JobMessage message_response = 1;
- GetJobStateResponse state_response = 2;
- }
-}
-
-// Enumeration of all JobStates
-message JobState {
- enum Enum {
- UNSPECIFIED = 0;
- STOPPED = 1;
- RUNNING = 2;
- DONE = 3;
- FAILED = 4;
- CANCELLED = 5;
- UPDATED = 6;
- DRAINING = 7;
- DRAINED = 8;
- STARTING = 9;
- CANCELLING = 10;
- }
-}
[12/12] beam git commit: This closes #3924: [BEAM-2576] Split Beam
portability framework into construction, job, and execution modules
Posted by ke...@apache.org.
This closes #3924: [BEAM-2576] Split Beam portability framework into construction, job, and execution modules
Dataflow runner: shade protobuf and beam model
Improve error message when standard_coders.yaml not found
Re-namespace model/fn-execution to model.fn_execution
Re-namespace model/job-management to model.job_management
Re-namespace model/pipeline to model.pipeline
Remove sdks/common
Move sdks/common/fn-api protos to model/fn-execution
Move sdk/common/runner-api protos into model/pipeline and model/job-management
Add model/ and submodules to Maven structure and gen_protos.py
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/92615ca4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/92615ca4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/92615ca4
Branch: refs/heads/master
Commit: 92615ca4c4e09bc63c37a933102035040c23aac9
Parents: 4ccb3ba b2daa0c
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Oct 11 12:24:52 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 12:24:52 2017 -0700
----------------------------------------------------------------------
model/fn-execution/pom.xml | 114 +++
.../src/main/proto/beam_fn_api.proto | 726 ++++++++++++++++
.../src/main/proto/beam_provision_api.proto | 60 ++
.../model/fnexecution/v1/standard_coders.yaml | 195 +++++
model/job-management/pom.xml | 114 +++
.../src/main/proto/beam_artifact_api.proto | 133 +++
.../src/main/proto/beam_job_api.proto | 173 ++++
model/pipeline/pom.xml | 89 ++
.../src/main/proto/beam_runner_api.proto | 842 +++++++++++++++++++
model/pipeline/src/main/proto/endpoints.proto | 46 +
.../src/main/proto/standard_window_fns.proto | 53 ++
model/pom.xml | 40 +
pom.xml | 15 +-
runners/apex/pom.xml | 4 +-
runners/core-construction-java/pom.xml | 7 +-
.../construction/ArtifactServiceStager.java | 18 +-
.../core/construction/CoderTranslation.java | 6 +-
.../core/construction/CombineTranslation.java | 12 +-
.../CreatePCollectionViewTranslation.java | 4 +-
.../construction/DisplayDataTranslation.java | 2 +-
.../core/construction/FlattenTranslator.java | 4 +-
.../construction/GroupByKeyTranslation.java | 4 +-
.../construction/PCollectionTranslation.java | 2 +-
.../construction/PTransformTranslation.java | 4 +-
.../core/construction/ParDoTranslation.java | 16 +-
.../core/construction/PipelineTranslation.java | 2 +-
.../core/construction/ReadTranslation.java | 10 +-
.../core/construction/RehydratedComponents.java | 4 +-
.../construction/RunnerPCollectionView.java | 2 +-
.../core/construction/SdkComponents.java | 4 +-
.../construction/TestStreamTranslation.java | 2 +-
.../core/construction/TriggerTranslation.java | 2 +-
.../construction/WindowIntoTranslation.java | 6 +-
.../WindowingStrategyTranslation.java | 18 +-
.../construction/WriteFilesTranslation.java | 10 +-
.../construction/ArtifactServiceStagerTest.java | 2 +-
.../core/construction/CoderTranslationTest.java | 4 +-
.../construction/CombineTranslationTest.java | 4 +-
.../CreatePCollectionViewTranslationTest.java | 4 +-
.../InMemoryArtifactStagerService.java | 16 +-
.../PCollectionTranslationTest.java | 2 +-
.../construction/PTransformTranslationTest.java | 6 +-
.../core/construction/ParDoTranslationTest.java | 6 +-
.../construction/PipelineTranslationTest.java | 2 +-
.../core/construction/ReadTranslationTest.java | 4 +-
.../core/construction/SdkComponentsTest.java | 2 +-
.../construction/TestStreamTranslationTest.java | 6 +-
.../construction/WindowIntoTranslationTest.java | 2 +-
.../WindowingStrategyTranslationTest.java | 2 +-
.../construction/WriteFilesTranslationTest.java | 4 +-
runners/core-java/pom.xml | 6 +-
.../runners/core/fn/FnApiControlClient.java | 2 +-
.../core/fn/FnApiControlClientPoolService.java | 4 +-
.../beam/runners/core/fn/FnDataService.java | 6 +-
.../beam/runners/core/fn/SdkHarnessClient.java | 10 +-
.../runners/core/fn/SdkHarnessDoFnRunner.java | 5 +-
.../core/triggers/TriggerStateMachines.java | 2 +-
.../fn/FnApiControlClientPoolServiceTest.java | 2 +-
.../runners/core/fn/FnApiControlClientTest.java | 2 +-
.../runners/core/fn/SdkHarnessClientTest.java | 2 +-
.../core/fn/SdkHarnessDoFnRunnerTest.java | 2 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
runners/direct-java/pom.xml | 8 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 2 +-
.../direct/TransformEvaluatorRegistry.java | 2 +-
runners/flink/pom.xml | 4 +-
.../FlinkStreamingTransformTranslators.java | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 18 +-
.../beam/runners/dataflow/DataflowRunner.java | 2 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 2 +-
.../options/DataflowPipelineOptions.java | 1 -
runners/local-artifact-service-java/pom.xml | 2 +-
.../LocalFileSystemArtifactStagerService.java | 57 +-
...ocalFileSystemArtifactStagerServiceTest.java | 143 ++--
runners/reference/job-server/pom.xml | 2 +-
.../job/ReferenceRunnerJobService.java | 16 +-
runners/reference/pom.xml | 1 -
runners/spark/pom.xml | 4 +-
sdks/common/fn-api/pom.xml | 114 ---
.../fn-api/src/main/proto/beam_fn_api.proto | 726 ----------------
.../src/main/proto/beam_provision_api.proto | 60 --
.../org/apache/beam/fn/v1/standard_coders.yaml | 195 -----
sdks/common/pom.xml | 39 -
sdks/common/runner-api/pom.xml | 109 ---
.../src/main/proto/beam_artifact_api.proto | 133 ---
.../src/main/proto/beam_job_api.proto | 173 ----
.../src/main/proto/beam_runner_api.proto | 842 -------------------
.../runner-api/src/main/proto/endpoints.proto | 46 -
.../src/main/proto/standard_window_fns.proto | 53 --
sdks/java/core/pom.xml | 8 +-
.../apache/beam/sdk/coders/CommonCoderTest.java | 4 +-
sdks/java/harness/pom.xml | 24 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 8 +-
.../beam/fn/harness/BeamFnDataWriteRunner.java | 8 +-
.../beam/fn/harness/BoundedSourceRunner.java | 2 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 8 +-
.../org/apache/beam/fn/harness/FnHarness.java | 8 +-
.../fn/harness/PTransformRunnerFactory.java | 2 +-
.../harness/channel/ManagedChannelFactory.java | 2 +-
.../fn/harness/control/BeamFnControlClient.java | 12 +-
.../harness/control/ProcessBundleHandler.java | 16 +-
.../fn/harness/control/RegisterHandler.java | 8 +-
.../BeamFnDataBufferingOutboundObserver.java | 4 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 4 +-
.../fn/harness/data/BeamFnDataGrpcClient.java | 6 +-
.../harness/data/BeamFnDataGrpcMultiplexer.java | 8 +-
.../harness/data/BeamFnDataInboundObserver.java | 4 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 6 +-
.../beam/fn/harness/state/BagUserState.java | 6 +-
.../fn/harness/state/BeamFnStateClient.java | 4 +-
.../state/BeamFnStateGrpcClientCache.java | 10 +-
.../harness/state/StateFetchingIterators.java | 26 +-
.../fn/harness/BeamFnDataReadRunnerTest.java | 8 +-
.../fn/harness/BeamFnDataWriteRunnerTest.java | 8 +-
.../fn/harness/BoundedSourceRunnerTest.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 4 +-
.../apache/beam/fn/harness/FnHarnessTest.java | 14 +-
.../channel/ManagedChannelFactoryTest.java | 19 +-
.../control/BeamFnControlClientTest.java | 6 +-
.../control/ProcessBundleHandlerTest.java | 10 +-
.../fn/harness/control/RegisterHandlerTest.java | 6 +-
...BeamFnDataBufferingOutboundObserverTest.java | 2 +-
.../harness/data/BeamFnDataGrpcClientTest.java | 6 +-
.../data/BeamFnDataGrpcMultiplexerTest.java | 4 +-
.../data/BeamFnDataInboundObserverTest.java | 2 +-
.../logging/BeamFnLoggingClientTest.java | 6 +-
.../beam/fn/harness/state/BagUserStateTest.java | 4 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 16 +-
.../fn/harness/state/FakeBeamFnStateClient.java | 16 +-
.../state/StateFetchingIteratorsTest.java | 6 +-
sdks/pom.xml | 1 -
sdks/python/gen_protos.py | 5 +-
132 files changed, 3046 insertions(+), 2897 deletions(-)
----------------------------------------------------------------------
[06/12] beam git commit: Dataflow runner: shade protobuf and beam
model
Posted by ke...@apache.org.
Dataflow runner: shade protobuf and beam model
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b2daa0cf
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b2daa0cf
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b2daa0cf
Branch: refs/heads/master
Commit: b2daa0cf432eb81d3f9283b9239f164a60ff92e1
Parents: 7fa37fd
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Oct 10 21:24:35 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:56:39 2017 -0700
----------------------------------------------------------------------
runners/google-cloud-dataflow-java/pom.xml | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b2daa0cf/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index da0d237..2e08181 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -181,6 +181,8 @@
<artifactSet>
<includes>
<include>com.google.guava:guava</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>org.apache.beam:beam-model-pipeline</include>
<include>org.apache.beam:beam-runners-core-construction-java</include>
</includes>
</artifactSet>
@@ -207,10 +209,18 @@
<shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.common</shadedPattern>
</relocation>
<relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>com.google.thirdparty</pattern>
<shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.thirdparty</shadedPattern>
</relocation>
<relocation>
+ <pattern>org.apache.beam.model</pattern>
+ <shadedPattern>org.apache.beam.runners.dataflow.repackaged.org.apache.beam.model</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>org.apache.beam.runners.core</pattern>
<shadedPattern>org.apache.beam.runners.dataflow.repackaged.org.apache.beam.runners.core</shadedPattern>
</relocation>
[10/12] beam git commit: Re-namespace model/job-management to
model.job_management
Posted by ke...@apache.org.
Re-namespace model/job-management to model.job_management
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ed772692
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ed772692
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ed772692
Branch: refs/heads/master
Commit: ed7726927c142302e83264ecadba81aa85bece19
Parents: 7c563ef
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Oct 5 15:03:02 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:56:39 2017 -0700
----------------------------------------------------------------------
.../src/main/proto/beam_artifact_api.proto | 4 +-
.../src/main/proto/beam_job_api.proto | 4 +-
.../construction/ArtifactServiceStager.java | 18 +--
.../construction/ArtifactServiceStagerTest.java | 2 +-
.../InMemoryArtifactStagerService.java | 16 +--
.../LocalFileSystemArtifactStagerService.java | 57 ++++----
...ocalFileSystemArtifactStagerServiceTest.java | 143 +++++++++++--------
.../job/ReferenceRunnerJobService.java | 16 +--
8 files changed, 145 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/model/job-management/src/main/proto/beam_artifact_api.proto
----------------------------------------------------------------------
diff --git a/model/job-management/src/main/proto/beam_artifact_api.proto b/model/job-management/src/main/proto/beam_artifact_api.proto
index e11551c..4b3ac0d 100644
--- a/model/job-management/src/main/proto/beam_artifact_api.proto
+++ b/model/job-management/src/main/proto/beam_artifact_api.proto
@@ -23,9 +23,9 @@
syntax = "proto3";
-package org.apache.beam.runner_api.v1;
+package org.apache.beam.model.job_management.v1;
-option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_package = "org.apache.beam.model.jobmanagement.v1";
option java_outer_classname = "ArtifactApi";
// A service to stage artifacts for use in a Job.
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/model/job-management/src/main/proto/beam_job_api.proto
----------------------------------------------------------------------
diff --git a/model/job-management/src/main/proto/beam_job_api.proto b/model/job-management/src/main/proto/beam_job_api.proto
index 3830f5c..6465383 100644
--- a/model/job-management/src/main/proto/beam_job_api.proto
+++ b/model/job-management/src/main/proto/beam_job_api.proto
@@ -23,9 +23,9 @@
syntax = "proto3";
-package org.apache.beam.runner_api.v1;
+package org.apache.beam.model.job_management.v1;
-option java_package = "org.apache.beam.sdk.common.runner.v1";
+option java_package = "org.apache.beam.model.jobmanagement.v1";
option java_outer_classname = "JobApi";
import "beam_runner_api.proto";
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
index c37f289..095b549 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
@@ -45,15 +45,15 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.ArtifactChunk;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.ArtifactMetadata;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.CommitManifestRequest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.Manifest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactRequest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactResponse;
-import org.apache.beam.sdk.common.runner.v1.ArtifactStagingServiceGrpc;
-import org.apache.beam.sdk.common.runner.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceBlockingStub;
-import org.apache.beam.sdk.common.runner.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceStub;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactChunk;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.CommitManifestRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.Manifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceBlockingStub;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceStub;
/** A client to stage files on an {@link ArtifactStagingServiceGrpc ArtifactService}. */
public class ArtifactServiceStager {
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java
index 264aaf8..13bd8dd 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ArtifactServiceStagerTest.java
@@ -39,7 +39,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java
index e89f829..1f7a4fb 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/InMemoryArtifactStagerService.java
@@ -29,14 +29,14 @@ import java.security.NoSuchAlgorithmException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.ArtifactMetadata;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.CommitManifestResponse;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.Manifest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactRequest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactRequest.ContentCase;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactResponse;
-import org.apache.beam.sdk.common.runner.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceImplBase;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.CommitManifestResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.Manifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactRequest.ContentCase;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.PutArtifactResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceImplBase;
/**
* An {@link ArtifactStagingServiceImplBase ArtifactStagingService} which stores the bytes of the
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java
----------------------------------------------------------------------
diff --git a/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java b/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java
index 6b42a3b..a9f595f 100644
--- a/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java
+++ b/runners/local-artifact-service-java/src/main/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerService.java
@@ -32,18 +32,14 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import javax.annotation.Nullable;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.ArtifactMetadata;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.CommitManifestRequest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.CommitManifestResponse;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactRequest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactRequest.ContentCase;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactResponse;
-import org.apache.beam.sdk.common.runner.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceImplBase;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** An {@code ArtifactStagingService} which stages files to a local temp directory. */
-public class LocalFileSystemArtifactStagerService extends ArtifactStagingServiceImplBase {
+public class LocalFileSystemArtifactStagerService
+ extends ArtifactStagingServiceGrpc.ArtifactStagingServiceImplBase {
private static final Logger LOG =
LoggerFactory.getLogger(LocalFileSystemArtifactStagerService.class);
@@ -69,14 +65,15 @@ public class LocalFileSystemArtifactStagerService extends ArtifactStagingService
}
@Override
- public StreamObserver<PutArtifactRequest> putArtifact(
- final StreamObserver<PutArtifactResponse> responseObserver) {
+ public StreamObserver<ArtifactApi.PutArtifactRequest> putArtifact(
+ final StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) {
return new CreateAndWriteFileObserver(responseObserver);
}
@Override
public void commitManifest(
- CommitManifestRequest request, StreamObserver<CommitManifestResponse> responseObserver) {
+ ArtifactApi.CommitManifestRequest request,
+ StreamObserver<ArtifactApi.CommitManifestResponse> responseObserver) {
try {
commitManifestOrThrow(request, responseObserver);
} catch (StatusRuntimeException e) {
@@ -93,10 +90,11 @@ public class LocalFileSystemArtifactStagerService extends ArtifactStagingService
}
private void commitManifestOrThrow(
- CommitManifestRequest request, StreamObserver<CommitManifestResponse> responseObserver)
+ ArtifactApi.CommitManifestRequest request,
+ StreamObserver<ArtifactApi.CommitManifestResponse> responseObserver)
throws IOException {
- Collection<ArtifactMetadata> missing = new ArrayList<>();
- for (ArtifactMetadata artifact : request.getManifest().getArtifactList()) {
+ Collection<ArtifactApi.ArtifactMetadata> missing = new ArrayList<>();
+ for (ArtifactApi.ArtifactMetadata artifact : request.getManifest().getArtifactList()) {
// TODO: Validate the checksums on the server side, to fail more aggressively if require
if (!getArtifactFile(artifact.getName()).exists()) {
missing.add(artifact);
@@ -114,7 +112,7 @@ public class LocalFileSystemArtifactStagerService extends ArtifactStagingService
request.getManifest().writeTo(mfOut);
}
responseObserver.onNext(
- CommitManifestResponse.newBuilder()
+ ArtifactApi.CommitManifestResponse.newBuilder()
.setStagingToken(stagingBase.getCanonicalPath())
.build());
responseObserver.onCompleted();
@@ -124,24 +122,27 @@ public class LocalFileSystemArtifactStagerService extends ArtifactStagingService
return new File(artifactsBase, artifactName);
}
- private class CreateAndWriteFileObserver implements StreamObserver<PutArtifactRequest> {
- private final StreamObserver<PutArtifactResponse> responseObserver;
+ private class CreateAndWriteFileObserver
+ implements StreamObserver<ArtifactApi.PutArtifactRequest> {
+ private final StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver;
private FileWritingObserver writer;
- private CreateAndWriteFileObserver(StreamObserver<PutArtifactResponse> responseObserver) {
+ private CreateAndWriteFileObserver(
+ StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) {
this.responseObserver = responseObserver;
}
@Override
- public void onNext(PutArtifactRequest value) {
+ public void onNext(ArtifactApi.PutArtifactRequest value) {
try {
if (writer == null) {
- if (!value.getContentCase().equals(ContentCase.METADATA)) {
+ if (!value.getContentCase().equals(ArtifactApi.PutArtifactRequest.ContentCase.METADATA)) {
throw Status.INVALID_ARGUMENT
.withDescription(
String.format(
"Expected the first %s to contain the Artifact Name, got %s",
- PutArtifactRequest.class.getSimpleName(), value.getContentCase()))
+ ArtifactApi.PutArtifactRequest.class.getSimpleName(),
+ value.getContentCase()))
.asRuntimeException();
}
writer = createFile(value.getMetadata());
@@ -159,7 +160,8 @@ public class LocalFileSystemArtifactStagerService extends ArtifactStagingService
}
}
- private FileWritingObserver createFile(ArtifactMetadata metadata) throws IOException {
+ private FileWritingObserver createFile(ArtifactApi.ArtifactMetadata metadata)
+ throws IOException {
File destination = getArtifactFile(metadata.getName());
if (!destination.createNewFile()) {
throw Status.ALREADY_EXISTS
@@ -189,22 +191,23 @@ public class LocalFileSystemArtifactStagerService extends ArtifactStagingService
}
}
- private static class FileWritingObserver implements StreamObserver<PutArtifactRequest> {
+ private static class FileWritingObserver
+ implements StreamObserver<ArtifactApi.PutArtifactRequest> {
private final File destination;
private final OutputStream target;
- private final StreamObserver<PutArtifactResponse> responseObserver;
+ private final StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver;
private FileWritingObserver(
File destination,
OutputStream target,
- StreamObserver<PutArtifactResponse> responseObserver) {
+ StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) {
this.destination = destination;
this.target = target;
this.responseObserver = responseObserver;
}
@Override
- public void onNext(PutArtifactRequest value) {
+ public void onNext(ArtifactApi.PutArtifactRequest value) {
try {
if (value.getData() == null) {
StatusRuntimeException e = Status.INVALID_ARGUMENT.withDescription(String.format(
@@ -234,7 +237,7 @@ public class LocalFileSystemArtifactStagerService extends ArtifactStagingService
cleanedUp(e);
return;
}
- responseObserver.onNext(PutArtifactResponse.getDefaultInstance());
+ responseObserver.onNext(ArtifactApi.PutArtifactResponse.getDefaultInstance());
responseObserver.onCompleted();
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java
----------------------------------------------------------------------
diff --git a/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java b/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java
index b7ba03f..d98253b 100644
--- a/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java
+++ b/runners/local-artifact-service-java/src/test/java/org/apache/beam/artifact/local/LocalFileSystemArtifactStagerServiceTest.java
@@ -34,15 +34,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.ArtifactChunk;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.ArtifactMetadata;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.CommitManifestRequest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.CommitManifestResponse;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.Manifest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactRequest;
-import org.apache.beam.sdk.common.runner.v1.ArtifactApi.PutArtifactResponse;
-import org.apache.beam.sdk.common.runner.v1.ArtifactStagingServiceGrpc;
-import org.apache.beam.sdk.common.runner.v1.ArtifactStagingServiceGrpc.ArtifactStagingServiceStub;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactStagingServiceGrpc;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Assert;
@@ -58,7 +51,7 @@ import org.junit.runners.JUnit4;
public class LocalFileSystemArtifactStagerServiceTest {
@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
- private ArtifactStagingServiceStub stub;
+ private ArtifactStagingServiceGrpc.ArtifactStagingServiceStub stub;
private LocalFileSystemArtifactStagerService stager;
private ServerImpl server;
@@ -87,17 +80,20 @@ public class LocalFileSystemArtifactStagerServiceTest {
@Test
public void singleDataPutArtifactSucceeds() throws Exception {
byte[] data = "foo-bar-baz".getBytes();
- RecordingStreamObserver<PutArtifactResponse> responseObserver = new RecordingStreamObserver<>();
- StreamObserver<PutArtifactRequest> requestObserver = stub.putArtifact(responseObserver);
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
String name = "my-artifact";
requestObserver.onNext(
- PutArtifactRequest.newBuilder()
- .setMetadata(ArtifactMetadata.newBuilder().setName(name).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setMetadata(ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build())
.build());
requestObserver.onNext(
- PutArtifactRequest.newBuilder()
- .setData(ArtifactChunk.newBuilder().setData(ByteString.copyFrom(data)).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder().setData(ByteString.copyFrom(data)).build())
.build());
requestObserver.onCompleted();
@@ -115,25 +111,36 @@ public class LocalFileSystemArtifactStagerServiceTest {
byte[] partOne = "foo-".getBytes();
byte[] partTwo = "bar-".getBytes();
byte[] partThree = "baz".getBytes();
- RecordingStreamObserver<PutArtifactResponse> responseObserver = new RecordingStreamObserver<>();
- StreamObserver<PutArtifactRequest> requestObserver = stub.putArtifact(responseObserver);
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
String name = "my-artifact";
requestObserver.onNext(
- PutArtifactRequest.newBuilder()
- .setMetadata(ArtifactMetadata.newBuilder().setName(name).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setMetadata(ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build())
.build());
requestObserver.onNext(
- PutArtifactRequest.newBuilder()
- .setData(ArtifactChunk.newBuilder().setData(ByteString.copyFrom(partOne)).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder()
+ .setData(ByteString.copyFrom(partOne))
+ .build())
.build());
requestObserver.onNext(
- PutArtifactRequest.newBuilder()
- .setData(ArtifactChunk.newBuilder().setData(ByteString.copyFrom(partTwo)).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder()
+ .setData(ByteString.copyFrom(partTwo))
+ .build())
.build());
requestObserver.onNext(
- PutArtifactRequest.newBuilder()
- .setData(ArtifactChunk.newBuilder().setData(ByteString.copyFrom(partThree)).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder()
+ .setData(ByteString.copyFrom(partThree))
+ .build())
.build());
requestObserver.onCompleted();
@@ -149,12 +156,15 @@ public class LocalFileSystemArtifactStagerServiceTest {
@Test
public void putArtifactBeforeNameFails() {
byte[] data = "foo-".getBytes();
- RecordingStreamObserver<PutArtifactResponse> responseObserver = new RecordingStreamObserver<>();
- StreamObserver<PutArtifactRequest> requestObserver = stub.putArtifact(responseObserver);
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
requestObserver.onNext(
- PutArtifactRequest.newBuilder()
- .setData(ArtifactChunk.newBuilder().setData(ByteString.copyFrom(data)).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder().setData(ByteString.copyFrom(data)).build())
.build());
responseObserver.awaitTerminalState();
@@ -164,11 +174,15 @@ public class LocalFileSystemArtifactStagerServiceTest {
@Test
public void putArtifactWithNoContentFails() {
- RecordingStreamObserver<PutArtifactResponse> responseObserver = new RecordingStreamObserver<>();
- StreamObserver<PutArtifactRequest> requestObserver = stub.putArtifact(responseObserver);
+ RecordingStreamObserver<ArtifactApi.PutArtifactResponse> responseObserver =
+ new RecordingStreamObserver<>();
+ StreamObserver<ArtifactApi.PutArtifactRequest> requestObserver =
+ stub.putArtifact(responseObserver);
requestObserver.onNext(
- PutArtifactRequest.newBuilder().setData(ArtifactChunk.getDefaultInstance()).build());
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(ArtifactApi.ArtifactChunk.getDefaultInstance())
+ .build());
responseObserver.awaitTerminalState();
@@ -177,36 +191,48 @@ public class LocalFileSystemArtifactStagerServiceTest {
@Test
public void commitManifestWithAllArtifactsSucceeds() {
- ArtifactMetadata firstArtifact = stageBytes("first-artifact", "foo, bar, baz, quux".getBytes());
- ArtifactMetadata secondArtifact = stageBytes("second-artifact", "spam, ham, eggs".getBytes());
-
- Manifest manifest =
- Manifest.newBuilder().addArtifact(firstArtifact).addArtifact(secondArtifact).build();
-
- RecordingStreamObserver<CommitManifestResponse> commitResponseObserver =
+ ArtifactApi.ArtifactMetadata firstArtifact =
+ stageBytes("first-artifact", "foo, bar, baz, quux".getBytes());
+ ArtifactApi.ArtifactMetadata secondArtifact =
+ stageBytes("second-artifact", "spam, ham, eggs".getBytes());
+
+ ArtifactApi.Manifest manifest =
+ ArtifactApi.Manifest.newBuilder()
+ .addArtifact(firstArtifact)
+ .addArtifact(secondArtifact)
+ .build();
+
+ RecordingStreamObserver<ArtifactApi.CommitManifestResponse> commitResponseObserver =
new RecordingStreamObserver<>();
stub.commitManifest(
- CommitManifestRequest.newBuilder().setManifest(manifest).build(), commitResponseObserver);
+ ArtifactApi.CommitManifestRequest.newBuilder().setManifest(manifest).build(),
+ commitResponseObserver);
commitResponseObserver.awaitTerminalState();
assertThat(commitResponseObserver.completed, is(true));
assertThat(commitResponseObserver.responses, Matchers.hasSize(1));
- CommitManifestResponse commitResponse = commitResponseObserver.responses.get(0);
+ ArtifactApi.CommitManifestResponse commitResponse = commitResponseObserver.responses.get(0);
assertThat(commitResponse.getStagingToken(), Matchers.not(Matchers.nullValue()));
}
@Test
public void commitManifestWithMissingArtifactFails() {
- ArtifactMetadata firstArtifact = stageBytes("first-artifact", "foo, bar, baz, quux".getBytes());
- ArtifactMetadata absentArtifact = ArtifactMetadata.newBuilder().setName("absent").build();
-
- Manifest manifest =
- Manifest.newBuilder().addArtifact(firstArtifact).addArtifact(absentArtifact).build();
-
- RecordingStreamObserver<CommitManifestResponse> commitResponseObserver =
+ ArtifactApi.ArtifactMetadata firstArtifact =
+ stageBytes("first-artifact", "foo, bar, baz, quux".getBytes());
+ ArtifactApi.ArtifactMetadata absentArtifact =
+ ArtifactApi.ArtifactMetadata.newBuilder().setName("absent").build();
+
+ ArtifactApi.Manifest manifest =
+ ArtifactApi.Manifest.newBuilder()
+ .addArtifact(firstArtifact)
+ .addArtifact(absentArtifact)
+ .build();
+
+ RecordingStreamObserver<ArtifactApi.CommitManifestResponse> commitResponseObserver =
new RecordingStreamObserver<>();
- stub.commitManifest(CommitManifestRequest.newBuilder().setManifest(manifest).build(),
+ stub.commitManifest(
+ ArtifactApi.CommitManifestRequest.newBuilder().setManifest(manifest).build(),
commitResponseObserver);
commitResponseObserver.awaitTerminalState();
@@ -214,19 +240,20 @@ public class LocalFileSystemArtifactStagerServiceTest {
assertThat(commitResponseObserver.error, Matchers.not(Matchers.nullValue()));
}
- private ArtifactMetadata stageBytes(String name, byte[] bytes) {
- StreamObserver<PutArtifactRequest> requests =
- stub.putArtifact(new RecordingStreamObserver<PutArtifactResponse>());
+ private ArtifactApi.ArtifactMetadata stageBytes(String name, byte[] bytes) {
+ StreamObserver<ArtifactApi.PutArtifactRequest> requests =
+ stub.putArtifact(new RecordingStreamObserver<ArtifactApi.PutArtifactResponse>());
requests.onNext(
- PutArtifactRequest.newBuilder()
- .setMetadata(ArtifactMetadata.newBuilder().setName(name).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setMetadata(ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build())
.build());
requests.onNext(
- PutArtifactRequest.newBuilder()
- .setData(ArtifactChunk.newBuilder().setData(ByteString.copyFrom(bytes)).build())
+ ArtifactApi.PutArtifactRequest.newBuilder()
+ .setData(
+ ArtifactApi.ArtifactChunk.newBuilder().setData(ByteString.copyFrom(bytes)).build())
.build());
requests.onCompleted();
- return ArtifactMetadata.newBuilder().setName(name).build();
+ return ArtifactApi.ArtifactMetadata.newBuilder().setName(name).build();
}
private static class RecordingStreamObserver<T> implements StreamObserver<T> {
http://git-wip-us.apache.org/repos/asf/beam/blob/ed772692/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java
----------------------------------------------------------------------
diff --git a/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java
index fc884ad..ded09ea 100644
--- a/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java
+++ b/runners/reference/job-server/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java
@@ -20,14 +20,14 @@ package org.apache.beam.runners.reference.job;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
-import org.apache.beam.sdk.common.runner.v1.JobApi;
-import org.apache.beam.sdk.common.runner.v1.JobApi.CancelJobRequest;
-import org.apache.beam.sdk.common.runner.v1.JobApi.CancelJobResponse;
-import org.apache.beam.sdk.common.runner.v1.JobApi.GetJobStateRequest;
-import org.apache.beam.sdk.common.runner.v1.JobApi.GetJobStateResponse;
-import org.apache.beam.sdk.common.runner.v1.JobApi.PrepareJobResponse;
-import org.apache.beam.sdk.common.runner.v1.JobApi.RunJobRequest;
-import org.apache.beam.sdk.common.runner.v1.JobServiceGrpc.JobServiceImplBase;
+import org.apache.beam.model.jobmanagement.v1.JobApi;
+import org.apache.beam.model.jobmanagement.v1.JobApi.CancelJobRequest;
+import org.apache.beam.model.jobmanagement.v1.JobApi.CancelJobResponse;
+import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateRequest;
+import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateResponse;
+import org.apache.beam.model.jobmanagement.v1.JobApi.PrepareJobResponse;
+import org.apache.beam.model.jobmanagement.v1.JobApi.RunJobRequest;
+import org.apache.beam.model.jobmanagement.v1.JobServiceGrpc.JobServiceImplBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
[03/12] beam git commit: Move sdks/common/fn-api protos to
model/fn-execution
Posted by ke...@apache.org.
Move sdks/common/fn-api protos to model/fn-execution
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/e233af9a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/e233af9a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/e233af9a
Branch: refs/heads/master
Commit: e233af9a2ad8c45153315d1daffbc5191eb176a8
Parents: 2f3af31
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Oct 5 14:15:37 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:53:37 2017 -0700
----------------------------------------------------------------------
model/fn-execution/pom.xml | 32 +
.../src/main/proto/beam_fn_api.proto | 726 +++++++++++++++++++
.../src/main/proto/beam_provision_api.proto | 60 ++
.../org/apache/beam/fn/v1/standard_coders.yaml | 195 +++++
pom.xml | 13 -
runners/apex/pom.xml | 2 +-
runners/core-java/pom.xml | 2 +-
runners/direct-java/pom.xml | 2 +-
runners/flink/pom.xml | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 2 +-
runners/spark/pom.xml | 2 +-
sdks/common/fn-api/pom.xml | 114 ---
.../fn-api/src/main/proto/beam_fn_api.proto | 726 -------------------
.../src/main/proto/beam_provision_api.proto | 60 --
.../org/apache/beam/fn/v1/standard_coders.yaml | 195 -----
sdks/common/pom.xml | 1 -
sdks/java/core/pom.xml | 2 +-
sdks/java/harness/pom.xml | 12 +-
18 files changed, 1026 insertions(+), 1122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/model/fn-execution/pom.xml
----------------------------------------------------------------------
diff --git a/model/fn-execution/pom.xml b/model/fn-execution/pom.xml
index 807feb6..b5b5fdf 100644
--- a/model/fn-execution/pom.xml
+++ b/model/fn-execution/pom.xml
@@ -79,4 +79,36 @@
</plugin>
</plugins>
</build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-pipeline</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/model/fn-execution/src/main/proto/beam_fn_api.proto
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/main/proto/beam_fn_api.proto b/model/fn-execution/src/main/proto/beam_fn_api.proto
new file mode 100644
index 0000000..5a01077
--- /dev/null
+++ b/model/fn-execution/src/main/proto/beam_fn_api.proto
@@ -0,0 +1,726 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Fn API and boostrapping.
+ *
+ * TODO: Usage of plural names in lists looks awkward in Java
+ * e.g. getOutputsMap, addCodersBuilder
+ *
+ * TODO: gRPC / proto field names conflict with generated code
+ * e.g. "class" in java, "output" in python
+ */
+
+syntax = "proto3";
+
+/* TODO: Consider consolidating common components in another package
+ * and lanaguage namespaces for re-use with Runner Api.
+ */
+
+package org.apache.beam.fn.v1;
+
+option java_package = "org.apache.beam.fn.v1";
+option java_outer_classname = "BeamFnApi";
+
+import "beam_runner_api.proto";
+import "endpoints.proto";
+import "google/protobuf/timestamp.proto";
+
+/*
+ * Constructs that define the pipeline shape.
+ *
+ * These are mostly unstable due to the missing pieces to be shared with
+ * the Runner Api like windowing strategy, display data, .... There are still
+ * some modelling questions related to whether a side input is modelled
+ * as another field on a PrimitiveTransform or as part of inputs and we
+ * still are missing things like the CompositeTransform.
+ */
+
+// A representation of an input or output definition on a primitive transform.
+// Stable
+message Target {
+ // A repeated list of target definitions.
+ message List {
+ repeated Target target = 1;
+ }
+
+ // (Required) The id of the PrimitiveTransform which is the target.
+ string primitive_transform_reference = 1;
+
+ // (Required) The local name of an input or output defined on the primitive
+ // transform.
+ string name = 2;
+}
+
+// A descriptor for connecting to a remote port using the Beam Fn Data API.
+// Allows for communication between two environments (for example between the
+// runner and the SDK).
+// Stable
+message RemoteGrpcPort {
+ // (Required) An API descriptor which describes where to
+ // connect to including any authentication that is required.
+ org.apache.beam.portability.v1.ApiServiceDescriptor api_service_descriptor = 1;
+}
+
+/*
+ * Control Plane API
+ *
+ * Progress reporting and splitting still need further vetting. Also, this may change
+ * with the addition of new types of instructions/responses related to metrics.
+ */
+
+// An API that describes the work that a SDK harness is meant to do.
+// Stable
+service BeamFnControl {
+ // Instructions sent by the runner to the SDK requesting different types
+ // of work.
+ rpc Control(
+ // A stream of responses to instructions the SDK was asked to be performed.
+ stream InstructionResponse
+ ) returns (
+ // A stream of instructions requested of the SDK to be performed.
+ stream InstructionRequest
+ ) {}
+}
+
+// A request sent by a runner which the SDK is asked to fulfill.
+// For any unsupported request type, an error should be returned with a
+// matching instruction id.
+// Stable
+message InstructionRequest {
+ // (Required) An unique identifier provided by the runner which represents
+ // this requests execution. The InstructionResponse MUST have the matching id.
+ string instruction_id = 1;
+
+ // (Required) A request that the SDK Harness needs to interpret.
+ oneof request {
+ RegisterRequest register = 1000;
+ ProcessBundleRequest process_bundle = 1001;
+ ProcessBundleProgressRequest process_bundle_progress = 1002;
+ ProcessBundleSplitRequest process_bundle_split = 1003;
+ }
+}
+
+// The response for an associated request the SDK had been asked to fulfill.
+// Stable
+message InstructionResponse {
+ // (Required) A reference provided by the runner which represents a requests
+ // execution. The InstructionResponse MUST have the matching id when
+ // responding to the runner.
+ string instruction_id = 1;
+
+ // If this is specified, then this instruction has failed.
+ // A human readable string representing the reason as to why processing has
+ // failed.
+ string error = 2;
+
+ // If the instruction did not fail, it is required to return an equivalent
+ // response type depending on the request this matches.
+ oneof response {
+ RegisterResponse register = 1000;
+ ProcessBundleResponse process_bundle = 1001;
+ ProcessBundleProgressResponse process_bundle_progress = 1002;
+ ProcessBundleSplitResponse process_bundle_split = 1003;
+ }
+}
+
+// A list of objects which can be referred to by the runner in
+// future requests.
+// Stable
+message RegisterRequest {
+ // (Optional) The set of descriptors used to process bundles.
+ repeated ProcessBundleDescriptor process_bundle_descriptor = 1;
+}
+
+// Stable
+message RegisterResponse {
+}
+
+// Definitions that should be used to construct the bundle processing graph.
+message ProcessBundleDescriptor {
+ // (Required) A pipeline level unique id which can be used as a reference to
+ // refer to this.
+ string id = 1;
+
+ // (Required) A map from pipeline-scoped id to PTransform.
+ map<string, org.apache.beam.runner_api.v1.PTransform> transforms = 2;
+
+ // (Required) A map from pipeline-scoped id to PCollection.
+ map<string, org.apache.beam.runner_api.v1.PCollection> pcollections = 3;
+
+ // (Required) A map from pipeline-scoped id to WindowingStrategy.
+ map<string, org.apache.beam.runner_api.v1.WindowingStrategy> windowing_strategies = 4;
+
+ // (Required) A map from pipeline-scoped id to Coder.
+ map<string, org.apache.beam.runner_api.v1.Coder> coders = 5;
+
+ // (Required) A map from pipeline-scoped id to Environment.
+ map<string, org.apache.beam.runner_api.v1.Environment> environments = 6;
+
+ // A descriptor describing the end point to use for State API
+ // calls. Required if the Runner intends to send remote references over the
+ // data plane or if any of the transforms rely on user state or side inputs.
+ org.apache.beam.portability.v1.ApiServiceDescriptor state_api_service_descriptor = 7;
+}
+
+// A request to process a given bundle.
+// Stable
+message ProcessBundleRequest {
+ // (Required) A reference to the process bundle descriptor that must be
+ // instantiated and executed by the SDK harness.
+ string process_bundle_descriptor_reference = 1;
+
+ // (Optional) A list of cache tokens that can be used by an SDK to reuse
+ // cached data returned by the State API across multiple bundles.
+ repeated bytes cache_tokens = 2;
+}
+
+// Stable
+message ProcessBundleResponse {
+ // (Optional) If metrics reporting is supported by the SDK, this represents
+ // the final metrics to record for this bundle.
+ Metrics metrics = 1;
+}
+
+// A request to report progress information for a given bundle.
+// This is an optional request to be handled and is used to support advanced
+// SDK features such as SplittableDoFn, user level metrics etc.
+message ProcessBundleProgressRequest {
+ // (Required) A reference to an active process bundle request with the given
+ // instruction id.
+ string instruction_reference = 1;
+}
+
+message Metrics {
+ // PTransform level metrics.
+ // These metrics are split into processed and active element groups for
+ // progress reporting purposes. This allows a Runner to see what is measured,
+ // what is estimated and what can be extrapolated to be able to accurately
+ // estimate the backlog of remaining work.
+ message PTransform {
+ // Metrics that are measured for processed and active element groups.
+ message Measured {
+ // (Required) Map from local input name to number of elements processed
+ // from this input.
+ map<string, int64> input_element_counts = 1;
+
+ // (Required) Map from local output name to number of elements produced
+ // for this output.
+ map<string, int64> output_element_counts = 2;
+
+ // (Optional) The total time spent so far in processing the elements in
+ // this group.
+ int64 total_time_spent = 3;
+
+ // TODO: Add other element group level metrics.
+ }
+
+ // Metrics for fully processed elements.
+ message ProcessedElements {
+ // (Required)
+ Measured measured = 1;
+ }
+
+ // Metrics for active elements.
+ // An element is considered active if the SDK has started but not finished
+ // processing it yet.
+ message ActiveElements {
+ // (Required)
+ Measured measured = 1;
+
+ // Estimated metrics.
+
+ // (Optional) Sum of estimated fraction of known work remaining for all
+ // active elements, as reported by this transform.
+ // If not reported, a Runner could extrapolate this from the processed
+ // elements.
+ // TODO: Handle the case when known work is infinite.
+ double fraction_remaining = 2;
+
+ // (Optional) Map from local output name to sum of estimated number
+ // of elements remaining for this output from all active elements,
+ // as reported by this transform.
+ // If not reported, a Runner could extrapolate this from the processed
+ // elements.
+ map<string, int64> output_elements_remaining = 3;
+ }
+
+ // (Required): Metrics for processed elements.
+ ProcessedElements processed_elements = 1;
+ // (Required): Metrics for active elements.
+ ActiveElements active_elements = 2;
+
+ // (Optional): Map from local output name to its watermark.
+ // The watermarks reported are tentative, to get a better sense of progress
+ // while processing a bundle but before it is committed. At bundle commit
+ // time, a Runner needs to also take into account the timers set to compute
+ // the actual watermarks.
+ map<string, int64> watermarks = 3;
+
+ // TODO: Define other transform level system metrics.
+ }
+
+ // User defined metrics
+ message User {
+ // TODO: Define it.
+ }
+
+ map<string, PTransform> ptransforms = 1;
+ map<string, User> user = 2;
+}
+
+message ProcessBundleProgressResponse {
+ // (Required)
+ Metrics metrics = 1;
+}
+
+message ProcessBundleSplitRequest {
+ // (Required) A reference to an active process bundle request with the given
+ // instruction id.
+ string instruction_reference = 1;
+
+ // (Required) The fraction of work (when compared to the known amount of work)
+ // the process bundle request should try to split at.
+ double fraction = 2;
+}
+
+// urn:org.apache.beam:restriction:element-count:1.0
+message ElementCountRestriction {
+ // A restriction representing the number of elements that should be processed.
+ // Effectively the range [0, count]
+ int64 count = 1;
+}
+
+// urn:org.apache.beam:restriction:element-count-skip:1.0
+message ElementCountSkipRestriction {
+ // A restriction representing the number of elements that should be skipped.
+ // Effectively the range (count, infinity]
+ int64 count = 1;
+}
+
+// Each primitive transform that is splittable is defined by a restriction
+// it is currently processing. During splitting, that currently active
+// restriction (R_initial) is split into 2 components:
+// * a restriction (R_done) representing all elements that will be fully
+// processed
+// * a restriction (R_todo) representing all elements that will not be fully
+// processed
+//
+// where:
+// R_initial = R_done ⋃ R_todo
+message PrimitiveTransformSplit {
+ // (Required) A reference to a primitive transform with the given id that
+ // is part of the active process bundle request with the given instruction
+ // id.
+ string primitive_transform_reference = 1;
+
+ // (Required) A function specification describing the restriction
+ // that has been completed by the primitive transform.
+ //
+ // For example, a remote GRPC source will have a specific urn and data
+ // block containing an ElementCountRestriction.
+ org.apache.beam.runner_api.v1.FunctionSpec completed_restriction = 2;
+
+ // (Required) A function specification describing the restriction
+ // representing the remainder of work for the primitive transform.
+ //
+ // FOr example, a remote GRPC source will have a specific urn and data
+ // block contain an ElemntCountSkipRestriction.
+ org.apache.beam.runner_api.v1.FunctionSpec remaining_restriction = 3;
+}
+
+message ProcessBundleSplitResponse {
+ // (Optional) A set of split responses for a currently active work item.
+ //
+ // If primitive transform B is a descendant of primitive transform A and both
+ // A and B report a split. Then B's restriction is reported as an element
+ // restriction pair and thus the fully reported restriction is:
+ // R = A_done
+ // ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ A_todo
+ // If there is a decendant of B named C, then C would similarly report a
+ // set of element pair restrictions.
+ //
+ // This restriction is processed and completed by the currently active process
+ // bundle request:
+ // A_done ⋃ (A_boundary ⋂ B_done)
+ // and these restrictions will be processed by future process bundle requests:
+ // A_boundary â‹‚ B_todo (passed to SDF B directly)
+ // A_todo (passed to SDF A directly)
+
+ // If primitive transform B and C are siblings and descendants of A and A, B,
+ // and C report a split. Then B and C's restrictions are relative to A's.
+ // R = A_done
+ // ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ (A_boundary ⋂ B_todo)
+ // ⋃ (A_boundary ⋂ C_todo)
+ // ⋃ A_todo
+ // If there is no descendant of B or C also reporting a split, than
+ // B_boundary = ∅ and C_boundary = ∅
+ //
+ // This restriction is processed and completed by the currently active process
+ // bundle request:
+ // A_done ⋃ (A_boundary ⋂ B_done)
+ // ⋃ (A_boundary ⋂ C_done)
+ // and these restrictions will be processed by future process bundle requests:
+ // A_boundary â‹‚ B_todo (passed to SDF B directly)
+ // A_boundary â‹‚ C_todo (passed to SDF C directly)
+ // A_todo (passed to SDF A directly)
+ //
+ // Note that descendants splits should only be reported if it is inexpensive
+ // to compute the boundary restriction intersected with descendants splits.
+ // Also note, that the boundary restriction may represent a set of elements
+ // produced by a parent primitive transform which can not be split at each
+ // element or that there are intermediate unsplittable primitive transforms
+ // between an ancestor splittable function and a descendant splittable
+ // function which may have more than one output per element. Finally note
+ // that the descendant splits should only be reported if the split
+ // information is relatively compact.
+ repeated PrimitiveTransformSplit splits = 1;
+}
+
+/*
+ * Data Plane API
+ */
+
+// Messages used to represent logical byte streams.
+// Stable
+message Elements {
+ // Represents multiple encoded elements in nested context for a given named
+ // instruction and target.
+ message Data {
+ // (Required) A reference to an active instruction request with the given
+ // instruction id.
+ string instruction_reference = 1;
+
+ // (Required) A definition representing a consumer or producer of this data.
+ // If received by a harness, this represents the consumer within that
+ // harness that should consume these bytes. If sent by a harness, this
+ // represents the producer of these bytes.
+ //
+ // Note that a single element may span multiple Data messages.
+ //
+ // Note that a sending/receiving pair should share the same target
+ // identifier.
+ Target target = 2;
+
+ // (Optional) Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // concatenated together.
+ //
+ // An empty data block represents the end of stream for the given
+ // instruction and target.
+ bytes data = 3;
+ }
+
+ // (Required) A list containing parts of logical byte streams.
+ repeated Data data = 1;
+}
+
+// Stable
+service BeamFnData {
+ // Used to send data between harnesses.
+ rpc Data(
+ // A stream of data representing input.
+ stream Elements
+ ) returns (
+ // A stream of data representing output.
+ stream Elements
+ ) {}
+}
+
+/*
+ * State API
+ */
+
+message StateRequest {
+ // (Required) An unique identifier provided by the SDK which represents this
+ // requests execution. The StateResponse corresponding with this request
+ // will have the matching id.
+ string id = 1;
+
+ // (Required) The associated instruction id of the work that is currently
+ // being processed. This allows for the runner to associate any modifications
+ // to state to be committed with the appropriate work execution.
+ string instruction_reference = 2;
+
+ // (Required) The state key this request is for.
+ StateKey state_key = 3;
+
+ // (Required) The action to take on this request.
+ oneof request {
+ // A request to get state.
+ StateGetRequest get = 1000;
+
+ // A request to append to state.
+ StateAppendRequest append = 1001;
+
+ // A request to clear state.
+ StateClearRequest clear = 1002;
+ }
+}
+
+message StateResponse {
+ // (Required) A reference provided by the SDK which represents a requests
+ // execution. The StateResponse must have the matching id when responding
+ // to the SDK.
+ string id = 1;
+
+ // (Optional) If this is specified, then the state request has failed.
+ // A human readable string representing the reason as to why the request
+ // failed.
+ string error = 2;
+
+ // (Optional) If this is specified, then the result of this state request
+ // can be cached using the supplied token.
+ bytes cache_token = 3;
+
+ // A corresponding response matching the request will be populated.
+ oneof response {
+ // A response to getting state.
+ StateGetResponse get = 1000;
+
+ // A response to appending to state.
+ StateAppendResponse append = 1001;
+
+ // A response to clearing state.
+ StateClearResponse clear = 1002;
+ }
+}
+
+service BeamFnState {
+ // Used to get/append/clear state stored by the runner on behalf of the SDK.
+ rpc State(
+ // A stream of state instructions requested of the runner.
+ stream StateRequest
+ ) returns (
+ // A stream of responses to state instructions the runner was asked to be
+ // performed.
+ stream StateResponse
+ ) {}
+}
+
+message StateKey {
+ message Runner {
+ // (Required) Opaque information supplied by the runner. Used to support
+ // remote references.
+ bytes key = 1;
+ }
+
+ message MultimapSideInput {
+ // (Required) The id of the PTransform containing a side input.
+ string ptransform_id = 1;
+ // (Required) The id of the side input.
+ string side_input_id = 2;
+ // (Required) The window (after mapping the currently executing elements
+ // window into the side input windows domain) encoded in a nested context.
+ bytes window = 3;
+ // (Required) The key encoded in a nested context.
+ bytes key = 4;
+ }
+
+ message BagUserState {
+ // (Required) The id of the PTransform containing user state.
+ string ptransform_id = 1;
+ // (Required) The id of the user state.
+ string user_state_id = 2;
+ // (Required) The window encoded in a nested context.
+ bytes window = 3;
+ // (Required) The key of the currently executing element encoded in a
+ // nested context.
+ bytes key = 4;
+ }
+
+ // (Required) One of the following state keys must be set.
+ oneof type {
+ Runner runner = 1;
+ MultimapSideInput multimap_side_input = 2;
+ BagUserState bag_user_state = 3;
+ // TODO: represent a state key for user map state
+ }
+}
+
+// A request to get state.
+message StateGetRequest {
+ // (Optional) If specified, signals to the runner that the response
+ // should resume from the following continuation token.
+ //
+ // If unspecified, signals to the runner that the response should start
+ // from the beginning of the logical continuable stream.
+ bytes continuation_token = 1;
+}
+
+// A response to get state representing a logical byte stream which can be
+// continued using the state API.
+message StateGetResponse {
+ // (Optional) If specified, represents a token which can be used with the
+ // state API to get the next chunk of this logical byte stream. The end of
+ // the logical byte stream is signalled by this field being unset.
+ bytes continuation_token = 1;
+
+ // Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // concatenated together.
+ bytes data = 2;
+}
+
+// A request to append state.
+message StateAppendRequest {
+ // Represents a part of a logical byte stream. Elements within
+ // the logical byte stream are encoded in the nested context and
+ // multiple append requests are concatenated together.
+ bytes data = 1;
+}
+
+// A response to append state.
+message StateAppendResponse {
+}
+
+// A request to clear state.
+message StateClearRequest {
+}
+
+// A response to clear state.
+message StateClearResponse {
+}
+
+/*
+ * Logging API
+ *
+ * This is very stable. There can be some changes to how we define a LogEntry,
+ * to increase/decrease the severity types, the way we format an exception/stack
+ * trace, or the log site.
+ */
+
+// A log entry
+message LogEntry {
+ // A list of log entries, enables buffering and batching of multiple
+ // log messages using the logging API.
+ message List {
+ // (Required) One or or more log messages.
+ repeated LogEntry log_entries = 1;
+ }
+
+ // The severity of the event described in a log entry, expressed as one of the
+ // severity levels listed below. For your reference, the levels are
+ // assigned the listed numeric values. The effect of using numeric values
+ // other than those listed is undefined.
+ //
+ // If you are writing log entries, you should map other severity encodings to
+ // one of these standard levels. For example, you might map all of
+ // Java's FINE, FINER, and FINEST levels to `Severity.DEBUG`.
+ //
+ // This list is intentionally not comprehensive; the intent is to provide a
+ // common set of "good enough" severity levels so that logging front ends
+ // can provide filtering and searching across log types. Users of the API are
+ // free not to use all severity levels in their log messages.
+ message Severity {
+ enum Enum {
+ UNSPECIFIED = 0;
+ // Trace level information, also the default log level unless
+ // another severity is specified.
+ TRACE = 1;
+ // Debugging information.
+ DEBUG = 2;
+ // Normal events.
+ INFO = 3;
+ // Normal but significant events, such as start up, shut down, or
+ // configuration.
+ NOTICE = 4;
+ // Warning events might cause problems.
+ WARN = 5;
+ // Error events are likely to cause problems.
+ ERROR = 6;
+ // Critical events cause severe problems or brief outages and may
+ // indicate that a person must take action.
+ CRITICAL = 7;
+ }
+ }
+
+ // (Required) The severity of the log statement.
+ Severity.Enum severity = 1;
+
+ // (Required) The time at which this log statement occurred.
+ google.protobuf.Timestamp timestamp = 2;
+
+ // (Required) A human readable message.
+ string message = 3;
+
+ // (Optional) An optional trace of the functions involved. For example, in
+ // Java this can include multiple causes and multiple suppressed exceptions.
+ string trace = 4;
+
+ // (Optional) A reference to the instruction this log statement is associated
+ // with.
+ string instruction_reference = 5;
+
+ // (Optional) A reference to the primitive transform this log statement is
+ // associated with.
+ string primitive_transform_reference = 6;
+
+ // (Optional) Human-readable name of the function or method being invoked,
+ // with optional context such as the class or package name. The format can
+ // vary by language. For example:
+ // qual.if.ied.Class.method (Java)
+ // dir/package.func (Go)
+ // module.function (Python)
+ // file.cc:382 (C++)
+ string log_location = 7;
+
+ // (Optional) The name of the thread this log statement is associated with.
+ string thread = 8;
+}
+
+message LogControl {
+}
+
+// Stable
+service BeamFnLogging {
+ // Allows for the SDK to emit log entries which the runner can
+ // associate with the active job.
+ rpc Logging(
+ // A stream of log entries batched into lists emitted by the SDK harness.
+ stream LogEntry.List
+ ) returns (
+ // A stream of log control messages used to configure the SDK.
+ stream LogControl
+ ) {}
+}
+
+/*
+ * Environment types
+ */
+// A Docker container configuration for launching the SDK harness to execute
+// user specified functions.
+message DockerContainer {
+ // (Required) A pipeline level unique id which can be used as a reference to
+ // refer to this.
+ string id = 1;
+
+ // (Required) The Docker container URI
+ // For example "dataflow.gcr.io/v1beta3/java-batch:1.5.1"
+ string uri = 2;
+
+ // (Optional) Docker registry specification.
+ // If unspecified, the uri is expected to be able to be fetched without
+ // requiring additional configuration by a runner.
+ string registry_reference = 3;
+}
+
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/model/fn-execution/src/main/proto/beam_provision_api.proto
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/main/proto/beam_provision_api.proto b/model/fn-execution/src/main/proto/beam_provision_api.proto
new file mode 100644
index 0000000..b0cd6b4
--- /dev/null
+++ b/model/fn-execution/src/main/proto/beam_provision_api.proto
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Protocol Buffers describing the Provision API, for communicating with a runner
+ * for job and environment provisioning information over GRPC.
+ */
+
+syntax = "proto3";
+
+package org.apache.beam.fn.v1;
+
+option java_package = "org.apache.beam.fn.v1";
+option java_outer_classname = "ProvisionApi";
+
+import "google/protobuf/struct.proto";
+
+// A service to provide runtime provisioning information to the SDK harness
+// worker instances -- such as pipeline options, resource constraints and
+// other job metadata -- needed by an SDK harness instance to initialize.
+service ProvisionService {
+ // Get provision information for the SDK harness worker instance.
+ rpc GetProvisionInfo(GetProvisionInfoRequest) returns (GetProvisionInfoResponse);
+}
+
+// A request to get the provision info of a SDK harness worker instance.
+message GetProvisionInfoRequest { }
+
+// A response containing the provision info of a SDK harness worker instance.
+message GetProvisionInfoResponse {
+ ProvisionInfo info = 1;
+}
+
+// Runtime provisioning information for a SDK harness worker instance,
+// such as pipeline options, resource constraints and other job metadata
+message ProvisionInfo {
+ // (required) The job ID.
+ string job_id = 1;
+ // (required) The job name.
+ string job_name = 2;
+
+ // (required) Pipeline options. For non-template jobs, the options are
+ // identical to what is passed to job submission.
+ google.protobuf.Struct pipeline_options = 3;
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml b/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
new file mode 100644
index 0000000..f37b2d3
--- /dev/null
+++ b/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
@@ -0,0 +1,195 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is broken into multiple sections delimited by ---. Each section specifies a set of
+# reference encodings for a single standardized coder used in a specific context.
+#
+# Each section contains up to 3 properties:
+#
+# coder: a common coder spec. Currently, a URN and URNs for component coders as necessary.
+# nested: a boolean meaning whether the coder was used in the nested context. Missing means to
+# test both contexts, a shorthand for when the coder is invariant across context.
+# examples: a map of {encoded bytes: original JSON object} encoded with the coder in the context.
+# The LHS (key) is a byte array encoded as a JSON-escaped string. The RHS (value) is
+# one of a few standard JSON types such as numbers, strings, dicts that map naturally
+# to the type encoded by the coder.
+#
+# These choices were made to strike a balance between portability, ease of use, and simple
+# legibility of this file itself.
+#
+# It is expected that future work will move the `coder` field into a format that it would be
+# represented by the Runner API, so that it can be understood by all SDKs and harnesses.
+#
+# If a coder is marked non-deterministic in the coder spec, then only the decoding should be validated.
+
+
+coder:
+ urn: "urn:beam:coders:bytes:0.1"
+nested: false
+examples:
+ "abc": abc
+ "ab\0c": "ab\0c"
+
+---
+
+coder:
+ urn: "urn:beam:coders:bytes:0.1"
+nested: true
+examples:
+ "\u0003abc": abc
+ "\u0004ab\0c": "ab\0c"
+ "\u00c8\u0001 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|":
+ " 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|"
+
+---
+
+coder:
+ urn: "urn:beam:coders:varint:0.1"
+examples:
+ "\0": 0
+ "\u0001": 1
+ "\u000A": 10
+ "\u00c8\u0001": 200
+ "\u00e8\u0007": 1000
+ "\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u0001": -1
+
+---
+
+coder:
+ urn: "urn:beam:coders:kv:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"},
+ {urn: "urn:beam:coders:varint:0.1"}]
+examples:
+ "\u0003abc\0": {key: abc, value: 0}
+ "\u0004ab\0c\u000A": {key: "ab\0c", value: 10}
+
+---
+
+coder:
+ urn: "urn:beam:coders:kv:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"},
+ {urn: "urn:beam:coders:bytes:0.1"}]
+nested: false
+examples:
+ "\u0003abcdef": {key: abc, value: def}
+ "\u0004ab\0cde\0f": {key: "ab\0c", value: "de\0f"}
+
+---
+
+coder:
+ urn: "urn:beam:coders:kv:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"},
+ {urn: "urn:beam:coders:bytes:0.1"}]
+nested: true
+examples:
+ "\u0003abc\u0003def": {key: abc, value: def}
+ "\u0004ab\0c\u0004de\0f": {key: "ab\0c", value: "de\0f"}
+
+---
+
+coder:
+ urn: "urn:beam:coders:interval_window:0.1"
+examples:
+ "\u0080\u0000\u0001\u0052\u009a\u00a4\u009b\u0068\u0080\u00dd\u00db\u0001" : {end: 1454293425000, span: 3600000}
+ "\u0080\u0000\u0001\u0053\u0034\u00ec\u0074\u00e8\u0080\u0090\u00fb\u00d3\u0009" : {end: 1456881825000, span: 2592000000}
+ "\u007f\u00df\u003b\u0064\u005a\u001c\u00ad\u0076\u00ed\u0002" : {end: -9223372036854410, span: 365}
+ "\u0080\u0020\u00c4\u009b\u00a5\u00e3\u0053\u00f7\u0000" : {end: 9223372036854775, span: 0}
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:varint:0.1"}]
+examples:
+ "\0\0\0\u0001\0": [0]
+ "\0\0\0\u0004\u0001\n\u00c8\u0001\u00e8\u0007": [1, 10, 200, 1000]
+ "\0\0\0\0": []
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"}]
+examples:
+ "\0\0\0\u0001\u0003abc": ["abc"]
+ "\0\0\0\u0002\u0004ab\0c\u0004de\0f": ["ab\0c", "de\0f"]
+ "\0\0\0\0": []
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"}]
+ # This is for iterables of unknown length, where the encoding is not
+ # deterministic.
+ non_deterministic: True
+examples:
+ "\u00ff\u00ff\u00ff\u00ff\u0000": []
+ "\u00ff\u00ff\u00ff\u00ff\u0001\u0003abc\u0000": ["abc"]
+ "\u00ff\u00ff\u00ff\u00ff\u0002\u0004ab\u0000c\u0004de\u0000f\u0000": ["ab\0c", "de\0f"]
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:global_window:0.1"}]
+examples:
+ "\0\0\0\u0001": [""]
+
+---
+
+coder:
+ urn: "urn:beam:coders:global_window:0.1"
+examples:
+ "": ""
+
+---
+
+# All windowed values consist of pane infos that represent NO_FIRING until full support is added
+# in the Python SDK (BEAM-1522).
+coder:
+ urn: "urn:beam:coders:windowed_value:0.1"
+ components: [{urn: "urn:beam:coders:varint:0.1"},
+ {urn: "urn:beam:coders:global_window:0.1"}]
+examples:
+ "\u0080\0\u0001R\u009a\u00a4\u009bh\0\0\0\u0001\u000f\u0002": {
+ value: 2,
+ timestamp: 1454293425000,
+ pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
+ windows: ["global"]
+ }
+
+---
+
+coder:
+ urn: "urn:beam:coders:windowed_value:0.1"
+ components: [{urn: "urn:beam:coders:varint:0.1"},
+ {urn: "urn:beam:coders:interval_window:0.1"}]
+examples:
+ "\u007f\u00ff\u00ff\u00ff\u00ff\u00f9\u00e5\u0080\0\0\0\u0001\u0080\0\u0001R\u009a\u00a4\u009bh\u00c0\u008b\u0011\u000f\u0004": {
+ value: 4,
+ timestamp: -400000,
+ pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
+ windows: [{end: 1454293425000, span: 280000}]
+ }
+
+ "\u007f\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u009c\0\0\0\u0002\u0080\0\u0001R\u009a\u00a4\u009bh\u0080\u00dd\u00db\u0001\u007f\u00df;dZ\u001c\u00adv\u00ed\u0002\u000f\u0002": {
+ value: 2,
+ timestamp: -100,
+ pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
+ windows: [{end: 1454293425000, span: 3600000}, {end: -9223372036854410, span: 365}]
+ }
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 212d703..efedb1d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -428,19 +428,6 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/runners/apex/pom.xml
----------------------------------------------------------------------
diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index 8ade583..f70e67e 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -179,7 +179,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/runners/core-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index 668c08c..087e24d 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -69,7 +69,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/runners/direct-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 4e28438..6e356fc 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -327,7 +327,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/runners/flink/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index effbdb4..e77dbc8 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -371,7 +371,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 873bb2e..da0d237 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -466,7 +466,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 3e9095a..0ba6125 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -333,7 +333,7 @@
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/sdks/common/fn-api/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/common/fn-api/pom.xml b/sdks/common/fn-api/pom.xml
deleted file mode 100644
index 498f83a..0000000
--- a/sdks/common/fn-api/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <packaging>jar</packaging>
- <parent>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-parent</artifactId>
- <version>2.3.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>beam-sdks-common-fn-api</artifactId>
- <name>Apache Beam :: SDKs :: Common :: Fn API</name>
- <description>This artifact generates the stub bindings.</description>
-
- <build>
- <resources>
- <resource>
- <directory>src/test/resources</directory>
- <filtering>true</filtering>
- </resource>
- <resource>
- <directory>${project.build.directory}/original_sources_to_package</directory>
- </resource>
- </resources>
-
- <plugins>
- <!-- Skip the checkstyle plugin on generated code -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <!-- Skip the findbugs plugin on generated code -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <configuration>
- <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <pluginId>grpc-java</pluginId>
- <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>compile-custom</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-model-pipeline</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- </dependency>
-
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-stub</artifactId>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/sdks/common/fn-api/src/main/proto/beam_fn_api.proto
----------------------------------------------------------------------
diff --git a/sdks/common/fn-api/src/main/proto/beam_fn_api.proto b/sdks/common/fn-api/src/main/proto/beam_fn_api.proto
deleted file mode 100644
index 5a01077..0000000
--- a/sdks/common/fn-api/src/main/proto/beam_fn_api.proto
+++ /dev/null
@@ -1,726 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Fn API and boostrapping.
- *
- * TODO: Usage of plural names in lists looks awkward in Java
- * e.g. getOutputsMap, addCodersBuilder
- *
- * TODO: gRPC / proto field names conflict with generated code
- * e.g. "class" in java, "output" in python
- */
-
-syntax = "proto3";
-
-/* TODO: Consider consolidating common components in another package
- * and lanaguage namespaces for re-use with Runner Api.
- */
-
-package org.apache.beam.fn.v1;
-
-option java_package = "org.apache.beam.fn.v1";
-option java_outer_classname = "BeamFnApi";
-
-import "beam_runner_api.proto";
-import "endpoints.proto";
-import "google/protobuf/timestamp.proto";
-
-/*
- * Constructs that define the pipeline shape.
- *
- * These are mostly unstable due to the missing pieces to be shared with
- * the Runner Api like windowing strategy, display data, .... There are still
- * some modelling questions related to whether a side input is modelled
- * as another field on a PrimitiveTransform or as part of inputs and we
- * still are missing things like the CompositeTransform.
- */
-
-// A representation of an input or output definition on a primitive transform.
-// Stable
-message Target {
- // A repeated list of target definitions.
- message List {
- repeated Target target = 1;
- }
-
- // (Required) The id of the PrimitiveTransform which is the target.
- string primitive_transform_reference = 1;
-
- // (Required) The local name of an input or output defined on the primitive
- // transform.
- string name = 2;
-}
-
-// A descriptor for connecting to a remote port using the Beam Fn Data API.
-// Allows for communication between two environments (for example between the
-// runner and the SDK).
-// Stable
-message RemoteGrpcPort {
- // (Required) An API descriptor which describes where to
- // connect to including any authentication that is required.
- org.apache.beam.portability.v1.ApiServiceDescriptor api_service_descriptor = 1;
-}
-
-/*
- * Control Plane API
- *
- * Progress reporting and splitting still need further vetting. Also, this may change
- * with the addition of new types of instructions/responses related to metrics.
- */
-
-// An API that describes the work that a SDK harness is meant to do.
-// Stable
-service BeamFnControl {
- // Instructions sent by the runner to the SDK requesting different types
- // of work.
- rpc Control(
- // A stream of responses to instructions the SDK was asked to be performed.
- stream InstructionResponse
- ) returns (
- // A stream of instructions requested of the SDK to be performed.
- stream InstructionRequest
- ) {}
-}
-
-// A request sent by a runner which the SDK is asked to fulfill.
-// For any unsupported request type, an error should be returned with a
-// matching instruction id.
-// Stable
-message InstructionRequest {
- // (Required) An unique identifier provided by the runner which represents
- // this requests execution. The InstructionResponse MUST have the matching id.
- string instruction_id = 1;
-
- // (Required) A request that the SDK Harness needs to interpret.
- oneof request {
- RegisterRequest register = 1000;
- ProcessBundleRequest process_bundle = 1001;
- ProcessBundleProgressRequest process_bundle_progress = 1002;
- ProcessBundleSplitRequest process_bundle_split = 1003;
- }
-}
-
-// The response for an associated request the SDK had been asked to fulfill.
-// Stable
-message InstructionResponse {
- // (Required) A reference provided by the runner which represents a requests
- // execution. The InstructionResponse MUST have the matching id when
- // responding to the runner.
- string instruction_id = 1;
-
- // If this is specified, then this instruction has failed.
- // A human readable string representing the reason as to why processing has
- // failed.
- string error = 2;
-
- // If the instruction did not fail, it is required to return an equivalent
- // response type depending on the request this matches.
- oneof response {
- RegisterResponse register = 1000;
- ProcessBundleResponse process_bundle = 1001;
- ProcessBundleProgressResponse process_bundle_progress = 1002;
- ProcessBundleSplitResponse process_bundle_split = 1003;
- }
-}
-
-// A list of objects which can be referred to by the runner in
-// future requests.
-// Stable
-message RegisterRequest {
- // (Optional) The set of descriptors used to process bundles.
- repeated ProcessBundleDescriptor process_bundle_descriptor = 1;
-}
-
-// Stable
-message RegisterResponse {
-}
-
-// Definitions that should be used to construct the bundle processing graph.
-message ProcessBundleDescriptor {
- // (Required) A pipeline level unique id which can be used as a reference to
- // refer to this.
- string id = 1;
-
- // (Required) A map from pipeline-scoped id to PTransform.
- map<string, org.apache.beam.runner_api.v1.PTransform> transforms = 2;
-
- // (Required) A map from pipeline-scoped id to PCollection.
- map<string, org.apache.beam.runner_api.v1.PCollection> pcollections = 3;
-
- // (Required) A map from pipeline-scoped id to WindowingStrategy.
- map<string, org.apache.beam.runner_api.v1.WindowingStrategy> windowing_strategies = 4;
-
- // (Required) A map from pipeline-scoped id to Coder.
- map<string, org.apache.beam.runner_api.v1.Coder> coders = 5;
-
- // (Required) A map from pipeline-scoped id to Environment.
- map<string, org.apache.beam.runner_api.v1.Environment> environments = 6;
-
- // A descriptor describing the end point to use for State API
- // calls. Required if the Runner intends to send remote references over the
- // data plane or if any of the transforms rely on user state or side inputs.
- org.apache.beam.portability.v1.ApiServiceDescriptor state_api_service_descriptor = 7;
-}
-
-// A request to process a given bundle.
-// Stable
-message ProcessBundleRequest {
- // (Required) A reference to the process bundle descriptor that must be
- // instantiated and executed by the SDK harness.
- string process_bundle_descriptor_reference = 1;
-
- // (Optional) A list of cache tokens that can be used by an SDK to reuse
- // cached data returned by the State API across multiple bundles.
- repeated bytes cache_tokens = 2;
-}
-
-// Stable
-message ProcessBundleResponse {
- // (Optional) If metrics reporting is supported by the SDK, this represents
- // the final metrics to record for this bundle.
- Metrics metrics = 1;
-}
-
-// A request to report progress information for a given bundle.
-// This is an optional request to be handled and is used to support advanced
-// SDK features such as SplittableDoFn, user level metrics etc.
-message ProcessBundleProgressRequest {
- // (Required) A reference to an active process bundle request with the given
- // instruction id.
- string instruction_reference = 1;
-}
-
-message Metrics {
- // PTransform level metrics.
- // These metrics are split into processed and active element groups for
- // progress reporting purposes. This allows a Runner to see what is measured,
- // what is estimated and what can be extrapolated to be able to accurately
- // estimate the backlog of remaining work.
- message PTransform {
- // Metrics that are measured for processed and active element groups.
- message Measured {
- // (Required) Map from local input name to number of elements processed
- // from this input.
- map<string, int64> input_element_counts = 1;
-
- // (Required) Map from local output name to number of elements produced
- // for this output.
- map<string, int64> output_element_counts = 2;
-
- // (Optional) The total time spent so far in processing the elements in
- // this group.
- int64 total_time_spent = 3;
-
- // TODO: Add other element group level metrics.
- }
-
- // Metrics for fully processed elements.
- message ProcessedElements {
- // (Required)
- Measured measured = 1;
- }
-
- // Metrics for active elements.
- // An element is considered active if the SDK has started but not finished
- // processing it yet.
- message ActiveElements {
- // (Required)
- Measured measured = 1;
-
- // Estimated metrics.
-
- // (Optional) Sum of estimated fraction of known work remaining for all
- // active elements, as reported by this transform.
- // If not reported, a Runner could extrapolate this from the processed
- // elements.
- // TODO: Handle the case when known work is infinite.
- double fraction_remaining = 2;
-
- // (Optional) Map from local output name to sum of estimated number
- // of elements remaining for this output from all active elements,
- // as reported by this transform.
- // If not reported, a Runner could extrapolate this from the processed
- // elements.
- map<string, int64> output_elements_remaining = 3;
- }
-
- // (Required): Metrics for processed elements.
- ProcessedElements processed_elements = 1;
- // (Required): Metrics for active elements.
- ActiveElements active_elements = 2;
-
- // (Optional): Map from local output name to its watermark.
- // The watermarks reported are tentative, to get a better sense of progress
- // while processing a bundle but before it is committed. At bundle commit
- // time, a Runner needs to also take into account the timers set to compute
- // the actual watermarks.
- map<string, int64> watermarks = 3;
-
- // TODO: Define other transform level system metrics.
- }
-
- // User defined metrics
- message User {
- // TODO: Define it.
- }
-
- map<string, PTransform> ptransforms = 1;
- map<string, User> user = 2;
-}
-
-message ProcessBundleProgressResponse {
- // (Required)
- Metrics metrics = 1;
-}
-
-message ProcessBundleSplitRequest {
- // (Required) A reference to an active process bundle request with the given
- // instruction id.
- string instruction_reference = 1;
-
- // (Required) The fraction of work (when compared to the known amount of work)
- // the process bundle request should try to split at.
- double fraction = 2;
-}
-
-// urn:org.apache.beam:restriction:element-count:1.0
-message ElementCountRestriction {
- // A restriction representing the number of elements that should be processed.
- // Effectively the range [0, count]
- int64 count = 1;
-}
-
-// urn:org.apache.beam:restriction:element-count-skip:1.0
-message ElementCountSkipRestriction {
- // A restriction representing the number of elements that should be skipped.
- // Effectively the range (count, infinity]
- int64 count = 1;
-}
-
-// Each primitive transform that is splittable is defined by a restriction
-// it is currently processing. During splitting, that currently active
-// restriction (R_initial) is split into 2 components:
-// * a restriction (R_done) representing all elements that will be fully
-// processed
-// * a restriction (R_todo) representing all elements that will not be fully
-// processed
-//
-// where:
-// R_initial = R_done ⋃ R_todo
-message PrimitiveTransformSplit {
- // (Required) A reference to a primitive transform with the given id that
- // is part of the active process bundle request with the given instruction
- // id.
- string primitive_transform_reference = 1;
-
- // (Required) A function specification describing the restriction
- // that has been completed by the primitive transform.
- //
- // For example, a remote GRPC source will have a specific urn and data
- // block containing an ElementCountRestriction.
- org.apache.beam.runner_api.v1.FunctionSpec completed_restriction = 2;
-
- // (Required) A function specification describing the restriction
- // representing the remainder of work for the primitive transform.
- //
- // FOr example, a remote GRPC source will have a specific urn and data
- // block contain an ElemntCountSkipRestriction.
- org.apache.beam.runner_api.v1.FunctionSpec remaining_restriction = 3;
-}
-
-message ProcessBundleSplitResponse {
- // (Optional) A set of split responses for a currently active work item.
- //
- // If primitive transform B is a descendant of primitive transform A and both
- // A and B report a split. Then B's restriction is reported as an element
- // restriction pair and thus the fully reported restriction is:
- // R = A_done
- // ⋃ (A_boundary ⋂ B_done)
- // ⋃ (A_boundary ⋂ B_todo)
- // ⋃ A_todo
- // If there is a decendant of B named C, then C would similarly report a
- // set of element pair restrictions.
- //
- // This restriction is processed and completed by the currently active process
- // bundle request:
- // A_done ⋃ (A_boundary ⋂ B_done)
- // and these restrictions will be processed by future process bundle requests:
- // A_boundary â‹‚ B_todo (passed to SDF B directly)
- // A_todo (passed to SDF A directly)
-
- // If primitive transform B and C are siblings and descendants of A and A, B,
- // and C report a split. Then B and C's restrictions are relative to A's.
- // R = A_done
- // ⋃ (A_boundary ⋂ B_done)
- // ⋃ (A_boundary ⋂ B_todo)
- // ⋃ (A_boundary ⋂ B_todo)
- // ⋃ (A_boundary ⋂ C_todo)
- // ⋃ A_todo
- // If there is no descendant of B or C also reporting a split, than
- // B_boundary = ∅ and C_boundary = ∅
- //
- // This restriction is processed and completed by the currently active process
- // bundle request:
- // A_done ⋃ (A_boundary ⋂ B_done)
- // ⋃ (A_boundary ⋂ C_done)
- // and these restrictions will be processed by future process bundle requests:
- // A_boundary â‹‚ B_todo (passed to SDF B directly)
- // A_boundary â‹‚ C_todo (passed to SDF C directly)
- // A_todo (passed to SDF A directly)
- //
- // Note that descendants splits should only be reported if it is inexpensive
- // to compute the boundary restriction intersected with descendants splits.
- // Also note, that the boundary restriction may represent a set of elements
- // produced by a parent primitive transform which can not be split at each
- // element or that there are intermediate unsplittable primitive transforms
- // between an ancestor splittable function and a descendant splittable
- // function which may have more than one output per element. Finally note
- // that the descendant splits should only be reported if the split
- // information is relatively compact.
- repeated PrimitiveTransformSplit splits = 1;
-}
-
-/*
- * Data Plane API
- */
-
-// Messages used to represent logical byte streams.
-// Stable
-message Elements {
- // Represents multiple encoded elements in nested context for a given named
- // instruction and target.
- message Data {
- // (Required) A reference to an active instruction request with the given
- // instruction id.
- string instruction_reference = 1;
-
- // (Required) A definition representing a consumer or producer of this data.
- // If received by a harness, this represents the consumer within that
- // harness that should consume these bytes. If sent by a harness, this
- // represents the producer of these bytes.
- //
- // Note that a single element may span multiple Data messages.
- //
- // Note that a sending/receiving pair should share the same target
- // identifier.
- Target target = 2;
-
- // (Optional) Represents a part of a logical byte stream. Elements within
- // the logical byte stream are encoded in the nested context and
- // concatenated together.
- //
- // An empty data block represents the end of stream for the given
- // instruction and target.
- bytes data = 3;
- }
-
- // (Required) A list containing parts of logical byte streams.
- repeated Data data = 1;
-}
-
-// Stable
-service BeamFnData {
- // Used to send data between harnesses.
- rpc Data(
- // A stream of data representing input.
- stream Elements
- ) returns (
- // A stream of data representing output.
- stream Elements
- ) {}
-}
-
-/*
- * State API
- */
-
-message StateRequest {
- // (Required) An unique identifier provided by the SDK which represents this
- // requests execution. The StateResponse corresponding with this request
- // will have the matching id.
- string id = 1;
-
- // (Required) The associated instruction id of the work that is currently
- // being processed. This allows for the runner to associate any modifications
- // to state to be committed with the appropriate work execution.
- string instruction_reference = 2;
-
- // (Required) The state key this request is for.
- StateKey state_key = 3;
-
- // (Required) The action to take on this request.
- oneof request {
- // A request to get state.
- StateGetRequest get = 1000;
-
- // A request to append to state.
- StateAppendRequest append = 1001;
-
- // A request to clear state.
- StateClearRequest clear = 1002;
- }
-}
-
-message StateResponse {
- // (Required) A reference provided by the SDK which represents a requests
- // execution. The StateResponse must have the matching id when responding
- // to the SDK.
- string id = 1;
-
- // (Optional) If this is specified, then the state request has failed.
- // A human readable string representing the reason as to why the request
- // failed.
- string error = 2;
-
- // (Optional) If this is specified, then the result of this state request
- // can be cached using the supplied token.
- bytes cache_token = 3;
-
- // A corresponding response matching the request will be populated.
- oneof response {
- // A response to getting state.
- StateGetResponse get = 1000;
-
- // A response to appending to state.
- StateAppendResponse append = 1001;
-
- // A response to clearing state.
- StateClearResponse clear = 1002;
- }
-}
-
-service BeamFnState {
- // Used to get/append/clear state stored by the runner on behalf of the SDK.
- rpc State(
- // A stream of state instructions requested of the runner.
- stream StateRequest
- ) returns (
- // A stream of responses to state instructions the runner was asked to be
- // performed.
- stream StateResponse
- ) {}
-}
-
-message StateKey {
- message Runner {
- // (Required) Opaque information supplied by the runner. Used to support
- // remote references.
- bytes key = 1;
- }
-
- message MultimapSideInput {
- // (Required) The id of the PTransform containing a side input.
- string ptransform_id = 1;
- // (Required) The id of the side input.
- string side_input_id = 2;
- // (Required) The window (after mapping the currently executing elements
- // window into the side input windows domain) encoded in a nested context.
- bytes window = 3;
- // (Required) The key encoded in a nested context.
- bytes key = 4;
- }
-
- message BagUserState {
- // (Required) The id of the PTransform containing user state.
- string ptransform_id = 1;
- // (Required) The id of the user state.
- string user_state_id = 2;
- // (Required) The window encoded in a nested context.
- bytes window = 3;
- // (Required) The key of the currently executing element encoded in a
- // nested context.
- bytes key = 4;
- }
-
- // (Required) One of the following state keys must be set.
- oneof type {
- Runner runner = 1;
- MultimapSideInput multimap_side_input = 2;
- BagUserState bag_user_state = 3;
- // TODO: represent a state key for user map state
- }
-}
-
-// A request to get state.
-message StateGetRequest {
- // (Optional) If specified, signals to the runner that the response
- // should resume from the following continuation token.
- //
- // If unspecified, signals to the runner that the response should start
- // from the beginning of the logical continuable stream.
- bytes continuation_token = 1;
-}
-
-// A response to get state representing a logical byte stream which can be
-// continued using the state API.
-message StateGetResponse {
- // (Optional) If specified, represents a token which can be used with the
- // state API to get the next chunk of this logical byte stream. The end of
- // the logical byte stream is signalled by this field being unset.
- bytes continuation_token = 1;
-
- // Represents a part of a logical byte stream. Elements within
- // the logical byte stream are encoded in the nested context and
- // concatenated together.
- bytes data = 2;
-}
-
-// A request to append state.
-message StateAppendRequest {
- // Represents a part of a logical byte stream. Elements within
- // the logical byte stream are encoded in the nested context and
- // multiple append requests are concatenated together.
- bytes data = 1;
-}
-
-// A response to append state.
-message StateAppendResponse {
-}
-
-// A request to clear state.
-message StateClearRequest {
-}
-
-// A response to clear state.
-message StateClearResponse {
-}
-
-/*
- * Logging API
- *
- * This is very stable. There can be some changes to how we define a LogEntry,
- * to increase/decrease the severity types, the way we format an exception/stack
- * trace, or the log site.
- */
-
-// A log entry
-message LogEntry {
- // A list of log entries, enables buffering and batching of multiple
- // log messages using the logging API.
- message List {
- // (Required) One or or more log messages.
- repeated LogEntry log_entries = 1;
- }
-
- // The severity of the event described in a log entry, expressed as one of the
- // severity levels listed below. For your reference, the levels are
- // assigned the listed numeric values. The effect of using numeric values
- // other than those listed is undefined.
- //
- // If you are writing log entries, you should map other severity encodings to
- // one of these standard levels. For example, you might map all of
- // Java's FINE, FINER, and FINEST levels to `Severity.DEBUG`.
- //
- // This list is intentionally not comprehensive; the intent is to provide a
- // common set of "good enough" severity levels so that logging front ends
- // can provide filtering and searching across log types. Users of the API are
- // free not to use all severity levels in their log messages.
- message Severity {
- enum Enum {
- UNSPECIFIED = 0;
- // Trace level information, also the default log level unless
- // another severity is specified.
- TRACE = 1;
- // Debugging information.
- DEBUG = 2;
- // Normal events.
- INFO = 3;
- // Normal but significant events, such as start up, shut down, or
- // configuration.
- NOTICE = 4;
- // Warning events might cause problems.
- WARN = 5;
- // Error events are likely to cause problems.
- ERROR = 6;
- // Critical events cause severe problems or brief outages and may
- // indicate that a person must take action.
- CRITICAL = 7;
- }
- }
-
- // (Required) The severity of the log statement.
- Severity.Enum severity = 1;
-
- // (Required) The time at which this log statement occurred.
- google.protobuf.Timestamp timestamp = 2;
-
- // (Required) A human readable message.
- string message = 3;
-
- // (Optional) An optional trace of the functions involved. For example, in
- // Java this can include multiple causes and multiple suppressed exceptions.
- string trace = 4;
-
- // (Optional) A reference to the instruction this log statement is associated
- // with.
- string instruction_reference = 5;
-
- // (Optional) A reference to the primitive transform this log statement is
- // associated with.
- string primitive_transform_reference = 6;
-
- // (Optional) Human-readable name of the function or method being invoked,
- // with optional context such as the class or package name. The format can
- // vary by language. For example:
- // qual.if.ied.Class.method (Java)
- // dir/package.func (Go)
- // module.function (Python)
- // file.cc:382 (C++)
- string log_location = 7;
-
- // (Optional) The name of the thread this log statement is associated with.
- string thread = 8;
-}
-
-message LogControl {
-}
-
-// Stable
-service BeamFnLogging {
- // Allows for the SDK to emit log entries which the runner can
- // associate with the active job.
- rpc Logging(
- // A stream of log entries batched into lists emitted by the SDK harness.
- stream LogEntry.List
- ) returns (
- // A stream of log control messages used to configure the SDK.
- stream LogControl
- ) {}
-}
-
-/*
- * Environment types
- */
-// A Docker container configuration for launching the SDK harness to execute
-// user specified functions.
-message DockerContainer {
- // (Required) A pipeline level unique id which can be used as a reference to
- // refer to this.
- string id = 1;
-
- // (Required) The Docker container URI
- // For example "dataflow.gcr.io/v1beta3/java-batch:1.5.1"
- string uri = 2;
-
- // (Optional) Docker registry specification.
- // If unspecified, the uri is expected to be able to be fetched without
- // requiring additional configuration by a runner.
- string registry_reference = 3;
-}
-
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/sdks/common/fn-api/src/main/proto/beam_provision_api.proto
----------------------------------------------------------------------
diff --git a/sdks/common/fn-api/src/main/proto/beam_provision_api.proto b/sdks/common/fn-api/src/main/proto/beam_provision_api.proto
deleted file mode 100644
index b0cd6b4..0000000
--- a/sdks/common/fn-api/src/main/proto/beam_provision_api.proto
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Protocol Buffers describing the Provision API, for communicating with a runner
- * for job and environment provisioning information over GRPC.
- */
-
-syntax = "proto3";
-
-package org.apache.beam.fn.v1;
-
-option java_package = "org.apache.beam.fn.v1";
-option java_outer_classname = "ProvisionApi";
-
-import "google/protobuf/struct.proto";
-
-// A service to provide runtime provisioning information to the SDK harness
-// worker instances -- such as pipeline options, resource constraints and
-// other job metadata -- needed by an SDK harness instance to initialize.
-service ProvisionService {
- // Get provision information for the SDK harness worker instance.
- rpc GetProvisionInfo(GetProvisionInfoRequest) returns (GetProvisionInfoResponse);
-}
-
-// A request to get the provision info of a SDK harness worker instance.
-message GetProvisionInfoRequest { }
-
-// A response containing the provision info of a SDK harness worker instance.
-message GetProvisionInfoResponse {
- ProvisionInfo info = 1;
-}
-
-// Runtime provisioning information for a SDK harness worker instance,
-// such as pipeline options, resource constraints and other job metadata
-message ProvisionInfo {
- // (required) The job ID.
- string job_id = 1;
- // (required) The job name.
- string job_name = 2;
-
- // (required) Pipeline options. For non-template jobs, the options are
- // identical to what is passed to job submission.
- google.protobuf.Struct pipeline_options = 3;
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/sdks/common/fn-api/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
----------------------------------------------------------------------
diff --git a/sdks/common/fn-api/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml b/sdks/common/fn-api/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
deleted file mode 100644
index f37b2d3..0000000
--- a/sdks/common/fn-api/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
+++ /dev/null
@@ -1,195 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This file is broken into multiple sections delimited by ---. Each section specifies a set of
-# reference encodings for a single standardized coder used in a specific context.
-#
-# Each section contains up to 3 properties:
-#
-# coder: a common coder spec. Currently, a URN and URNs for component coders as necessary.
-# nested: a boolean meaning whether the coder was used in the nested context. Missing means to
-# test both contexts, a shorthand for when the coder is invariant across context.
-# examples: a map of {encoded bytes: original JSON object} encoded with the coder in the context.
-# The LHS (key) is a byte array encoded as a JSON-escaped string. The RHS (value) is
-# one of a few standard JSON types such as numbers, strings, dicts that map naturally
-# to the type encoded by the coder.
-#
-# These choices were made to strike a balance between portability, ease of use, and simple
-# legibility of this file itself.
-#
-# It is expected that future work will move the `coder` field into a format that it would be
-# represented by the Runner API, so that it can be understood by all SDKs and harnesses.
-#
-# If a coder is marked non-deterministic in the coder spec, then only the decoding should be validated.
-
-
-coder:
- urn: "urn:beam:coders:bytes:0.1"
-nested: false
-examples:
- "abc": abc
- "ab\0c": "ab\0c"
-
----
-
-coder:
- urn: "urn:beam:coders:bytes:0.1"
-nested: true
-examples:
- "\u0003abc": abc
- "\u0004ab\0c": "ab\0c"
- "\u00c8\u0001 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|":
- " 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|"
-
----
-
-coder:
- urn: "urn:beam:coders:varint:0.1"
-examples:
- "\0": 0
- "\u0001": 1
- "\u000A": 10
- "\u00c8\u0001": 200
- "\u00e8\u0007": 1000
- "\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u0001": -1
-
----
-
-coder:
- urn: "urn:beam:coders:kv:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"},
- {urn: "urn:beam:coders:varint:0.1"}]
-examples:
- "\u0003abc\0": {key: abc, value: 0}
- "\u0004ab\0c\u000A": {key: "ab\0c", value: 10}
-
----
-
-coder:
- urn: "urn:beam:coders:kv:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"},
- {urn: "urn:beam:coders:bytes:0.1"}]
-nested: false
-examples:
- "\u0003abcdef": {key: abc, value: def}
- "\u0004ab\0cde\0f": {key: "ab\0c", value: "de\0f"}
-
----
-
-coder:
- urn: "urn:beam:coders:kv:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"},
- {urn: "urn:beam:coders:bytes:0.1"}]
-nested: true
-examples:
- "\u0003abc\u0003def": {key: abc, value: def}
- "\u0004ab\0c\u0004de\0f": {key: "ab\0c", value: "de\0f"}
-
----
-
-coder:
- urn: "urn:beam:coders:interval_window:0.1"
-examples:
- "\u0080\u0000\u0001\u0052\u009a\u00a4\u009b\u0068\u0080\u00dd\u00db\u0001" : {end: 1454293425000, span: 3600000}
- "\u0080\u0000\u0001\u0053\u0034\u00ec\u0074\u00e8\u0080\u0090\u00fb\u00d3\u0009" : {end: 1456881825000, span: 2592000000}
- "\u007f\u00df\u003b\u0064\u005a\u001c\u00ad\u0076\u00ed\u0002" : {end: -9223372036854410, span: 365}
- "\u0080\u0020\u00c4\u009b\u00a5\u00e3\u0053\u00f7\u0000" : {end: 9223372036854775, span: 0}
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:varint:0.1"}]
-examples:
- "\0\0\0\u0001\0": [0]
- "\0\0\0\u0004\u0001\n\u00c8\u0001\u00e8\u0007": [1, 10, 200, 1000]
- "\0\0\0\0": []
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"}]
-examples:
- "\0\0\0\u0001\u0003abc": ["abc"]
- "\0\0\0\u0002\u0004ab\0c\u0004de\0f": ["ab\0c", "de\0f"]
- "\0\0\0\0": []
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"}]
- # This is for iterables of unknown length, where the encoding is not
- # deterministic.
- non_deterministic: True
-examples:
- "\u00ff\u00ff\u00ff\u00ff\u0000": []
- "\u00ff\u00ff\u00ff\u00ff\u0001\u0003abc\u0000": ["abc"]
- "\u00ff\u00ff\u00ff\u00ff\u0002\u0004ab\u0000c\u0004de\u0000f\u0000": ["ab\0c", "de\0f"]
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:global_window:0.1"}]
-examples:
- "\0\0\0\u0001": [""]
-
----
-
-coder:
- urn: "urn:beam:coders:global_window:0.1"
-examples:
- "": ""
-
----
-
-# All windowed values consist of pane infos that represent NO_FIRING until full support is added
-# in the Python SDK (BEAM-1522).
-coder:
- urn: "urn:beam:coders:windowed_value:0.1"
- components: [{urn: "urn:beam:coders:varint:0.1"},
- {urn: "urn:beam:coders:global_window:0.1"}]
-examples:
- "\u0080\0\u0001R\u009a\u00a4\u009bh\0\0\0\u0001\u000f\u0002": {
- value: 2,
- timestamp: 1454293425000,
- pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
- windows: ["global"]
- }
-
----
-
-coder:
- urn: "urn:beam:coders:windowed_value:0.1"
- components: [{urn: "urn:beam:coders:varint:0.1"},
- {urn: "urn:beam:coders:interval_window:0.1"}]
-examples:
- "\u007f\u00ff\u00ff\u00ff\u00ff\u00f9\u00e5\u0080\0\0\0\u0001\u0080\0\u0001R\u009a\u00a4\u009bh\u00c0\u008b\u0011\u000f\u0004": {
- value: 4,
- timestamp: -400000,
- pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
- windows: [{end: 1454293425000, span: 280000}]
- }
-
- "\u007f\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u009c\0\0\0\u0002\u0080\0\u0001R\u009a\u00a4\u009bh\u0080\u00dd\u00db\u0001\u007f\u00df;dZ\u001c\u00adv\u00ed\u0002\u000f\u0002": {
- value: 2,
- timestamp: -100,
- pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
- windows: [{end: 1454293425000, span: 3600000}, {end: -9223372036854410, span: 365}]
- }
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/sdks/common/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/common/pom.xml b/sdks/common/pom.xml
index d950759..d4ecad0 100644
--- a/sdks/common/pom.xml
+++ b/sdks/common/pom.xml
@@ -33,6 +33,5 @@
<name>Apache Beam :: SDKs :: Common</name>
<modules>
- <module>fn-api</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 36271eb..6c74cd9 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -302,7 +302,7 @@
<!-- test dependencies -->
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
+ <artifactId>beam-model-fn-execution</artifactId>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
http://git-wip-us.apache.org/repos/asf/beam/blob/e233af9a/sdks/java/harness/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/harness/pom.xml b/sdks/java/harness/pom.xml
index 7f9156c..de24f7f 100644
--- a/sdks/java/harness/pom.xml
+++ b/sdks/java/harness/pom.xml
@@ -51,10 +51,10 @@
<include>com.google.guava:guava</include>
<!-- java harness dependencies that are not staged -->
<include>org.apache.beam:beam-model-pipeline</include>
+ <include>org.apache.beam:beam-model-fn-execution</include>
<include>org.apache.beam:beam-runners-core-construction-java</include>
<include>org.apache.beam:beam-runners-core-java</include>
<include>org.apache.beam:beam-runners-google-cloud-dataflow-java</include>
- <include>org.apache.beam:beam-sdks-common-fn-api</include>
<include>io.netty:netty-transport-native-epoll</include>
</includes>
</artifactSet>
@@ -131,6 +131,11 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-model-fn-execution</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
@@ -164,11 +169,6 @@
</dependency>
<dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-common-fn-api</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
[11/12] beam git commit: Re-namespace model/fn-execution to
model.fn_execution
Posted by ke...@apache.org.
Re-namespace model/fn-execution to model.fn_execution
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/80c6f4ec
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/80c6f4ec
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/80c6f4ec
Branch: refs/heads/master
Commit: 80c6f4ec0c2a3cc3a441289a9cc8ff53cb70f863
Parents: ed77269
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Oct 5 15:03:38 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 09:56:39 2017 -0700
----------------------------------------------------------------------
.../src/main/proto/beam_fn_api.proto | 4 +-
.../src/main/proto/beam_provision_api.proto | 4 +-
.../org/apache/beam/fn/v1/standard_coders.yaml | 195 -------------------
.../model/fnexecution/v1/standard_coders.yaml | 195 +++++++++++++++++++
.../runners/core/fn/FnApiControlClient.java | 2 +-
.../core/fn/FnApiControlClientPoolService.java | 4 +-
.../beam/runners/core/fn/FnDataService.java | 6 +-
.../beam/runners/core/fn/SdkHarnessClient.java | 10 +-
.../runners/core/fn/SdkHarnessDoFnRunner.java | 5 +-
.../fn/FnApiControlClientPoolServiceTest.java | 2 +-
.../runners/core/fn/FnApiControlClientTest.java | 2 +-
.../runners/core/fn/SdkHarnessClientTest.java | 2 +-
.../core/fn/SdkHarnessDoFnRunnerTest.java | 2 +-
.../apache/beam/sdk/coders/CommonCoderTest.java | 2 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 4 +-
.../beam/fn/harness/BeamFnDataWriteRunner.java | 4 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 6 +-
.../org/apache/beam/fn/harness/FnHarness.java | 2 +-
.../fn/harness/control/BeamFnControlClient.java | 10 +-
.../harness/control/ProcessBundleHandler.java | 12 +-
.../fn/harness/control/RegisterHandler.java | 6 +-
.../BeamFnDataBufferingOutboundObserver.java | 4 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 2 +-
.../fn/harness/data/BeamFnDataGrpcClient.java | 4 +-
.../harness/data/BeamFnDataGrpcMultiplexer.java | 4 +-
.../harness/data/BeamFnDataInboundObserver.java | 4 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 4 +-
.../beam/fn/harness/state/BagUserState.java | 6 +-
.../fn/harness/state/BeamFnStateClient.java | 4 +-
.../state/BeamFnStateGrpcClientCache.java | 6 +-
.../harness/state/StateFetchingIterators.java | 27 ++-
.../fn/harness/BeamFnDataReadRunnerTest.java | 2 +-
.../fn/harness/BeamFnDataWriteRunnerTest.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 2 +-
.../apache/beam/fn/harness/FnHarnessTest.java | 12 +-
.../channel/ManagedChannelFactoryTest.java | 19 +-
.../control/BeamFnControlClientTest.java | 4 +-
.../control/ProcessBundleHandlerTest.java | 6 +-
.../fn/harness/control/RegisterHandlerTest.java | 4 +-
...BeamFnDataBufferingOutboundObserverTest.java | 2 +-
.../harness/data/BeamFnDataGrpcClientTest.java | 4 +-
.../data/BeamFnDataGrpcMultiplexerTest.java | 2 +-
.../data/BeamFnDataInboundObserverTest.java | 2 +-
.../logging/BeamFnLoggingClientTest.java | 4 +-
.../beam/fn/harness/state/BagUserStateTest.java | 4 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 16 +-
.../fn/harness/state/FakeBeamFnStateClient.java | 16 +-
.../state/StateFetchingIteratorsTest.java | 6 +-
48 files changed, 325 insertions(+), 326 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/model/fn-execution/src/main/proto/beam_fn_api.proto
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/main/proto/beam_fn_api.proto b/model/fn-execution/src/main/proto/beam_fn_api.proto
index f519581..18c26ef 100644
--- a/model/fn-execution/src/main/proto/beam_fn_api.proto
+++ b/model/fn-execution/src/main/proto/beam_fn_api.proto
@@ -32,9 +32,9 @@ syntax = "proto3";
* and lanaguage namespaces for re-use with Runner Api.
*/
-package org.apache.beam.fn.v1;
+package org.apache.beam.model.fn_execution.v1;
-option java_package = "org.apache.beam.fn.v1";
+option java_package = "org.apache.beam.model.fnexecution.v1";
option java_outer_classname = "BeamFnApi";
import "beam_runner_api.proto";
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/model/fn-execution/src/main/proto/beam_provision_api.proto
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/main/proto/beam_provision_api.proto b/model/fn-execution/src/main/proto/beam_provision_api.proto
index b0cd6b4..ed5c78a 100644
--- a/model/fn-execution/src/main/proto/beam_provision_api.proto
+++ b/model/fn-execution/src/main/proto/beam_provision_api.proto
@@ -23,9 +23,9 @@
syntax = "proto3";
-package org.apache.beam.fn.v1;
+package org.apache.beam.model.fn_execution.v1;
-option java_package = "org.apache.beam.fn.v1";
+option java_package = "org.apache.beam.model.fnexecution.v1";
option java_outer_classname = "ProvisionApi";
import "google/protobuf/struct.proto";
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml b/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
deleted file mode 100644
index f37b2d3..0000000
--- a/model/fn-execution/src/test/resources/org/apache/beam/fn/v1/standard_coders.yaml
+++ /dev/null
@@ -1,195 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This file is broken into multiple sections delimited by ---. Each section specifies a set of
-# reference encodings for a single standardized coder used in a specific context.
-#
-# Each section contains up to 3 properties:
-#
-# coder: a common coder spec. Currently, a URN and URNs for component coders as necessary.
-# nested: a boolean meaning whether the coder was used in the nested context. Missing means to
-# test both contexts, a shorthand for when the coder is invariant across context.
-# examples: a map of {encoded bytes: original JSON object} encoded with the coder in the context.
-# The LHS (key) is a byte array encoded as a JSON-escaped string. The RHS (value) is
-# one of a few standard JSON types such as numbers, strings, dicts that map naturally
-# to the type encoded by the coder.
-#
-# These choices were made to strike a balance between portability, ease of use, and simple
-# legibility of this file itself.
-#
-# It is expected that future work will move the `coder` field into a format that it would be
-# represented by the Runner API, so that it can be understood by all SDKs and harnesses.
-#
-# If a coder is marked non-deterministic in the coder spec, then only the decoding should be validated.
-
-
-coder:
- urn: "urn:beam:coders:bytes:0.1"
-nested: false
-examples:
- "abc": abc
- "ab\0c": "ab\0c"
-
----
-
-coder:
- urn: "urn:beam:coders:bytes:0.1"
-nested: true
-examples:
- "\u0003abc": abc
- "\u0004ab\0c": "ab\0c"
- "\u00c8\u0001 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|":
- " 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|"
-
----
-
-coder:
- urn: "urn:beam:coders:varint:0.1"
-examples:
- "\0": 0
- "\u0001": 1
- "\u000A": 10
- "\u00c8\u0001": 200
- "\u00e8\u0007": 1000
- "\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u0001": -1
-
----
-
-coder:
- urn: "urn:beam:coders:kv:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"},
- {urn: "urn:beam:coders:varint:0.1"}]
-examples:
- "\u0003abc\0": {key: abc, value: 0}
- "\u0004ab\0c\u000A": {key: "ab\0c", value: 10}
-
----
-
-coder:
- urn: "urn:beam:coders:kv:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"},
- {urn: "urn:beam:coders:bytes:0.1"}]
-nested: false
-examples:
- "\u0003abcdef": {key: abc, value: def}
- "\u0004ab\0cde\0f": {key: "ab\0c", value: "de\0f"}
-
----
-
-coder:
- urn: "urn:beam:coders:kv:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"},
- {urn: "urn:beam:coders:bytes:0.1"}]
-nested: true
-examples:
- "\u0003abc\u0003def": {key: abc, value: def}
- "\u0004ab\0c\u0004de\0f": {key: "ab\0c", value: "de\0f"}
-
----
-
-coder:
- urn: "urn:beam:coders:interval_window:0.1"
-examples:
- "\u0080\u0000\u0001\u0052\u009a\u00a4\u009b\u0068\u0080\u00dd\u00db\u0001" : {end: 1454293425000, span: 3600000}
- "\u0080\u0000\u0001\u0053\u0034\u00ec\u0074\u00e8\u0080\u0090\u00fb\u00d3\u0009" : {end: 1456881825000, span: 2592000000}
- "\u007f\u00df\u003b\u0064\u005a\u001c\u00ad\u0076\u00ed\u0002" : {end: -9223372036854410, span: 365}
- "\u0080\u0020\u00c4\u009b\u00a5\u00e3\u0053\u00f7\u0000" : {end: 9223372036854775, span: 0}
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:varint:0.1"}]
-examples:
- "\0\0\0\u0001\0": [0]
- "\0\0\0\u0004\u0001\n\u00c8\u0001\u00e8\u0007": [1, 10, 200, 1000]
- "\0\0\0\0": []
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"}]
-examples:
- "\0\0\0\u0001\u0003abc": ["abc"]
- "\0\0\0\u0002\u0004ab\0c\u0004de\0f": ["ab\0c", "de\0f"]
- "\0\0\0\0": []
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:bytes:0.1"}]
- # This is for iterables of unknown length, where the encoding is not
- # deterministic.
- non_deterministic: True
-examples:
- "\u00ff\u00ff\u00ff\u00ff\u0000": []
- "\u00ff\u00ff\u00ff\u00ff\u0001\u0003abc\u0000": ["abc"]
- "\u00ff\u00ff\u00ff\u00ff\u0002\u0004ab\u0000c\u0004de\u0000f\u0000": ["ab\0c", "de\0f"]
-
----
-
-coder:
- urn: "urn:beam:coders:stream:0.1"
- components: [{urn: "urn:beam:coders:global_window:0.1"}]
-examples:
- "\0\0\0\u0001": [""]
-
----
-
-coder:
- urn: "urn:beam:coders:global_window:0.1"
-examples:
- "": ""
-
----
-
-# All windowed values consist of pane infos that represent NO_FIRING until full support is added
-# in the Python SDK (BEAM-1522).
-coder:
- urn: "urn:beam:coders:windowed_value:0.1"
- components: [{urn: "urn:beam:coders:varint:0.1"},
- {urn: "urn:beam:coders:global_window:0.1"}]
-examples:
- "\u0080\0\u0001R\u009a\u00a4\u009bh\0\0\0\u0001\u000f\u0002": {
- value: 2,
- timestamp: 1454293425000,
- pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
- windows: ["global"]
- }
-
----
-
-coder:
- urn: "urn:beam:coders:windowed_value:0.1"
- components: [{urn: "urn:beam:coders:varint:0.1"},
- {urn: "urn:beam:coders:interval_window:0.1"}]
-examples:
- "\u007f\u00ff\u00ff\u00ff\u00ff\u00f9\u00e5\u0080\0\0\0\u0001\u0080\0\u0001R\u009a\u00a4\u009bh\u00c0\u008b\u0011\u000f\u0004": {
- value: 4,
- timestamp: -400000,
- pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
- windows: [{end: 1454293425000, span: 280000}]
- }
-
- "\u007f\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u009c\0\0\0\u0002\u0080\0\u0001R\u009a\u00a4\u009bh\u0080\u00dd\u00db\u0001\u007f\u00df;dZ\u001c\u00adv\u00ed\u0002\u000f\u0002": {
- value: 2,
- timestamp: -100,
- pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
- windows: [{end: 1454293425000, span: 3600000}, {end: -9223372036854410, span: 365}]
- }
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/model/fn-execution/src/test/resources/org/apache/beam/model/fnexecution/v1/standard_coders.yaml
----------------------------------------------------------------------
diff --git a/model/fn-execution/src/test/resources/org/apache/beam/model/fnexecution/v1/standard_coders.yaml b/model/fn-execution/src/test/resources/org/apache/beam/model/fnexecution/v1/standard_coders.yaml
new file mode 100644
index 0000000..f37b2d3
--- /dev/null
+++ b/model/fn-execution/src/test/resources/org/apache/beam/model/fnexecution/v1/standard_coders.yaml
@@ -0,0 +1,195 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is broken into multiple sections delimited by ---. Each section specifies a set of
+# reference encodings for a single standardized coder used in a specific context.
+#
+# Each section contains up to 3 properties:
+#
+# coder: a common coder spec. Currently, a URN and URNs for component coders as necessary.
+# nested: a boolean meaning whether the coder was used in the nested context. Missing means to
+# test both contexts, a shorthand for when the coder is invariant across context.
+# examples: a map of {encoded bytes: original JSON object} encoded with the coder in the context.
+# The LHS (key) is a byte array encoded as a JSON-escaped string. The RHS (value) is
+# one of a few standard JSON types such as numbers, strings, dicts that map naturally
+# to the type encoded by the coder.
+#
+# These choices were made to strike a balance between portability, ease of use, and simple
+# legibility of this file itself.
+#
+# It is expected that future work will move the `coder` field into a format that it would be
+# represented by the Runner API, so that it can be understood by all SDKs and harnesses.
+#
+# If a coder is marked non-deterministic in the coder spec, then only the decoding should be validated.
+
+
+coder:
+ urn: "urn:beam:coders:bytes:0.1"
+nested: false
+examples:
+ "abc": abc
+ "ab\0c": "ab\0c"
+
+---
+
+coder:
+ urn: "urn:beam:coders:bytes:0.1"
+nested: true
+examples:
+ "\u0003abc": abc
+ "\u0004ab\0c": "ab\0c"
+ "\u00c8\u0001 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|":
+ " 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|"
+
+---
+
+coder:
+ urn: "urn:beam:coders:varint:0.1"
+examples:
+ "\0": 0
+ "\u0001": 1
+ "\u000A": 10
+ "\u00c8\u0001": 200
+ "\u00e8\u0007": 1000
+ "\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u0001": -1
+
+---
+
+coder:
+ urn: "urn:beam:coders:kv:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"},
+ {urn: "urn:beam:coders:varint:0.1"}]
+examples:
+ "\u0003abc\0": {key: abc, value: 0}
+ "\u0004ab\0c\u000A": {key: "ab\0c", value: 10}
+
+---
+
+coder:
+ urn: "urn:beam:coders:kv:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"},
+ {urn: "urn:beam:coders:bytes:0.1"}]
+nested: false
+examples:
+ "\u0003abcdef": {key: abc, value: def}
+ "\u0004ab\0cde\0f": {key: "ab\0c", value: "de\0f"}
+
+---
+
+coder:
+ urn: "urn:beam:coders:kv:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"},
+ {urn: "urn:beam:coders:bytes:0.1"}]
+nested: true
+examples:
+ "\u0003abc\u0003def": {key: abc, value: def}
+ "\u0004ab\0c\u0004de\0f": {key: "ab\0c", value: "de\0f"}
+
+---
+
+coder:
+ urn: "urn:beam:coders:interval_window:0.1"
+examples:
+ "\u0080\u0000\u0001\u0052\u009a\u00a4\u009b\u0068\u0080\u00dd\u00db\u0001" : {end: 1454293425000, span: 3600000}
+ "\u0080\u0000\u0001\u0053\u0034\u00ec\u0074\u00e8\u0080\u0090\u00fb\u00d3\u0009" : {end: 1456881825000, span: 2592000000}
+ "\u007f\u00df\u003b\u0064\u005a\u001c\u00ad\u0076\u00ed\u0002" : {end: -9223372036854410, span: 365}
+ "\u0080\u0020\u00c4\u009b\u00a5\u00e3\u0053\u00f7\u0000" : {end: 9223372036854775, span: 0}
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:varint:0.1"}]
+examples:
+ "\0\0\0\u0001\0": [0]
+ "\0\0\0\u0004\u0001\n\u00c8\u0001\u00e8\u0007": [1, 10, 200, 1000]
+ "\0\0\0\0": []
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"}]
+examples:
+ "\0\0\0\u0001\u0003abc": ["abc"]
+ "\0\0\0\u0002\u0004ab\0c\u0004de\0f": ["ab\0c", "de\0f"]
+ "\0\0\0\0": []
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:bytes:0.1"}]
+ # This is for iterables of unknown length, where the encoding is not
+ # deterministic.
+ non_deterministic: True
+examples:
+ "\u00ff\u00ff\u00ff\u00ff\u0000": []
+ "\u00ff\u00ff\u00ff\u00ff\u0001\u0003abc\u0000": ["abc"]
+ "\u00ff\u00ff\u00ff\u00ff\u0002\u0004ab\u0000c\u0004de\u0000f\u0000": ["ab\0c", "de\0f"]
+
+---
+
+coder:
+ urn: "urn:beam:coders:stream:0.1"
+ components: [{urn: "urn:beam:coders:global_window:0.1"}]
+examples:
+ "\0\0\0\u0001": [""]
+
+---
+
+coder:
+ urn: "urn:beam:coders:global_window:0.1"
+examples:
+ "": ""
+
+---
+
+# All windowed values consist of pane infos that represent NO_FIRING until full support is added
+# in the Python SDK (BEAM-1522).
+coder:
+ urn: "urn:beam:coders:windowed_value:0.1"
+ components: [{urn: "urn:beam:coders:varint:0.1"},
+ {urn: "urn:beam:coders:global_window:0.1"}]
+examples:
+ "\u0080\0\u0001R\u009a\u00a4\u009bh\0\0\0\u0001\u000f\u0002": {
+ value: 2,
+ timestamp: 1454293425000,
+ pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
+ windows: ["global"]
+ }
+
+---
+
+coder:
+ urn: "urn:beam:coders:windowed_value:0.1"
+ components: [{urn: "urn:beam:coders:varint:0.1"},
+ {urn: "urn:beam:coders:interval_window:0.1"}]
+examples:
+ "\u007f\u00ff\u00ff\u00ff\u00ff\u00f9\u00e5\u0080\0\0\0\u0001\u0080\0\u0001R\u009a\u00a4\u009bh\u00c0\u008b\u0011\u000f\u0004": {
+ value: 4,
+ timestamp: -400000,
+ pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
+ windows: [{end: 1454293425000, span: 280000}]
+ }
+
+ "\u007f\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u009c\0\0\0\u0002\u0080\0\u0001R\u009a\u00a4\u009bh\u0080\u00dd\u00db\u0001\u007f\u00df;dZ\u001c\u00adv\u00ed\u0002\u000f\u0002": {
+ value: 2,
+ timestamp: -100,
+ pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
+ windows: [{end: 1454293425000, span: 3600000}, {end: -9223372036854410, span: 365}]
+ }
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java
index 0e96745..7546851 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClient.java
@@ -25,7 +25,7 @@ import io.grpc.stub.StreamObserver;
import java.io.Closeable;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java
index c8b9bda..fd28040 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolService.java
@@ -19,8 +19,8 @@ package org.apache.beam.runners.core.fn;
import io.grpc.stub.StreamObserver;
import java.util.concurrent.BlockingQueue;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnControlGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java
index c8b87cc..fdde79c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/FnDataService.java
@@ -19,7 +19,7 @@ package org.apache.beam.runners.core.fn;
import com.google.auto.value.AutoValue;
import com.google.common.util.concurrent.ListenableFuture;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
@@ -32,8 +32,8 @@ public interface FnDataService {
/**
* A logical endpoint is a pair of an instruction ID corresponding to the {@link
- * org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest} and the {@link
- * org.apache.beam.fn.v1.BeamFnApi.Target} within the processing graph. This enables the same
+ * BeamFnApi.ProcessBundleRequest} and the {@link
+ * BeamFnApi.Target} within the processing graph. This enables the same
* {@link FnDataService} to be re-used across multiple bundles.
*/
@AutoValue
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java
index 38ebaed..bfd1837 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessClient.java
@@ -24,7 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import java.io.IOException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
/**
* A high-level client for an SDK harness.
@@ -54,7 +54,7 @@ public class SdkHarnessClient {
/**
* An active bundle for a particular {@link
- * org.apache.beam.fn.v1.BeamFnApi.ProcessBundleDescriptor}.
+ * BeamFnApi.ProcessBundleDescriptor}.
*/
@AutoValue
public abstract static class ActiveBundle<InputT> {
@@ -96,7 +96,7 @@ public class SdkHarnessClient {
}
/**
- * Registers a {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleDescriptor} for future
+ * Registers a {@link BeamFnApi.ProcessBundleDescriptor} for future
* processing.
*
* <p>A client may block on the result future, but may also proceed without blocking.
@@ -128,10 +128,10 @@ public class SdkHarnessClient {
/**
* Start a new bundle for the given {@link
- * org.apache.beam.fn.v1.BeamFnApi.ProcessBundleDescriptor} identifier.
+ * BeamFnApi.ProcessBundleDescriptor} identifier.
*
* <p>The input channels for the returned {@link ActiveBundle} are derived from the
- * instructions in the {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleDescriptor}.
+ * instructions in the {@link BeamFnApi.ProcessBundleDescriptor}.
*/
public ActiveBundle newBundle(String processBundleDescriptorId) {
String bundleId = idGenerator.getId();
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java
index 82afa6f..ec4d344 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunner.java
@@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkState;
import java.util.concurrent.ExecutionException;
import javax.annotation.Nullable;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -46,11 +47,11 @@ public class SdkHarnessDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT,
/**
* Returns a new {@link SdkHarnessDoFnRunner} suitable for just a particular {@link
- * org.apache.beam.fn.v1.BeamFnApi.ProcessBundleDescriptor} (referenced by id here).
+ * ProcessBundleDescriptor} (referenced by id here).
*
* <p>The {@link FnDataReceiver} must be the correct data plane service referenced
* in the primitive instructions in the
- * {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleDescriptor}.
+ * {@link ProcessBundleDescriptor}.
*
* <p>Also outside of this class, the appropriate receivers must be registered with the
* output data plane channels of the descriptor.
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java
index bc9a137..da02d92 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientPoolServiceTest.java
@@ -27,7 +27,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import io.grpc.stub.StreamObserver;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java
index 55afb4b..279e974 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/FnApiControlClientTest.java
@@ -28,7 +28,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import io.grpc.stub.StreamObserver;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java
index cbd24a6..7783b2f 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessClientTest.java
@@ -23,7 +23,7 @@ import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.SettableFuture;
import java.util.concurrent.Future;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java
index 98f38e1..8f16004 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/fn/SdkHarnessDoFnRunnerTest.java
@@ -23,7 +23,7 @@ import static org.mockito.Mockito.when;
import com.google.common.util.concurrent.SettableFuture;
import java.io.IOException;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
index 1db7a2b..8adc08c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CommonCoderTest.java
@@ -71,7 +71,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class CommonCoderTest {
private static final String STANDARD_CODERS_YAML_PATH =
- "/org/apache/beam/fn/v1/standard_coders.yaml";
+ "/org/apache/beam/model/fnexecution/v1/standard_coders.yaml";
private static final Map<String, Class<?>> coders = ImmutableMap.<String, Class<?>>builder()
.put("urn:beam:coders:bytes:0.1", ByteCoder.class)
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
index e520780..ff3dfb2 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java
@@ -34,7 +34,7 @@ import org.apache.beam.fn.harness.data.BeamFnDataClient;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.CoderTranslation;
@@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory;
* Registers as a consumer for data over the Beam Fn API. Multiplexes any received data
* to all consumers in the specified output map.
*
- * <p>Can be re-used serially across {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest}s.
+ * <p>Can be re-used serially across {@link BeamFnApi.ProcessBundleRequest}s.
* For each request, call {@link #registerInputLocation()} to start and call
* {@link #blockTillReadFinishes()} to finish.
*/
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
index e08c422..bf1994e 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java
@@ -33,7 +33,7 @@ import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.CoderTranslation;
@@ -47,7 +47,7 @@ import org.apache.beam.sdk.values.KV;
* Registers as a consumer with the Beam Fn Data Api. Consumes elements and encodes them for
* transmission.
*
- * <p>Can be re-used serially across {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest}s.
+ * <p>Can be re-used serially across {@link BeamFnApi.ProcessBundleRequest}s.
* For each request, call {@link #registerForOutput()} to start and call {@link #close()} to finish.
*/
public class BeamFnDataWriteRunner<InputT> {
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
index 2c0da7f..cad8985 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
@@ -45,9 +45,9 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BagUserState;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
-import org.apache.beam.fn.v1.BeamFnApi.StateKey;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.construction.ParDoTranslation;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
index 8483f6c..d6c461f 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
@@ -31,7 +31,7 @@ import org.apache.beam.fn.harness.fn.ThrowingFunction;
import org.apache.beam.fn.harness.logging.BeamFnLoggingClient;
import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
import org.apache.beam.fn.harness.stream.StreamObserverFactory;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
index 2ab6a41..3c98e77 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java
@@ -32,8 +32,8 @@ import java.util.concurrent.LinkedBlockingDeque;
import java.util.function.BiFunction;
import java.util.function.Function;
import org.apache.beam.fn.harness.fn.ThrowingFunction;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnControlGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,14 +43,14 @@ import org.slf4j.LoggerFactory;
* an unbounded number of requests.
*
* <p>Also can delegate to a set of handlers based upon the
- * {@link org.apache.beam.fn.v1.BeamFnApi.InstructionRequest.RequestCase request type}.
+ * {@link BeamFnApi.InstructionRequest.RequestCase request type}.
*
* <p>When the inbound instruction stream finishes successfully, the {@code onFinish} is
* completed successfully signaling to the caller that this client will not produce any more
- * {@link org.apache.beam.fn.v1.BeamFnApi.InstructionRequest}s. If the inbound instruction stream
+ * {@link BeamFnApi.InstructionRequest}s. If the inbound instruction stream
* errors, the {@code onFinish} is completed exceptionally propagating the failure reason
* to the caller and signaling that this client will not produce any more
- * {@link org.apache.beam.fn.v1.BeamFnApi.InstructionRequest}s.
+ * {@link BeamFnApi.InstructionRequest}s.
*/
public class BeamFnControlClient {
private static final String FAKE_INSTRUCTION_ID = "FAKE_INSTRUCTION_ID";
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
index 788efec..598583c 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
@@ -43,11 +43,11 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -57,7 +57,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Processes {@link org.apache.beam.fn.v1.BeamFnApi.ProcessBundleRequest}s by materializing
+ * Processes {@link BeamFnApi.ProcessBundleRequest}s by materializing
* the set of required runners for each {@link RunnerApi.FunctionSpec},
* wiring them together based upon the {@code input} and {@code output} map definitions.
*
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
index 704f782..503536a 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/RegisterHandler.java
@@ -24,8 +24,8 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.RegisterResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.RegisterResponse;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
/**
* A handler and datastore for types that be can be registered via the Fn API.
*
- * <p>Allows for {@link org.apache.beam.fn.v1.BeamFnApi.RegisterRequest}s to occur in parallel with
+ * <p>Allows for {@link BeamFnApi.RegisterRequest}s to occur in parallel with
* subsequent requests that may lookup registered values by blocking lookups until registration
* occurs.
*/
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
index 7223e87..eedac4a 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
@@ -24,7 +24,7 @@ import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
* A buffering outbound {@link Consumer} for the Beam Fn Data API.
*
* <p>Encodes individually consumed elements with the provided {@link Coder} producing
- * a single {@link org.apache.beam.fn.v1.BeamFnApi.Elements} message when the buffer threshold
+ * a single {@link BeamFnApi.Elements} message when the buffer threshold
* is surpassed.
*
* <p>The default buffer threshold can be overridden by specifying the experiment
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
index ca158ff..c3b7fd2 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataClient.java
@@ -21,7 +21,7 @@ package org.apache.beam.fn.harness.data;
import java.util.concurrent.CompletableFuture;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
index fe8af62..9333410 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClient.java
@@ -27,8 +27,8 @@ import java.util.function.BiFunction;
import java.util.function.Function;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnDataGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnDataGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
index da64112..cfe726a 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexer.java
@@ -26,7 +26,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import java.util.function.Function;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.values.KV;
import org.slf4j.Logger;
@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
* Endpoints.ApiServiceDescriptor}.
*
* <p>Multiplexes data for inbound consumers based upon their individual {@link
- * org.apache.beam.fn.v1.BeamFnApi.Target}s.
+ * org.apache.beam.model.fnexecution.v1.BeamFnApi.Target}s.
*
* <p>Multiplexing inbound and outbound streams is as thread safe as the consumers of those streams.
* For inbound streams, this is as thread safe as the inbound observers. For outbound streams, this
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
index ac603bd..64a12e0 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
@@ -21,14 +21,14 @@ import java.io.InputStream;
import java.util.concurrent.CompletableFuture;
import java.util.function.Consumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Decodes individually consumed {@link org.apache.beam.fn.v1.BeamFnApi.Elements.Data} with the
+ * Decodes individually consumed {@link BeamFnApi.Elements.Data} with the
* provided {@link Coder} passing the individual decoded elements to the provided consumer.
*/
public class BeamFnDataInboundObserver<T> implements Consumer<BeamFnApi.Elements.Data> {
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
index 82cdff1..240e954 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java
@@ -46,8 +46,8 @@ import java.util.logging.LogManager;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java
index 2d7f0c8..7064db4 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java
@@ -27,9 +27,9 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.function.Supplier;
import org.apache.beam.fn.harness.stream.DataStreams;
-import org.apache.beam.fn.v1.BeamFnApi.StateAppendRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateClearRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateAppendRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateClearRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
import org.apache.beam.sdk.coders.Coder;
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java
index 682adb9..c2dfd63 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateClient.java
@@ -18,8 +18,8 @@
package org.apache.beam.fn.harness.state;
import java.util.concurrent.CompletableFuture;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
/**
* The {@link BeamFnStateClient} is able to forward state requests to a handler which returns
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
index 07e7f95..2ca0704 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
@@ -28,9 +28,9 @@ import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
import org.apache.beam.fn.harness.data.BeamFnDataGrpcClient;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
-import org.apache.beam.fn.v1.BeamFnStateGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnStateGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
index 53dbd8f..b64c946 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
@@ -1,17 +1,3 @@
-package org.apache.beam.fn.harness.state;
-
-import com.google.common.base.Throwables;
-import com.google.protobuf.ByteString;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.function.Supplier;
-import org.apache.beam.fn.v1.BeamFnApi.StateGetRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -29,6 +15,19 @@ import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.beam.fn.harness.state;
+
+import com.google.common.base.Throwables;
+import com.google.protobuf.ByteString;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Supplier;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateGetRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
/**
* Adapters which convert a a logical series of chunks using continuation tokens over the Beam
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
index e0ef252..de68d41 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
@@ -51,7 +51,7 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.test.TestExecutors;
import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.model.pipeline.v1.RunnerApi.MessageWithComponents;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
index dca7eb1..486f114 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
@@ -48,7 +48,7 @@ import org.apache.beam.fn.harness.data.BeamFnDataClient;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.model.pipeline.v1.RunnerApi.MessageWithComponents;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
index cd2616b..e4422a3 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
@@ -42,7 +42,7 @@ import org.apache.beam.fn.harness.PTransformRunnerFactory.Registrar;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.FakeBeamFnStateClient;
-import org.apache.beam.fn.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.ParDoTranslation;
import org.apache.beam.sdk.coders.KvCoder;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
index 72bced9..fc89acf 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
@@ -30,12 +30,12 @@ import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.function.Consumer;
import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.InstructionRequest;
-import org.apache.beam.fn.v1.BeamFnApi.InstructionResponse;
-import org.apache.beam.fn.v1.BeamFnApi.LogControl;
-import org.apache.beam.fn.v1.BeamFnControlGrpc;
-import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.LogControl;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
index 8e3b152..6f27e21 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assume.assumeTrue;
import io.grpc.ManagedChannel;
-import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.Rule;
import org.junit.Test;
@@ -37,9 +37,8 @@ public class ManagedChannelFactoryTest {
@Test
public void testDefaultChannel() {
- ApiServiceDescriptor apiServiceDescriptor = ApiServiceDescriptor.newBuilder()
- .setUrl("localhost:123")
- .build();
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder().setUrl("localhost:123").build();
ManagedChannel channel = ManagedChannelFactory.from(PipelineOptionsFactory.create())
.forDescriptor(apiServiceDescriptor);
assertEquals("localhost:123", channel.authority());
@@ -49,9 +48,8 @@ public class ManagedChannelFactoryTest {
@Test
public void testEpollHostPortChannel() {
assumeTrue(io.netty.channel.epoll.Epoll.isAvailable());
- ApiServiceDescriptor apiServiceDescriptor = ApiServiceDescriptor.newBuilder()
- .setUrl("localhost:123")
- .build();
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder().setUrl("localhost:123").build();
ManagedChannel channel = ManagedChannelFactory.from(
PipelineOptionsFactory.fromArgs(new String[]{ "--experiments=beam_fn_api_epoll" }).create())
.forDescriptor(apiServiceDescriptor);
@@ -62,9 +60,10 @@ public class ManagedChannelFactoryTest {
@Test
public void testEpollDomainSocketChannel() throws Exception {
assumeTrue(io.netty.channel.epoll.Epoll.isAvailable());
- ApiServiceDescriptor apiServiceDescriptor = ApiServiceDescriptor.newBuilder()
- .setUrl("unix://" + tmpFolder.newFile().getAbsolutePath())
- .build();
+ Endpoints.ApiServiceDescriptor apiServiceDescriptor =
+ Endpoints.ApiServiceDescriptor.newBuilder()
+ .setUrl("unix://" + tmpFolder.newFile().getAbsolutePath())
+ .build();
ManagedChannel channel = ManagedChannelFactory.from(
PipelineOptionsFactory.fromArgs(new String[]{ "--experiments=beam_fn_api_epoll" }).create())
.forDescriptor(apiServiceDescriptor);
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
index f11eb52..8dc62b3 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
@@ -40,8 +40,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import org.apache.beam.fn.harness.fn.ThrowingFunction;
import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnControlGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnControlGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.junit.Test;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 536fdc3..15b5866 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -44,9 +44,9 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.BeamFnStateClient;
import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
index 9710a68..40b2145 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
@@ -25,8 +25,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.beam.fn.harness.test.TestExecutors;
import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnApi.RegisterResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.RegisterResponse;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
index c2b4542..4898b90 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
@@ -30,7 +30,7 @@ import java.util.Collection;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.LengthPrefixCoder;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
index 36e814b..5e2545d 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
@@ -46,8 +46,8 @@ import java.util.function.Function;
import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnDataGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnDataGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.LengthPrefixCoder;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
index 5468dcd..c63dd62 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
@@ -31,7 +31,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.values.KV;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
index 54aba8b..c939423 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
@@ -32,7 +32,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
-import org.apache.beam.fn.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
index 9901dea..161ce18 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
@@ -42,8 +42,8 @@ import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.LogRecord;
import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi;
+import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc;
import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java
index f3c76ac..6d3e078 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java
@@ -25,8 +25,8 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.protobuf.ByteString;
import java.io.IOException;
-import org.apache.beam.fn.v1.BeamFnApi.StateKey;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
index 901a9e1..e8c616d 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
@@ -41,10 +41,10 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Function;
import org.apache.beam.fn.harness.IdGenerator;
import org.apache.beam.fn.harness.test.TestStreams;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
-import org.apache.beam.fn.v1.BeamFnStateGrpc;
-import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnStateGrpc;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.After;
import org.junit.Before;
@@ -60,7 +60,7 @@ public class BeamFnStateGrpcClientCacheTest {
private static final String TEST_ERROR = "TEST ERROR";
private static final String SERVER_ERROR = "SERVER ERROR";
- private ApiServiceDescriptor apiServiceDescriptor;
+ private Endpoints.ApiServiceDescriptor apiServiceDescriptor;
private ManagedChannel testChannel;
private Server testServer;
private BeamFnStateGrpcClientCache clientCache;
@@ -75,7 +75,7 @@ public class BeamFnStateGrpcClientCacheTest {
TestStreams.withOnNext(values::add).build();
apiServiceDescriptor =
- ApiServiceDescriptor.newBuilder()
+ Endpoints.ApiServiceDescriptor.newBuilder()
.setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString())
.build();
testServer = InProcessServerBuilder.forName(apiServiceDescriptor.getUrl())
@@ -95,7 +95,7 @@ public class BeamFnStateGrpcClientCacheTest {
clientCache = new BeamFnStateGrpcClientCache(
PipelineOptionsFactory.create(),
IdGenerator::generate,
- (ApiServiceDescriptor descriptor) -> testChannel,
+ (Endpoints.ApiServiceDescriptor descriptor) -> testChannel,
this::createStreamForTest);
}
@@ -111,7 +111,7 @@ public class BeamFnStateGrpcClientCacheTest {
clientCache.forApiServiceDescriptor(apiServiceDescriptor));
assertNotSame(clientCache.forApiServiceDescriptor(apiServiceDescriptor),
clientCache.forApiServiceDescriptor(
- ApiServiceDescriptor.getDefaultInstance()));
+ Endpoints.ApiServiceDescriptor.getDefaultInstance()));
}
@Test
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
index 60080e1..e991db6 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
@@ -25,14 +25,14 @@ import java.util.Collections;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.beam.fn.v1.BeamFnApi.StateAppendResponse;
-import org.apache.beam.fn.v1.BeamFnApi.StateClearResponse;
-import org.apache.beam.fn.v1.BeamFnApi.StateGetResponse;
-import org.apache.beam.fn.v1.BeamFnApi.StateKey;
-import org.apache.beam.fn.v1.BeamFnApi.StateKey.TypeCase;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest.RequestCase;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateAppendResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateClearResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateGetResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey.TypeCase;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.RequestCase;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
/** A fake implementation of a {@link BeamFnStateClient} to aid with testing. */
public class FakeBeamFnStateClient implements BeamFnStateClient {
http://git-wip-us.apache.org/repos/asf/beam/blob/80c6f4ec/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java
index 67e36e1..0c2f922 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java
@@ -24,9 +24,9 @@ import com.google.protobuf.ByteString;
import java.util.Iterator;
import java.util.concurrent.CompletableFuture;
import org.apache.beam.fn.harness.state.StateFetchingIterators.LazyBlockingStateFetchingIterator;
-import org.apache.beam.fn.v1.BeamFnApi.StateGetResponse;
-import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateGetResponse;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
+import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
import org.junit.Test;
import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
[07/12] beam git commit: Re-namespace model/pipeline to model.pipeline
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
index 51a047a..07e7f95 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCache.java
@@ -31,8 +31,8 @@ import org.apache.beam.fn.harness.data.BeamFnDataGrpcClient;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
import org.apache.beam.fn.v1.BeamFnStateGrpc;
-import org.apache.beam.portability.v1.Endpoints;
-import org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
import org.apache.beam.sdk.options.PipelineOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
index 0526183..53dbd8f 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java
@@ -10,6 +10,8 @@ import java.util.function.Supplier;
import org.apache.beam.fn.v1.BeamFnApi.StateGetRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
+import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -27,7 +29,6 @@ import org.apache.beam.fn.v1.BeamFnApi.StateRequest.Builder;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
/**
* Adapters which convert a a logical series of chunks using continuation tokens over the Beam
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
index 9b76fe1..e0ef252 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java
@@ -52,12 +52,12 @@ import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.test.TestExecutors;
import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.MessageWithComponents;
import org.apache.beam.runners.core.construction.CoderTranslation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.MessageWithComponents;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
index 8e9ebb8..dca7eb1 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java
@@ -49,12 +49,12 @@ import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.MessageWithComponents;
import org.apache.beam.runners.core.construction.CoderTranslation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.MessageWithComponents;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java
index 135495a..50009c0 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java
@@ -39,7 +39,7 @@ import java.util.ServiceLoader;
import org.apache.beam.fn.harness.PTransformRunnerFactory.Registrar;
import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.CountingSource;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
index 9113be7..cd2616b 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java
@@ -43,10 +43,10 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.fn.ThrowingRunnable;
import org.apache.beam.fn.harness.state.FakeBeamFnStateClient;
import org.apache.beam.fn.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.ParDoTranslation;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.CombiningState;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
index cdc4b01..72bced9 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java
@@ -36,7 +36,7 @@ import org.apache.beam.fn.v1.BeamFnApi.InstructionResponse;
import org.apache.beam.fn.v1.BeamFnApi.LogControl;
import org.apache.beam.fn.v1.BeamFnControlGrpc;
import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
index 62bb1ba..8e3b152 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/channel/ManagedChannelFactoryTest.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assume.assumeTrue;
import io.grpc.ManagedChannel;
-import org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
index fedc7d4..f11eb52 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java
@@ -42,7 +42,7 @@ import org.apache.beam.fn.harness.fn.ThrowingFunction;
import org.apache.beam.fn.harness.test.TestStreams;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnControlGrpc;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 026348c..536fdc3 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -47,8 +47,8 @@ import org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
-import org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
index 2b275af..9710a68 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/RegisterHandlerTest.java
@@ -27,7 +27,7 @@ import org.apache.beam.fn.harness.test.TestExecutors;
import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnApi.RegisterResponse;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
index 2f3bc2c..36e814b 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcClientTest.java
@@ -48,7 +48,7 @@ import org.apache.beam.fn.harness.fn.ThrowingConsumer;
import org.apache.beam.fn.harness.test.TestStreams;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnDataGrpc;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.LengthPrefixCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
index fdef03d..5468dcd 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataGrpcMultiplexerTest.java
@@ -32,7 +32,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.beam.fn.harness.test.TestStreams;
import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.values.KV;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
index c50695c..9901dea 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java
@@ -44,7 +44,7 @@ import java.util.logging.LogRecord;
import org.apache.beam.fn.harness.test.TestStreams;
import org.apache.beam.fn.v1.BeamFnApi;
import org.apache.beam.fn.v1.BeamFnLoggingGrpc;
-import org.apache.beam.portability.v1.Endpoints;
+import org.apache.beam.model.pipeline.v1.Endpoints;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.Test;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
index a51e7b4..901a9e1 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java
@@ -44,7 +44,7 @@ import org.apache.beam.fn.harness.test.TestStreams;
import org.apache.beam.fn.v1.BeamFnApi.StateRequest;
import org.apache.beam.fn.v1.BeamFnApi.StateResponse;
import org.apache.beam.fn.v1.BeamFnStateGrpc;
-import org.apache.beam.portability.v1.Endpoints.ApiServiceDescriptor;
+import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.After;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/beam/blob/7c563ef8/sdks/python/gen_protos.py
----------------------------------------------------------------------
diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
index 4ca3561..59d6cb7 100644
--- a/sdks/python/gen_protos.py
+++ b/sdks/python/gen_protos.py
@@ -35,7 +35,6 @@ BEAM_PROTO_PATHS = [
os.path.join('..', '..', 'model', 'pipeline', 'src', 'main', 'proto'),
os.path.join('..', '..', 'model', 'job-management', 'src', 'main', 'proto'),
os.path.join('..', '..', 'model', 'fn-execution', 'src', 'main', 'proto'),
- os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto')
]
PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'portability', 'api')