You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/10/11 19:26:08 UTC
[12/12] beam git commit: This closes #3924: [BEAM-2576] Split Beam
portability framework into construction, job, and execution modules
This closes #3924: [BEAM-2576] Split Beam portability framework into construction, job, and execution modules
Dataflow runner: shade protobuf and beam model
Improve error message when standard_coders.yaml not found
Re-namespace model/fn-execution to model.fn_execution
Re-namespace model/job-management to model.job_management
Re-namespace model/pipeline to model.pipeline
Remove sdks/common
Move sdks/common/fn-api protos to model/fn-execution
Move sdk/common/runner-api protos into model/pipeline and model/job-management
Add model/ and submodules to Maven structure and gen_protos.py
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/92615ca4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/92615ca4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/92615ca4
Branch: refs/heads/master
Commit: 92615ca4c4e09bc63c37a933102035040c23aac9
Parents: 4ccb3ba b2daa0c
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Oct 11 12:24:52 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Oct 11 12:24:52 2017 -0700
----------------------------------------------------------------------
model/fn-execution/pom.xml | 114 +++
.../src/main/proto/beam_fn_api.proto | 726 ++++++++++++++++
.../src/main/proto/beam_provision_api.proto | 60 ++
.../model/fnexecution/v1/standard_coders.yaml | 195 +++++
model/job-management/pom.xml | 114 +++
.../src/main/proto/beam_artifact_api.proto | 133 +++
.../src/main/proto/beam_job_api.proto | 173 ++++
model/pipeline/pom.xml | 89 ++
.../src/main/proto/beam_runner_api.proto | 842 +++++++++++++++++++
model/pipeline/src/main/proto/endpoints.proto | 46 +
.../src/main/proto/standard_window_fns.proto | 53 ++
model/pom.xml | 40 +
pom.xml | 15 +-
runners/apex/pom.xml | 4 +-
runners/core-construction-java/pom.xml | 7 +-
.../construction/ArtifactServiceStager.java | 18 +-
.../core/construction/CoderTranslation.java | 6 +-
.../core/construction/CombineTranslation.java | 12 +-
.../CreatePCollectionViewTranslation.java | 4 +-
.../construction/DisplayDataTranslation.java | 2 +-
.../core/construction/FlattenTranslator.java | 4 +-
.../construction/GroupByKeyTranslation.java | 4 +-
.../construction/PCollectionTranslation.java | 2 +-
.../construction/PTransformTranslation.java | 4 +-
.../core/construction/ParDoTranslation.java | 16 +-
.../core/construction/PipelineTranslation.java | 2 +-
.../core/construction/ReadTranslation.java | 10 +-
.../core/construction/RehydratedComponents.java | 4 +-
.../construction/RunnerPCollectionView.java | 2 +-
.../core/construction/SdkComponents.java | 4 +-
.../construction/TestStreamTranslation.java | 2 +-
.../core/construction/TriggerTranslation.java | 2 +-
.../construction/WindowIntoTranslation.java | 6 +-
.../WindowingStrategyTranslation.java | 18 +-
.../construction/WriteFilesTranslation.java | 10 +-
.../construction/ArtifactServiceStagerTest.java | 2 +-
.../core/construction/CoderTranslationTest.java | 4 +-
.../construction/CombineTranslationTest.java | 4 +-
.../CreatePCollectionViewTranslationTest.java | 4 +-
.../InMemoryArtifactStagerService.java | 16 +-
.../PCollectionTranslationTest.java | 2 +-
.../construction/PTransformTranslationTest.java | 6 +-
.../core/construction/ParDoTranslationTest.java | 6 +-
.../construction/PipelineTranslationTest.java | 2 +-
.../core/construction/ReadTranslationTest.java | 4 +-
.../core/construction/SdkComponentsTest.java | 2 +-
.../construction/TestStreamTranslationTest.java | 6 +-
.../construction/WindowIntoTranslationTest.java | 2 +-
.../WindowingStrategyTranslationTest.java | 2 +-
.../construction/WriteFilesTranslationTest.java | 4 +-
runners/core-java/pom.xml | 6 +-
.../runners/core/fn/FnApiControlClient.java | 2 +-
.../core/fn/FnApiControlClientPoolService.java | 4 +-
.../beam/runners/core/fn/FnDataService.java | 6 +-
.../beam/runners/core/fn/SdkHarnessClient.java | 10 +-
.../runners/core/fn/SdkHarnessDoFnRunner.java | 5 +-
.../core/triggers/TriggerStateMachines.java | 2 +-
.../fn/FnApiControlClientPoolServiceTest.java | 2 +-
.../runners/core/fn/FnApiControlClientTest.java | 2 +-
.../runners/core/fn/SdkHarnessClientTest.java | 2 +-
.../core/fn/SdkHarnessDoFnRunnerTest.java | 2 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
runners/direct-java/pom.xml | 8 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 2 +-
.../direct/TransformEvaluatorRegistry.java | 2 +-
runners/flink/pom.xml | 4 +-
.../FlinkStreamingTransformTranslators.java | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 18 +-
.../beam/runners/dataflow/DataflowRunner.java | 2 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 2 +-
.../options/DataflowPipelineOptions.java | 1 -
runners/local-artifact-service-java/pom.xml | 2 +-
.../LocalFileSystemArtifactStagerService.java | 57 +-
...ocalFileSystemArtifactStagerServiceTest.java | 143 ++--
runners/reference/job-server/pom.xml | 2 +-
.../job/ReferenceRunnerJobService.java | 16 +-
runners/reference/pom.xml | 1 -
runners/spark/pom.xml | 4 +-
sdks/common/fn-api/pom.xml | 114 ---
.../fn-api/src/main/proto/beam_fn_api.proto | 726 ----------------
.../src/main/proto/beam_provision_api.proto | 60 --
.../org/apache/beam/fn/v1/standard_coders.yaml | 195 -----
sdks/common/pom.xml | 39 -
sdks/common/runner-api/pom.xml | 109 ---
.../src/main/proto/beam_artifact_api.proto | 133 ---
.../src/main/proto/beam_job_api.proto | 173 ----
.../src/main/proto/beam_runner_api.proto | 842 -------------------
.../runner-api/src/main/proto/endpoints.proto | 46 -
.../src/main/proto/standard_window_fns.proto | 53 --
sdks/java/core/pom.xml | 8 +-
.../apache/beam/sdk/coders/CommonCoderTest.java | 4 +-
sdks/java/harness/pom.xml | 24 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 8 +-
.../beam/fn/harness/BeamFnDataWriteRunner.java | 8 +-
.../beam/fn/harness/BoundedSourceRunner.java | 2 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 8 +-
.../org/apache/beam/fn/harness/FnHarness.java | 8 +-
.../fn/harness/PTransformRunnerFactory.java | 2 +-
.../harness/channel/ManagedChannelFactory.java | 2 +-
.../fn/harness/control/BeamFnControlClient.java | 12 +-
.../harness/control/ProcessBundleHandler.java | 16 +-
.../fn/harness/control/RegisterHandler.java | 8 +-
.../BeamFnDataBufferingOutboundObserver.java | 4 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 4 +-
.../fn/harness/data/BeamFnDataGrpcClient.java | 6 +-
.../harness/data/BeamFnDataGrpcMultiplexer.java | 8 +-
.../harness/data/BeamFnDataInboundObserver.java | 4 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 6 +-
.../beam/fn/harness/state/BagUserState.java | 6 +-
.../fn/harness/state/BeamFnStateClient.java | 4 +-
.../state/BeamFnStateGrpcClientCache.java | 10 +-
.../harness/state/StateFetchingIterators.java | 26 +-
.../fn/harness/BeamFnDataReadRunnerTest.java | 8 +-
.../fn/harness/BeamFnDataWriteRunnerTest.java | 8 +-
.../fn/harness/BoundedSourceRunnerTest.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 4 +-
.../apache/beam/fn/harness/FnHarnessTest.java | 14 +-
.../channel/ManagedChannelFactoryTest.java | 19 +-
.../control/BeamFnControlClientTest.java | 6 +-
.../control/ProcessBundleHandlerTest.java | 10 +-
.../fn/harness/control/RegisterHandlerTest.java | 6 +-
...BeamFnDataBufferingOutboundObserverTest.java | 2 +-
.../harness/data/BeamFnDataGrpcClientTest.java | 6 +-
.../data/BeamFnDataGrpcMultiplexerTest.java | 4 +-
.../data/BeamFnDataInboundObserverTest.java | 2 +-
.../logging/BeamFnLoggingClientTest.java | 6 +-
.../beam/fn/harness/state/BagUserStateTest.java | 4 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 16 +-
.../fn/harness/state/FakeBeamFnStateClient.java | 16 +-
.../state/StateFetchingIteratorsTest.java | 6 +-
sdks/pom.xml | 1 -
sdks/python/gen_protos.py | 5 +-
132 files changed, 3046 insertions(+), 2897 deletions(-)
----------------------------------------------------------------------