You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2025/01/16 23:15:31 UTC
(beam) branch revert-33434-hybridProtos created (now 2bd5d7b3bd4)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a change to branch revert-33434-hybridProtos
in repository https://gitbox.apache.org/repos/asf/beam.git
at 2bd5d7b3bd4 Revert "Migrate Beam Go to use the new Opaque Protocol Buffers (#33434)"
This branch includes the following new commits:
new 2bd5d7b3bd4 Revert "Migrate Beam Go to use the new Opaque Protocol Buffers (#33434)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(beam) 01/01: Revert "Migrate Beam Go to use the new Opaque Protocol Buffers (#33434)"
Posted by lo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch revert-33434-hybridProtos
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 2bd5d7b3bd4c5872312874a6095297437d3cead9
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Thu Jan 16 15:15:20 2025 -0800
Revert "Migrate Beam Go to use the new Opaque Protocol Buffers (#33434)"
This reverts commit 4797d75fb9560b532c8b8cc65a17d70823293949.
---
sdks/go/cmd/beamctl/cmd/artifact.go | 2 +-
sdks/go/container/boot.go | 2 +-
sdks/go/container/boot_test.go | 38 +-
sdks/go/container/pool/workerpool.go | 22 +-
sdks/go/container/pool/workerpool_test.go | 60 +-
sdks/go/container/tools/buffered_logging_test.go | 24 +-
sdks/go/container/tools/logging.go | 8 +-
sdks/go/container/tools/logging_test.go | 8 +-
sdks/go/container/tools/provision_test.go | 4 +-
sdks/go/examples/snippets/06schemas_test.go | 62 +-
sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go | 18 +-
sdks/go/pkg/beam/artifact/gcsproxy/staging.go | 28 +-
sdks/go/pkg/beam/artifact/materialize.go | 124 +-
sdks/go/pkg/beam/artifact/materialize_test.go | 94 +-
sdks/go/pkg/beam/artifact/server_test.go | 14 +-
sdks/go/pkg/beam/artifact/stage.go | 37 +-
sdks/go/pkg/beam/artifact/stage_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/translate.go | 20 +-
.../pkg/beam/core/runtime/exec/translate_test.go | 77 +-
sdks/go/pkg/beam/core/runtime/graphx/coder.go | 24 +-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 164 +-
.../beam/core/runtime/graphx/schema/schema_test.go | 850 +-
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 534 +-
.../pkg/beam/core/runtime/graphx/translate_test.go | 102 +-
sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 24 +-
sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go | 66 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 32 +-
.../pkg/beam/core/runtime/harness/datamgr_test.go | 100 +-
sdks/go/pkg/beam/core/runtime/harness/harness.go | 175 +-
.../pkg/beam/core/runtime/harness/harness_test.go | 56 +-
sdks/go/pkg/beam/core/runtime/harness/logging.go | 18 +-
.../pkg/beam/core/runtime/harness/logging_test.go | 2 +-
.../go/pkg/beam/core/runtime/harness/monitoring.go | 32 +-
.../runtime/harness/statecache/statecache_test.go | 14 +-
sdks/go/pkg/beam/core/runtime/harness/statemgr.go | 204 +-
.../pkg/beam/core/runtime/harness/statemgr_test.go | 104 +-
.../pkg/beam/core/runtime/harness/worker_status.go | 2 +-
.../core/runtime/harness/worker_status_test.go | 2 +-
.../beam/core/runtime/metricsx/metricsx_test.go | 36 +-
sdks/go/pkg/beam/core/runtime/pipelinex/clone.go | 24 +-
.../pkg/beam/core/runtime/pipelinex/clone_test.go | 10 +-
sdks/go/pkg/beam/core/runtime/pipelinex/replace.go | 46 +-
.../beam/core/runtime/pipelinex/replace_test.go | 292 +-
sdks/go/pkg/beam/core/runtime/pipelinex/util.go | 10 +-
.../pkg/beam/core/runtime/pipelinex/util_test.go | 18 +-
sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/namespace.go | 60 +-
.../pkg/beam/core/runtime/xlangx/namespace_test.go | 460 +-
sdks/go/pkg/beam/core/runtime/xlangx/payload.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/registry.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/resolve.go | 30 +-
.../pkg/beam/core/runtime/xlangx/resolve_test.go | 20 +-
sdks/go/pkg/beam/io/pubsubio/pubsubio.go | 16 +-
.../beam/model/fnexecution_v1/beam_fn_api.pb.go | 5960 +++------
.../model/fnexecution_v1/beam_fn_api_grpc.pb.go | 497 +-
.../fnexecution_v1/beam_fn_api_protoopaque.pb.go | 8793 -------------
.../model/fnexecution_v1/beam_provision_api.pb.go | 305 +-
.../fnexecution_v1/beam_provision_api_grpc.pb.go | 47 +-
.../beam_provision_api_protoopaque.pb.go | 575 -
.../model/jobmanagement_v1/beam_artifact_api.pb.go | 1509 +--
.../jobmanagement_v1/beam_artifact_api_grpc.pb.go | 366 +-
.../beam_artifact_api_protoopaque.pb.go | 2109 ---
.../jobmanagement_v1/beam_expansion_api.pb.go | 451 +-
.../jobmanagement_v1/beam_expansion_api_grpc.pb.go | 49 +-
.../beam_expansion_api_protoopaque.pb.go | 780 --
.../beam/model/jobmanagement_v1/beam_job_api.pb.go | 1670 ++-
.../model/jobmanagement_v1/beam_job_api_grpc.pb.go | 187 +-
.../beam_job_api_protoopaque.pb.go | 2611 ----
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 8416 +++++-------
.../model/pipeline_v1/beam_runner_api_grpc.pb.go | 81 +-
.../pipeline_v1/beam_runner_api_protoopaque.pb.go | 12779 -------------------
sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go | 163 +-
.../model/pipeline_v1/endpoints_protoopaque.pb.go | 280 -
.../model/pipeline_v1/external_transforms.pb.go | 780 +-
.../external_transforms_protoopaque.pb.go | 974 --
sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go | 1385 +-
.../model/pipeline_v1/metrics_protoopaque.pb.go | 1754 ---
sdks/go/pkg/beam/model/pipeline_v1/schema.pb.go | 2231 +---
.../model/pipeline_v1/schema_protoopaque.pb.go | 2998 -----
.../model/pipeline_v1/standard_window_fns.pb.go | 334 +-
.../standard_window_fns_protoopaque.pb.go | 685 -
sdks/go/pkg/beam/provision/provision_test.go | 4 +-
.../beam/runners/dataflow/dataflowlib/execute.go | 4 +-
.../beam/runners/dataflow/dataflowlib/job_test.go | 16 +-
.../runners/dataflow/dataflowlib/metrics_test.go | 12 +-
sdks/go/pkg/beam/runners/prism/internal/coders.go | 24 +-
.../pkg/beam/runners/prism/internal/coders_test.go | 194 +-
.../beam/runners/prism/internal/environments.go | 14 +-
sdks/go/pkg/beam/runners/prism/internal/execute.go | 23 +-
.../beam/runners/prism/internal/handlecombine.go | 30 +-
.../runners/prism/internal/handlecombine_test.go | 136 +-
.../pkg/beam/runners/prism/internal/handlepardo.go | 42 +-
.../beam/runners/prism/internal/handlerunner.go | 10 +-
.../runners/prism/internal/jobservices/artifact.go | 34 +-
.../prism/internal/jobservices/management.go | 108 +-
.../prism/internal/jobservices/management_test.go | 130 +-
.../runners/prism/internal/jobservices/metrics.go | 30 +-
.../prism/internal/jobservices/metrics_test.go | 14 +-
.../prism/internal/jobservices/server_test.go | 38 +-
.../pkg/beam/runners/prism/internal/preprocess.go | 2 +-
.../beam/runners/prism/internal/preprocess_test.go | 284 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 92 +-
sdks/go/pkg/beam/runners/prism/internal/web/web.go | 12 +-
.../beam/runners/prism/internal/worker/bundle.go | 88 +-
.../beam/runners/prism/internal/worker/worker.go | 122 +-
.../runners/prism/internal/worker/worker_test.go | 108 +-
.../beam/runners/universal/extworker/extworker.go | 20 +-
.../runners/universal/extworker/extworker_test.go | 60 +-
.../beam/runners/universal/runnerlib/execute.go | 8 +-
.../go/pkg/beam/runners/universal/runnerlib/job.go | 12 +-
.../pkg/beam/runners/universal/runnerlib/stage.go | 44 +-
.../beam/runners/universal/runnerlib/stage_test.go | 38 +-
sdks/go/pkg/beam/testing/teststream/teststream.go | 19 +-
.../pkg/beam/testing/teststream/teststream_test.go | 4 +-
.../pkg/beam/transforms/xlang/schema/external.go | 4 +-
sdks/go/pkg/beam/x/hooks/perf/perf.go | 6 +-
sdks/go/scripts/genproto.sh | 22 +-
.../go/test/regression/coders/fromyaml/fromyaml.go | 8 +-
sdks/java/container/boot.go | 4 +-
sdks/java/container/boot_test.go | 6 +-
sdks/java/expansion-service/container/boot.go | 11 +-
.../transform-service/controller-container/boot.go | 7 +-
sdks/python/container/boot.go | 6 +-
sdks/typescript/container/boot.go | 6 +-
124 files changed, 12021 insertions(+), 52793 deletions(-)
diff --git a/sdks/go/cmd/beamctl/cmd/artifact.go b/sdks/go/cmd/beamctl/cmd/artifact.go
index b1ca18e9643..1b5b7c9df8f 100644
--- a/sdks/go/cmd/beamctl/cmd/artifact.go
+++ b/sdks/go/cmd/beamctl/cmd/artifact.go
@@ -95,7 +95,7 @@ func listFn(cmd *cobra.Command, args []string) error {
}
for _, a := range md.GetManifest().GetArtifact() {
- cmd.Println(a.GetName())
+ cmd.Println(a.Name)
}
return nil
}
diff --git a/sdks/go/container/boot.go b/sdks/go/container/boot.go
index f10b2d59abd..3f8562f6ca9 100644
--- a/sdks/go/container/boot.go
+++ b/sdks/go/container/boot.go
@@ -184,7 +184,7 @@ func main() {
enableGoogleCloudProfiler := strings.Contains(options, enableGoogleCloudProfilerOption)
if enableGoogleCloudProfiler {
- err := configureGoogleCloudProfilerEnvVars(ctx, logger, info.GetMetadata())
+ err := configureGoogleCloudProfilerEnvVars(ctx, logger, info.Metadata)
if err != nil {
logger.Printf(ctx, "could not configure Google Cloud Profiler variables, got %v", err)
}
diff --git a/sdks/go/container/boot_test.go b/sdks/go/container/boot_test.go
index 97c0e4172dd..49c78047249 100644
--- a/sdks/go/container/boot_test.go
+++ b/sdks/go/container/boot_test.go
@@ -29,11 +29,11 @@ import (
)
func TestEnsureEndpointsSet_AllSet(t *testing.T) {
- provisionInfo := fnpb.ProvisionInfo_builder{
- LoggingEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testLoggingEndpointUrl"}.Build(),
- ArtifactEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testArtifactEndpointUrl"}.Build(),
- ControlEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testControlEndpointUrl"}.Build(),
- }.Build()
+ provisionInfo := &fnpb.ProvisionInfo{
+ LoggingEndpoint: &pipepb.ApiServiceDescriptor{Url: "testLoggingEndpointUrl"},
+ ArtifactEndpoint: &pipepb.ApiServiceDescriptor{Url: "testArtifactEndpointUrl"},
+ ControlEndpoint: &pipepb.ApiServiceDescriptor{Url: "testControlEndpointUrl"},
+ }
*loggingEndpoint = ""
*artifactEndpoint = ""
*controlEndpoint = ""
@@ -53,11 +53,11 @@ func TestEnsureEndpointsSet_AllSet(t *testing.T) {
}
func TestEnsureEndpointsSet_OneMissing(t *testing.T) {
- provisionInfo := fnpb.ProvisionInfo_builder{
- LoggingEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testLoggingEndpointUrl"}.Build(),
- ArtifactEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testArtifactEndpointUrl"}.Build(),
- ControlEndpoint: pipepb.ApiServiceDescriptor_builder{Url: ""}.Build(),
- }.Build()
+ provisionInfo := &fnpb.ProvisionInfo{
+ LoggingEndpoint: &pipepb.ApiServiceDescriptor{Url: "testLoggingEndpointUrl"},
+ ArtifactEndpoint: &pipepb.ApiServiceDescriptor{Url: "testArtifactEndpointUrl"},
+ ControlEndpoint: &pipepb.ApiServiceDescriptor{Url: ""},
+ }
*loggingEndpoint = ""
*artifactEndpoint = ""
*controlEndpoint = ""
@@ -85,7 +85,7 @@ func TestGetGoWorkerArtifactName_NoArtifacts(t *testing.T) {
func TestGetGoWorkerArtifactName_OneArtifact(t *testing.T) {
artifact := constructArtifactInformation(t, artifact.URNGoWorkerBinaryRole, "test/path", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact}
+ artifacts := []*pipepb.ArtifactInformation{&artifact}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -99,7 +99,7 @@ func TestGetGoWorkerArtifactName_OneArtifact(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsFirstIsWorker(t *testing.T) {
artifact1 := constructArtifactInformation(t, artifact.URNGoWorkerBinaryRole, "test/path", "sha")
artifact2 := constructArtifactInformation(t, "other role", "test/path2", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -113,7 +113,7 @@ func TestGetGoWorkerArtifactName_MultipleArtifactsFirstIsWorker(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsSecondIsWorker(t *testing.T) {
artifact1 := constructArtifactInformation(t, "other role", "test/path", "sha")
artifact2 := constructArtifactInformation(t, artifact.URNGoWorkerBinaryRole, "test/path2", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -127,7 +127,7 @@ func TestGetGoWorkerArtifactName_MultipleArtifactsSecondIsWorker(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsLegacyWay(t *testing.T) {
artifact1 := constructArtifactInformation(t, "other role", "test/path", "sha")
artifact2 := constructArtifactInformation(t, "other role", "worker", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -141,7 +141,7 @@ func TestGetGoWorkerArtifactName_MultipleArtifactsLegacyWay(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsNoneMatch(t *testing.T) {
artifact1 := constructArtifactInformation(t, "other role", "test/path", "sha")
artifact2 := constructArtifactInformation(t, "other role", "test/path2", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
_, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err == nil {
@@ -193,16 +193,16 @@ func TestCopyExe(t *testing.T) {
}
}
-func constructArtifactInformation(t *testing.T, roleUrn string, path string, sha string) *pipepb.ArtifactInformation {
+func constructArtifactInformation(t *testing.T, roleUrn string, path string, sha string) pipepb.ArtifactInformation {
t.Helper()
- typePayload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{Path: path, Sha256: sha}.Build())
+ typePayload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{Path: path, Sha256: sha})
- return pipepb.ArtifactInformation_builder{
+ return pipepb.ArtifactInformation{
RoleUrn: roleUrn,
TypeUrn: artifact.URNFileArtifact,
TypePayload: typePayload,
- }.Build()
+ }
}
func TestConfigureGoogleCloudProfilerEnvVars(t *testing.T) {
diff --git a/sdks/go/container/pool/workerpool.go b/sdks/go/container/pool/workerpool.go
index 4c3a308fd0e..5f3642eac03 100644
--- a/sdks/go/container/pool/workerpool.go
+++ b/sdks/go/container/pool/workerpool.go
@@ -80,24 +80,24 @@ func (s *Process) StartWorker(_ context.Context, req *fnpb.StartWorkerRequest) (
s.mu.Lock()
defer s.mu.Unlock()
if s.workers == nil {
- return fnpb.StartWorkerResponse_builder{
+ return &fnpb.StartWorkerResponse{
Error: "worker pool shutting down",
- }.Build(), nil
+ }, nil
}
if _, ok := s.workers[req.GetWorkerId()]; ok {
- return fnpb.StartWorkerResponse_builder{
+ return &fnpb.StartWorkerResponse{
Error: fmt.Sprintf("worker with ID %q already exists", req.GetWorkerId()),
- }.Build(), nil
+ }, nil
}
if req.GetLoggingEndpoint() == nil {
- return fnpb.StartWorkerResponse_builder{Error: fmt.Sprintf("Missing logging endpoint for worker %v", req.GetWorkerId())}.Build(), nil
+ return &fnpb.StartWorkerResponse{Error: fmt.Sprintf("Missing logging endpoint for worker %v", req.GetWorkerId())}, nil
}
if req.GetControlEndpoint() == nil {
- return fnpb.StartWorkerResponse_builder{Error: fmt.Sprintf("Missing control endpoint for worker %v", req.GetWorkerId())}.Build(), nil
+ return &fnpb.StartWorkerResponse{Error: fmt.Sprintf("Missing control endpoint for worker %v", req.GetWorkerId())}, nil
}
- if req.GetLoggingEndpoint().HasAuthentication() || req.GetControlEndpoint().HasAuthentication() {
- return fnpb.StartWorkerResponse_builder{Error: "[BEAM-10610] Secure endpoints not supported."}.Build(), nil
+ if req.GetLoggingEndpoint().Authentication != nil || req.GetControlEndpoint().Authentication != nil {
+ return &fnpb.StartWorkerResponse{Error: "[BEAM-10610] Secure endpoints not supported."}, nil
}
ctx := grpcx.WriteWorkerID(s.root, req.GetWorkerId())
@@ -118,7 +118,7 @@ func (s *Process) StartWorker(_ context.Context, req *fnpb.StartWorkerRequest) (
cmd.Env = nil // Use the current environment.
if err := cmd.Start(); err != nil {
- return fnpb.StartWorkerResponse_builder{Error: fmt.Sprintf("Unable to start boot for worker %v: %v", req.GetWorkerId(), err)}.Build(), nil
+ return &fnpb.StartWorkerResponse{Error: fmt.Sprintf("Unable to start boot for worker %v: %v", req.GetWorkerId(), err)}, nil
}
return &fnpb.StartWorkerResponse{}, nil
}
@@ -137,9 +137,9 @@ func (s *Process) StopWorker(_ context.Context, req *fnpb.StopWorkerRequest) (*f
delete(s.workers, req.GetWorkerId())
return &fnpb.StopWorkerResponse{}, nil
}
- return fnpb.StopWorkerResponse_builder{
+ return &fnpb.StopWorkerResponse{
Error: fmt.Sprintf("no worker with id %q running", req.GetWorkerId()),
- }.Build(), nil
+ }, nil
}
diff --git a/sdks/go/container/pool/workerpool_test.go b/sdks/go/container/pool/workerpool_test.go
index ed001ec0149..add2e25681f 100644
--- a/sdks/go/container/pool/workerpool_test.go
+++ b/sdks/go/container/pool/workerpool_test.go
@@ -31,15 +31,15 @@ func TestProcess(t *testing.T) {
t.Skip("Binary `true` doesn't exist, skipping tests.")
}
- endpoint := pipepb.ApiServiceDescriptor_builder{
+ endpoint := &pipepb.ApiServiceDescriptor{
Url: "localhost:0",
- }.Build()
- secureEndpoint := pipepb.ApiServiceDescriptor_builder{
+ }
+ secureEndpoint := &pipepb.ApiServiceDescriptor{
Url: "localhost:0",
- Authentication: pipepb.AuthenticationSpec_builder{
+ Authentication: &pipepb.AuthenticationSpec{
Urn: "beam:authentication:oauth2_client_credentials_grant:v1",
- }.Build(),
- }.Build()
+ },
+ }
ctx, cancelFn := context.WithCancel(context.Background())
t.Cleanup(cancelFn)
@@ -54,60 +54,60 @@ func TestProcess(t *testing.T) {
errExpected bool
}{
{
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "Worker1",
ControlEndpoint: endpoint,
LoggingEndpoint: endpoint,
- }.Build(),
+ },
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "Worker2",
ControlEndpoint: endpoint,
LoggingEndpoint: endpoint,
- }.Build(),
+ },
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "Worker1",
ControlEndpoint: endpoint,
LoggingEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true, // Repeated start
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "missingControl",
LoggingEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "missingLogging",
ControlEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "secureLogging",
LoggingEndpoint: secureEndpoint,
ControlEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "secureControl",
LoggingEndpoint: endpoint,
ControlEndpoint: secureEndpoint,
- }.Build(),
+ },
errExpected: true,
},
}
for _, test := range startTests {
resp, err := server.StartWorker(ctx, test.req)
if test.errExpected {
- if err != nil || resp.GetError() == "" {
+ if err != nil || resp.Error == "" {
t.Errorf("Expected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
} else {
- if err != nil || resp.GetError() != "" {
+ if err != nil || resp.Error != "" {
t.Errorf("Unexpected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
}
@@ -117,29 +117,29 @@ func TestProcess(t *testing.T) {
errExpected bool
}{
{
- req: fnpb.StopWorkerRequest_builder{
+ req: &fnpb.StopWorkerRequest{
WorkerId: "Worker1",
- }.Build(),
+ },
}, {
- req: fnpb.StopWorkerRequest_builder{
+ req: &fnpb.StopWorkerRequest{
WorkerId: "Worker1",
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StopWorkerRequest_builder{
+ req: &fnpb.StopWorkerRequest{
WorkerId: "NonExistent",
- }.Build(),
+ },
errExpected: true,
},
}
for _, test := range stopTests {
resp, err := server.StopWorker(ctx, test.req)
if test.errExpected {
- if err != nil || resp.GetError() == "" {
+ if err != nil || resp.Error == "" {
t.Errorf("Expected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
} else {
- if err != nil || resp.GetError() != "" {
+ if err != nil || resp.Error != "" {
t.Errorf("Unexpected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
}
diff --git a/sdks/go/container/tools/buffered_logging_test.go b/sdks/go/container/tools/buffered_logging_test.go
index 9f11cf319c4..9f542d2d5ab 100644
--- a/sdks/go/container/tools/buffered_logging_test.go
+++ b/sdks/go/container/tools/buffered_logging_test.go
@@ -63,11 +63,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "test message"; got != want {
+ if got, want := received.Message, "test message"; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
})
@@ -96,11 +96,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()
for i, message := range received {
- if got, want := message.GetMessage(), messages[i]; got != want {
+ if got, want := message.Message, messages[i]; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := message.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := message.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
}
@@ -125,11 +125,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "test error"; got != want {
+ if got, want := received.Message, "test error"; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_ERROR; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_ERROR; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
})
@@ -158,11 +158,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()
for i, message := range received {
- if got, want := message.GetMessage(), messages[i]; got != want {
+ if got, want := message.Message, messages[i]; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := message.GetSeverity(), fnpb.LogEntry_Severity_ERROR; got != want {
+ if got, want := message.Severity, fnpb.LogEntry_Severity_ERROR; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
}
@@ -177,11 +177,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "foo bar"; got != want {
+ if got, want := received.Message, "foo bar"; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got severity %v, want %v", got, want)
}
})
@@ -229,11 +229,11 @@ func TestBufferedLogger(t *testing.T) {
messages = append(messages, lastMessage)
for i, message := range received {
- if got, want := message.GetMessage(), messages[i]; got != want {
+ if got, want := message.Message, messages[i]; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := message.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := message.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
}
diff --git a/sdks/go/container/tools/logging.go b/sdks/go/container/tools/logging.go
index ed0f7b82c9f..ced13b744d4 100644
--- a/sdks/go/container/tools/logging.go
+++ b/sdks/go/container/tools/logging.go
@@ -86,15 +86,15 @@ func (l *Logger) Log(ctx context.Context, sev fnpb.LogEntry_Severity_Enum, messa
}
}
- err := l.client.Send(fnpb.LogEntry_List_builder{
+ err := l.client.Send(&fnpb.LogEntry_List{
LogEntries: []*fnpb.LogEntry{
- fnpb.LogEntry_builder{
+ {
Severity: sev,
Timestamp: timestamppb.Now(),
Message: message,
- }.Build(),
+ },
},
- }.Build())
+ })
if err != nil {
exitErr = err
return
diff --git a/sdks/go/container/tools/logging_test.go b/sdks/go/container/tools/logging_test.go
index b837b40fa24..8730a0fe9c1 100644
--- a/sdks/go/container/tools/logging_test.go
+++ b/sdks/go/container/tools/logging_test.go
@@ -50,11 +50,11 @@ func TestLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "foo bar"; got != want {
+ if got, want := received.Message, "foo bar"; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got severity %v, want %v", got, want)
}
})
@@ -66,11 +66,11 @@ func TestLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "failed to install dependency bar"; got != want {
+ if got, want := received.Message, "failed to install dependency bar"; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_ERROR; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_ERROR; got != want {
t.Errorf("l.Errorf(\"failed to install dependency %%v\", \"bar\"): got severity %v, want %v", got, want)
}
})
diff --git a/sdks/go/container/tools/provision_test.go b/sdks/go/container/tools/provision_test.go
index 1820092c0cf..dbb78040632 100644
--- a/sdks/go/container/tools/provision_test.go
+++ b/sdks/go/container/tools/provision_test.go
@@ -66,7 +66,7 @@ type ProvisionServiceServicer struct {
}
func (p ProvisionServiceServicer) GetProvisionInfo(ctx context.Context, req *fnpb.GetProvisionInfoRequest) (*fnpb.GetProvisionInfoResponse, error) {
- return fnpb.GetProvisionInfoResponse_builder{Info: fnpb.ProvisionInfo_builder{RetrievalToken: "token"}.Build()}.Build(), nil
+ return &fnpb.GetProvisionInfoResponse{Info: &fnpb.ProvisionInfo{RetrievalToken: "token"}}, nil
}
func setup(addr *string, wg *sync.WaitGroup) {
@@ -103,7 +103,7 @@ func TestProvisionInfo(t *testing.T) {
if err != nil {
t.Errorf("error in response: %v", err)
}
- want := fnpb.ProvisionInfo_builder{RetrievalToken: "token"}.Build()
+ want := &fnpb.ProvisionInfo{RetrievalToken: "token"}
if got.GetRetrievalToken() != want.GetRetrievalToken() {
t.Errorf("provision.Info() = %v, want %v", got, want)
}
diff --git a/sdks/go/examples/snippets/06schemas_test.go b/sdks/go/examples/snippets/06schemas_test.go
index df8799cc141..325a96274c2 100644
--- a/sdks/go/examples/snippets/06schemas_test.go
+++ b/sdks/go/examples/snippets/06schemas_test.go
@@ -31,49 +31,55 @@ import (
)
func atomicSchemaField(name string, typ pipepb.AtomicType) *pipepb.Field {
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
- Type: pipepb.FieldType_builder{
- AtomicType: typ.Enum(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: typ,
+ },
+ },
+ }
}
func rowSchemaField(name string, typ *pipepb.Schema) *pipepb.Field {
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
- Type: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: typ,
- }.Build(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: typ,
+ },
+ },
+ },
+ }
}
func listSchemaField(name string, typ *pipepb.Field) *pipepb.Field {
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
- Type: pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: typ.GetType(),
- }.Build(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: typ.GetType(),
+ },
+ },
+ },
+ }
}
func nillable(f *pipepb.Field) *pipepb.Field {
- f.GetType().SetNullable(true)
+ f.Type.Nullable = true
return f
}
func TestSchemaTypes(t *testing.T) {
- transactionSchema := pipepb.Schema_builder{
+ transactionSchema := &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("bank", pipepb.AtomicType_STRING),
atomicSchemaField("purchaseAmount", pipepb.AtomicType_DOUBLE),
},
- }.Build()
- shippingAddressSchema := pipepb.Schema_builder{
+ }
+ shippingAddressSchema := &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("streetAddress", pipepb.AtomicType_STRING),
atomicSchemaField("city", pipepb.AtomicType_STRING),
@@ -81,7 +87,7 @@ func TestSchemaTypes(t *testing.T) {
atomicSchemaField("country", pipepb.AtomicType_STRING),
atomicSchemaField("postCode", pipepb.AtomicType_STRING),
},
- }.Build()
+ }
tests := []struct {
rt reflect.Type
@@ -95,7 +101,7 @@ func TestSchemaTypes(t *testing.T) {
st: shippingAddressSchema,
}, {
rt: reflect.TypeOf(Purchase{}),
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("userId", pipepb.AtomicType_STRING),
atomicSchemaField("itemId", pipepb.AtomicType_INT64),
@@ -104,15 +110,15 @@ func TestSchemaTypes(t *testing.T) {
listSchemaField("transactions",
rowSchemaField("n/a", transactionSchema)),
},
- }.Build(),
+ },
}, {
rt: tnType,
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("seconds", pipepb.AtomicType_INT64),
atomicSchemaField("nanos", pipepb.AtomicType_INT32),
},
- }.Build(),
+ },
preReg: func(reg *schema.Registry) {
reg.RegisterLogicalType(schema.ToLogicalType(tnType.Name(), tnType, tnStorageType))
},
diff --git a/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go b/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
index 602af701b22..ceb8a319be9 100644
--- a/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
+++ b/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
@@ -75,7 +75,7 @@ func NewRetrievalServer(md *jobpb.ProxyManifest) (*RetrievalServer, error) {
// GetManifest returns the manifest for all artifacts.
func (s *RetrievalServer) GetManifest(ctx context.Context, req *jobpb.GetManifestRequest) (*jobpb.GetManifestResponse, error) {
- return jobpb.GetManifestResponse_builder{Manifest: s.md}.Build(), nil
+ return &jobpb.GetManifestResponse{Manifest: s.md}, nil
}
// GetArtifact returns a given artifact.
@@ -105,7 +105,7 @@ func (s *RetrievalServer) GetArtifact(req *jobpb.LegacyGetArtifactRequest, strea
for {
n, err := r.Read(data)
if n > 0 {
- if err := stream.Send(jobpb.ArtifactChunk_builder{Data: data[:n]}.Build()); err != nil {
+ if err := stream.Send(&jobpb.ArtifactChunk{Data: data[:n]}); err != nil {
return errors.Wrap(err, "chunk send failed")
}
}
@@ -122,20 +122,20 @@ func (s *RetrievalServer) GetArtifact(req *jobpb.LegacyGetArtifactRequest, strea
func validate(md *jobpb.ProxyManifest) error {
keys := make(map[string]bool)
for _, a := range md.GetManifest().GetArtifact() {
- if _, seen := keys[a.GetName()]; seen {
- return errors.Errorf("multiple artifact with name %v", a.GetName())
+ if _, seen := keys[a.Name]; seen {
+ return errors.Errorf("multiple artifact with name %v", a.Name)
}
- keys[a.GetName()] = true
+ keys[a.Name] = true
}
for _, l := range md.GetLocation() {
- fresh, seen := keys[l.GetName()]
+ fresh, seen := keys[l.Name]
if !seen {
- return errors.Errorf("no artifact named %v for location %v", l.GetName(), l.GetUri())
+ return errors.Errorf("no artifact named %v for location %v", l.Name, l.Uri)
}
if !fresh {
- return errors.Errorf("multiple locations for %v:%v", l.GetName(), l.GetUri())
+ return errors.Errorf("multiple locations for %v:%v", l.Name, l.Uri)
}
- keys[l.GetName()] = false
+ keys[l.Name] = false
}
for key, fresh := range keys {
diff --git a/sdks/go/pkg/beam/artifact/gcsproxy/staging.go b/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
index 4031d94c4df..9113e780f33 100644
--- a/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
+++ b/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
@@ -75,7 +75,7 @@ func (s *StagingServer) CommitManifest(ctx context.Context, req *jobpb.CommitMan
}
s.mu.Unlock()
- data, err := proto.Marshal(jobpb.ProxyManifest_builder{Manifest: manifest, Location: loc}.Build())
+ data, err := proto.Marshal(&jobpb.ProxyManifest{Manifest: manifest, Location: loc})
if err != nil {
return nil, errors.Wrap(err, "failed to marshal proxy manifest")
}
@@ -93,7 +93,7 @@ func (s *StagingServer) CommitManifest(ctx context.Context, req *jobpb.CommitMan
// now, but would be needed for a staging server that serves multiple
// jobs. Such a server would also use the ID sent with each request.
- return jobpb.CommitManifestResponse_builder{RetrievalToken: gcsx.MakeObject(s.bucket, s.manifest)}.Build(), nil
+ return &jobpb.CommitManifestResponse{RetrievalToken: gcsx.MakeObject(s.bucket, s.manifest)}, nil
}
// matchLocations ensures that all artifacts have been staged and have valid
@@ -101,18 +101,18 @@ func (s *StagingServer) CommitManifest(ctx context.Context, req *jobpb.CommitMan
func matchLocations(artifacts []*jobpb.ArtifactMetadata, blobs map[string]staged) ([]*jobpb.ProxyManifest_Location, error) {
var loc []*jobpb.ProxyManifest_Location
for _, a := range artifacts {
- info, ok := blobs[a.GetName()]
+ info, ok := blobs[a.Name]
if !ok {
- return nil, errors.Errorf("artifact %v not staged", a.GetName())
+ return nil, errors.Errorf("artifact %v not staged", a.Name)
}
- if a.GetSha256() == "" {
- a.SetSha256(info.hash)
+ if a.Sha256 == "" {
+ a.Sha256 = info.hash
}
- if info.hash != a.GetSha256() {
- return nil, errors.Errorf("staged artifact for %v has invalid SHA256: %v, want %v", a.GetName(), info.hash, a.GetSha256())
+ if info.hash != a.Sha256 {
+ return nil, errors.Errorf("staged artifact for %v has invalid SHA256: %v, want %v", a.Name, info.hash, a.Sha256)
}
- loc = append(loc, jobpb.ProxyManifest_Location_builder{Name: a.GetName(), Uri: info.object}.Build())
+ loc = append(loc, &jobpb.ProxyManifest_Location{Name: a.Name, Uri: info.object})
}
return loc, nil
}
@@ -129,7 +129,7 @@ func (s *StagingServer) PutArtifact(ps jobpb.LegacyArtifactStagingService_PutArt
if md == nil {
return errors.Errorf("expected header as first message: %v", header)
}
- object := path.Join(s.root, md.GetName())
+ object := path.Join(s.root, md.Name)
// Stream content to GCS. We don't have to worry about partial
// or abandoned writes, because object writes are atomic.
@@ -142,15 +142,15 @@ func (s *StagingServer) PutArtifact(ps jobpb.LegacyArtifactStagingService_PutArt
r := &reader{sha256W: sha256.New(), stream: ps}
if err := gcsx.WriteObject(ctx, cl, s.bucket, object, r); err != nil {
- return errors.Wrapf(err, "failed to stage artifact %v", md.GetName())
+ return errors.Wrapf(err, "failed to stage artifact %v", md.Name)
}
hash := r.SHA256()
- if md.GetSha256() != "" && md.GetSha256() != hash {
- return errors.Errorf("invalid SHA256 for artifact %v: %v want %v", md.GetName(), hash, md.GetSha256())
+ if md.Sha256 != "" && md.Sha256 != hash {
+ return errors.Errorf("invalid SHA256 for artifact %v: %v want %v", md.Name, hash, md.Sha256)
}
s.mu.Lock()
- s.blobs[md.GetName()] = staged{object: gcsx.MakeObject(s.bucket, object), hash: hash}
+ s.blobs[md.Name] = staged{object: gcsx.MakeObject(s.bucket, object), hash: hash}
s.mu.Unlock()
return ps.SendAndClose(&jobpb.PutArtifactResponse{})
diff --git a/sdks/go/pkg/beam/artifact/materialize.go b/sdks/go/pkg/beam/artifact/materialize.go
index e955941061d..624e30efcd2 100644
--- a/sdks/go/pkg/beam/artifact/materialize.go
+++ b/sdks/go/pkg/beam/artifact/materialize.go
@@ -78,59 +78,59 @@ func newMaterialize(ctx context.Context, endpoint string, dependencies []*pipepb
}
func newMaterializeWithClient(ctx context.Context, client jobpb.ArtifactRetrievalServiceClient, dependencies []*pipepb.ArtifactInformation, dest string) ([]*pipepb.ArtifactInformation, error) {
- resolution, err := client.ResolveArtifacts(ctx, jobpb.ResolveArtifactsRequest_builder{Artifacts: dependencies}.Build())
+ resolution, err := client.ResolveArtifacts(ctx, &jobpb.ResolveArtifactsRequest{Artifacts: dependencies})
if err != nil {
return nil, err
}
var artifacts []*pipepb.ArtifactInformation
var list []retrievable
- for _, dep := range resolution.GetReplacements() {
+ for _, dep := range resolution.Replacements {
path, err := extractStagingToPath(dep)
if err != nil {
return nil, err
}
- filePayload := pipepb.ArtifactFilePayload_builder{
+ filePayload := pipepb.ArtifactFilePayload{
Path: path,
- }.Build()
- if dep.GetTypeUrn() == URNFileArtifact {
- typePayload := &pipepb.ArtifactFilePayload{}
- if err := proto.Unmarshal(dep.GetTypePayload(), typePayload); err != nil {
+ }
+ if dep.TypeUrn == URNFileArtifact {
+ typePayload := pipepb.ArtifactFilePayload{}
+ if err := proto.Unmarshal(dep.TypePayload, &typePayload); err != nil {
return nil, errors.Wrap(err, "failed to parse artifact file payload")
}
- filePayload.SetSha256(typePayload.GetSha256())
- } else if dep.GetTypeUrn() == URNUrlArtifact {
- typePayload := &pipepb.ArtifactUrlPayload{}
- if err := proto.Unmarshal(dep.GetTypePayload(), typePayload); err != nil {
+ filePayload.Sha256 = typePayload.Sha256
+ } else if dep.TypeUrn == URNUrlArtifact {
+ typePayload := pipepb.ArtifactUrlPayload{}
+ if err := proto.Unmarshal(dep.TypePayload, &typePayload); err != nil {
return nil, errors.Wrap(err, "failed to parse artifact url payload")
}
- filePayload.SetSha256(typePayload.GetSha256())
+ filePayload.Sha256 = typePayload.Sha256
}
- newTypePayload, err := proto.Marshal(filePayload)
+ newTypePayload, err := proto.Marshal(&filePayload)
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact type payload")
}
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNFileArtifact,
TypePayload: newTypePayload,
- RoleUrn: dep.GetRoleUrn(),
- RolePayload: dep.GetRolePayload(),
- }.Build())
+ RoleUrn: dep.RoleUrn,
+ RolePayload: dep.RolePayload,
+ })
- rolePayload, err := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
+ rolePayload, err := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
StagedName: path,
- }.Build())
+ })
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact role payload")
}
list = append(list, &artifact{
client: client,
- dep: pipepb.ArtifactInformation_builder{
- TypeUrn: dep.GetTypeUrn(),
- TypePayload: dep.GetTypePayload(),
+ dep: &pipepb.ArtifactInformation{
+ TypeUrn: dep.TypeUrn,
+ TypePayload: dep.TypePayload,
RoleUrn: URNStagingTo,
RolePayload: rolePayload,
- }.Build(),
+ },
})
}
@@ -147,39 +147,39 @@ func generateId() string {
func extractStagingToPath(artifact *pipepb.ArtifactInformation) (string, error) {
var stagedName string
- if artifact.GetRoleUrn() == URNStagingTo {
- role := &pipepb.ArtifactStagingToRolePayload{}
- if err := proto.Unmarshal(artifact.GetRolePayload(), role); err != nil {
+ if artifact.RoleUrn == URNStagingTo {
+ role := pipepb.ArtifactStagingToRolePayload{}
+ if err := proto.Unmarshal(artifact.RolePayload, &role); err != nil {
return "", err
}
- stagedName = role.GetStagedName()
- } else if artifact.GetTypeUrn() == URNFileArtifact {
- ty := &pipepb.ArtifactFilePayload{}
- if err := proto.Unmarshal(artifact.GetTypePayload(), ty); err != nil {
+ stagedName = role.StagedName
+ } else if artifact.TypeUrn == URNFileArtifact {
+ ty := pipepb.ArtifactFilePayload{}
+ if err := proto.Unmarshal(artifact.TypePayload, &ty); err != nil {
return "", err
}
- stagedName = generateId() + "-" + filepath.Base(ty.GetPath())
- } else if artifact.GetTypeUrn() == URNUrlArtifact {
- ty := &pipepb.ArtifactUrlPayload{}
- if err := proto.Unmarshal(artifact.GetTypePayload(), ty); err != nil {
+ stagedName = generateId() + "-" + filepath.Base(ty.Path)
+ } else if artifact.TypeUrn == URNUrlArtifact {
+ ty := pipepb.ArtifactUrlPayload{}
+ if err := proto.Unmarshal(artifact.TypePayload, &ty); err != nil {
return "", err
}
- stagedName = generateId() + "-" + path.Base(ty.GetUrl())
+ stagedName = generateId() + "-" + path.Base(ty.Url)
} else {
- return "", errors.Errorf("failed to extract staging path for artifact type %v role %v", artifact.GetTypeUrn(), artifact.GetRoleUrn())
+ return "", errors.Errorf("failed to extract staging path for artifact type %v role %v", artifact.TypeUrn, artifact.RoleUrn)
}
return stagedName, nil
}
func MustExtractFilePayload(artifact *pipepb.ArtifactInformation) (string, string) {
- if artifact.GetTypeUrn() != URNFileArtifact {
- log.Fatalf("Unsupported artifact type %v", artifact.GetTypeUrn())
+ if artifact.TypeUrn != URNFileArtifact {
+ log.Fatalf("Unsupported artifact type %v", artifact.TypeUrn)
}
- ty := &pipepb.ArtifactFilePayload{}
- if err := proto.Unmarshal(artifact.GetTypePayload(), ty); err != nil {
+ ty := pipepb.ArtifactFilePayload{}
+ if err := proto.Unmarshal(artifact.TypePayload, &ty); err != nil {
log.Fatalf("failed to parse artifact file payload: %v", err)
}
- return ty.GetPath(), ty.GetSha256()
+ return ty.Path, ty.Sha256
}
type artifact struct {
@@ -208,7 +208,7 @@ func (a artifact) retrieve(ctx context.Context, dest string) error {
return err
}
- stream, err := a.client.GetArtifact(ctx, jobpb.GetArtifactRequest_builder{Artifact: a.dep}.Build())
+ stream, err := a.client.GetArtifact(ctx, &jobpb.GetArtifactRequest{Artifact: a.dep})
if err != nil {
return err
}
@@ -244,10 +244,10 @@ func writeChunks(stream jobpb.ArtifactRetrievalService_GetArtifactClient, w io.W
if err != nil {
return "", err
}
- if _, err := sha256W.Write(chunk.GetData()); err != nil {
+ if _, err := sha256W.Write(chunk.Data); err != nil {
panic(err) // cannot fail
}
- if _, err := w.Write(chunk.GetData()); err != nil {
+ if _, err := w.Write(chunk.Data); err != nil {
return "", errors.Wrapf(err, "chunk write failed")
}
}
@@ -263,7 +263,7 @@ func legacyMaterialize(ctx context.Context, endpoint string, rt string, dest str
client := jobpb.NewLegacyArtifactRetrievalServiceClient(cc)
- m, err := client.GetManifest(ctx, jobpb.GetManifestRequest_builder{RetrievalToken: rt}.Build())
+ m, err := client.GetManifest(ctx, &jobpb.GetManifestRequest{RetrievalToken: rt})
if err != nil {
return nil, errors.Wrap(err, "failed to get manifest")
}
@@ -272,25 +272,25 @@ func legacyMaterialize(ctx context.Context, endpoint string, rt string, dest str
var artifacts []*pipepb.ArtifactInformation
var list []retrievable
for _, md := range mds {
- typePayload, err := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: md.GetName(),
- Sha256: md.GetSha256(),
- }.Build())
+ typePayload, err := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: md.Name,
+ Sha256: md.Sha256,
+ })
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact type payload")
}
- rolePayload, err := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: md.GetName(),
- }.Build())
+ rolePayload, err := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: md.Name,
+ })
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact role payload")
}
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNFileArtifact,
TypePayload: typePayload,
RoleUrn: URNStagingTo,
RolePayload: rolePayload,
- }.Build())
+ })
list = append(list, &legacyArtifact{
client: client,
rt: rt,
@@ -382,7 +382,7 @@ func (a legacyArtifact) retrieve(ctx context.Context, dest string) error {
// previous retrieval attempt and may leave a corrupt/partial local file on
// failure.
func Retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceClient, a *jobpb.ArtifactMetadata, rt string, dest string) error {
- filename := filepath.Join(dest, filepath.FromSlash(a.GetName()))
+ filename := filepath.Join(dest, filepath.FromSlash(a.Name))
_, err := os.Stat(filename)
if err != nil && !os.IsNotExist(err) {
@@ -392,7 +392,7 @@ func Retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceCl
// File already exists. Validate or delete.
hash, err := computeSHA256(filename)
- if err == nil && a.GetSha256() == hash {
+ if err == nil && a.Sha256 == hash {
// NOTE(herohde) 10/5/2017: We ignore permissions here, because
// they may differ from the requested permissions due to umask
// settings on unix systems (which we in turn want to respect).
@@ -417,12 +417,12 @@ func Retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceCl
// It expects the file to not exist, but does not clean up on failure and
// may leave a corrupt file.
func retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceClient, a *jobpb.ArtifactMetadata, rt string, filename string) error {
- stream, err := client.GetArtifact(ctx, jobpb.LegacyGetArtifactRequest_builder{Name: a.GetName(), RetrievalToken: rt}.Build())
+ stream, err := client.GetArtifact(ctx, &jobpb.LegacyGetArtifactRequest{Name: a.Name, RetrievalToken: rt})
if err != nil {
return err
}
- fd, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(a.GetPermissions()))
+ fd, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(a.Permissions))
if err != nil {
return err
}
@@ -442,8 +442,8 @@ func retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceCl
}
// Artifact Sha256 hash is an optional field in metadata so we should only validate when its present.
- if a.GetSha256() != "" && sha256Hash != a.GetSha256() {
- return errors.Errorf("bad SHA256 for %v: %v, want %v", filename, sha256Hash, a.GetSha256())
+ if a.Sha256 != "" && sha256Hash != a.Sha256 {
+ return errors.Errorf("bad SHA256 for %v: %v, want %v", filename, sha256Hash, a.Sha256)
}
return nil
}
@@ -459,10 +459,10 @@ func retrieveChunks(stream jobpb.LegacyArtifactRetrievalService_GetArtifactClien
return "", err
}
- if _, err := sha256W.Write(chunk.GetData()); err != nil {
+ if _, err := sha256W.Write(chunk.Data); err != nil {
panic(err) // cannot fail
}
- if _, err := w.Write(chunk.GetData()); err != nil {
+ if _, err := w.Write(chunk.Data); err != nil {
return "", errors.Wrapf(err, "chunk write failed")
}
}
diff --git a/sdks/go/pkg/beam/artifact/materialize_test.go b/sdks/go/pkg/beam/artifact/materialize_test.go
index 2eb3fb6c7a7..31890ed045c 100644
--- a/sdks/go/pkg/beam/artifact/materialize_test.go
+++ b/sdks/go/pkg/beam/artifact/materialize_test.go
@@ -49,12 +49,12 @@ func TestRetrieve(t *testing.T) {
client := jobpb.NewLegacyArtifactRetrievalServiceClient(cc)
for _, a := range artifacts {
- filename := makeFilename(dst, a.GetName())
+ filename := makeFilename(dst, a.Name)
if err := Retrieve(ctx, client, a, rt, dst); err != nil {
- t.Errorf("failed to retrieve %v: %v", a.GetName(), err)
+ t.Errorf("failed to retrieve %v: %v", a.Name, err)
continue
}
- verifySHA256(t, filename, a.GetSha256())
+ verifySHA256(t, filename, a.Sha256)
}
}
@@ -78,7 +78,7 @@ func TestMultiRetrieve(t *testing.T) {
}
for _, a := range artifacts {
- verifySHA256(t, makeFilename(dst, a.GetName()), a.GetSha256())
+ verifySHA256(t, makeFilename(dst, a.Name), a.Sha256)
}
}
@@ -112,18 +112,20 @@ func stage(ctx context.Context, scl jobpb.LegacyArtifactStagingServiceClient, t
sha256W.Write(data)
hash := hex.EncodeToString(sha256W.Sum(nil))
md := makeArtifact(key, hash)
- pmd := jobpb.PutArtifactMetadata_builder{
+ pmd := &jobpb.PutArtifactMetadata{
Metadata: md,
StagingSessionToken: st,
- }.Build()
+ }
stream, err := scl.PutArtifact(ctx)
if err != nil {
t.Fatalf("put failed: %v", err)
}
- header := jobpb.PutArtifactRequest_builder{
- Metadata: proto.ValueOrDefault(pmd),
- }.Build()
+ header := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Metadata{
+ Metadata: pmd,
+ },
+ }
if err := stream.Send(header); err != nil {
t.Fatalf("send header failed: %v", err)
}
@@ -134,11 +136,13 @@ func stage(ctx context.Context, scl jobpb.LegacyArtifactStagingServiceClient, t
end = size
}
- chunk := jobpb.PutArtifactRequest_builder{
- Data: jobpb.ArtifactChunk_builder{
- Data: data[i:end],
- }.Build(),
- }.Build()
+ chunk := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Data{
+ Data: &jobpb.ArtifactChunk{
+ Data: data[i:end],
+ },
+ },
+ }
if err := stream.Send(chunk); err != nil {
t.Fatalf("send chunk[%v:%v] failed: %v", i, end, err)
}
@@ -189,8 +193,8 @@ func TestNewRetrieveWithFileGeneratedStageName(t *testing.T) {
generated := make(map[string]string)
for _, md := range mds {
name, _ := MustExtractFilePayload(md)
- payload, _ := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: name}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: name})
generated[name] = string(payload)
}
@@ -216,8 +220,8 @@ func TestNewRetrieveWithUrlGeneratedStageName(t *testing.T) {
generated := make(map[string]string)
for _, md := range mds {
name, _ := MustExtractFilePayload(md)
- payload, _ := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: name}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: name})
generated[name] = string(payload)
}
@@ -294,14 +298,14 @@ type fakeRetrievalService struct {
func (fake *fakeRetrievalService) resolvedArtifactsWithStagingTo() []*pipepb.ArtifactInformation {
var artifacts []*pipepb.ArtifactInformation
for name, contents := range fake.artifacts {
- payload, _ := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: name}.Build())
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ payload, _ := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: name})
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: "resolved",
TypePayload: []byte(contents),
RoleUrn: URNStagingTo,
RolePayload: payload,
- }.Build())
+ })
}
return artifacts
}
@@ -309,12 +313,12 @@ func (fake *fakeRetrievalService) resolvedArtifactsWithStagingTo() []*pipepb.Art
func (fake *fakeRetrievalService) fileArtifactsWithoutStagingTo() []*pipepb.ArtifactInformation {
var artifacts []*pipepb.ArtifactInformation
for name := range fake.artifacts {
- payload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: filepath.Join("/tmp", name)}.Build())
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ payload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: filepath.Join("/tmp", name)})
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNFileArtifact,
TypePayload: payload,
- }.Build())
+ })
}
return artifacts
}
@@ -322,44 +326,44 @@ func (fake *fakeRetrievalService) fileArtifactsWithoutStagingTo() []*pipepb.Arti
func (fake *fakeRetrievalService) urlArtifactsWithoutStagingTo() []*pipepb.ArtifactInformation {
var artifacts []*pipepb.ArtifactInformation
for name := range fake.artifacts {
- payload, _ := proto.Marshal(pipepb.ArtifactUrlPayload_builder{
- Url: path.Join("gs://tmp", name)}.Build())
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ payload, _ := proto.Marshal(&pipepb.ArtifactUrlPayload{
+ Url: path.Join("gs://tmp", name)})
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNUrlArtifact,
TypePayload: payload,
- }.Build())
+ })
}
return artifacts
}
func (fake *fakeRetrievalService) unresolvedArtifacts() []*pipepb.ArtifactInformation {
return []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ &pipepb.ArtifactInformation{
TypeUrn: "unresolved",
- }.Build(),
+ },
}
}
func (fake *fakeRetrievalService) ResolveArtifacts(ctx context.Context, request *jobpb.ResolveArtifactsRequest, opts ...grpc.CallOption) (*jobpb.ResolveArtifactsResponse, error) {
- response := &jobpb.ResolveArtifactsResponse{}
- for _, dep := range request.GetArtifacts() {
- if dep.GetTypeUrn() == "unresolved" {
- response.SetReplacements(append(response.GetReplacements(), fake.resolvedArtifactsWithStagingTo()...))
+ response := jobpb.ResolveArtifactsResponse{}
+ for _, dep := range request.Artifacts {
+ if dep.TypeUrn == "unresolved" {
+ response.Replacements = append(response.Replacements, fake.resolvedArtifactsWithStagingTo()...)
} else {
- response.SetReplacements(append(response.GetReplacements(), dep))
+ response.Replacements = append(response.Replacements, dep)
}
}
- return response, nil
+ return &response, nil
}
func (fake *fakeRetrievalService) GetArtifact(ctx context.Context, request *jobpb.GetArtifactRequest, opts ...grpc.CallOption) (jobpb.ArtifactRetrievalService_GetArtifactClient, error) {
- switch request.GetArtifact().GetTypeUrn() {
+ switch request.Artifact.TypeUrn {
case "resolved":
- return &fakeGetArtifactResponseStream{data: request.GetArtifact().GetTypePayload()}, nil
+ return &fakeGetArtifactResponseStream{data: request.Artifact.TypePayload}, nil
case URNFileArtifact, URNUrlArtifact:
- return &fakeGetArtifactResponseStream{data: request.GetArtifact().GetRolePayload()}, nil
+ return &fakeGetArtifactResponseStream{data: request.Artifact.RolePayload}, nil
default:
- return nil, errors.Errorf("Unsupported artifact %v", request.GetArtifact())
+ return nil, errors.Errorf("Unsupported artifact %v", request.Artifact)
}
}
@@ -371,7 +375,7 @@ type fakeGetArtifactResponseStream struct {
func (fake *fakeGetArtifactResponseStream) Recv() (*jobpb.GetArtifactResponse, error) {
if fake.index < len(fake.data) {
fake.index++
- return jobpb.GetArtifactResponse_builder{Data: fake.data[fake.index-1 : fake.index]}.Build(), nil
+ return &jobpb.GetArtifactResponse{Data: fake.data[fake.index-1 : fake.index]}, nil
}
return nil, io.EOF
}
@@ -447,11 +451,11 @@ func makeTempFile(t *testing.T, filename string, size int) string {
}
func makeArtifact(key, hash string) *jobpb.ArtifactMetadata {
- return jobpb.ArtifactMetadata_builder{
+ return &jobpb.ArtifactMetadata{
Name: key,
Sha256: hash,
Permissions: 0644,
- }.Build()
+ }
}
func makeFilename(dir, key string) string {
diff --git a/sdks/go/pkg/beam/artifact/server_test.go b/sdks/go/pkg/beam/artifact/server_test.go
index f0ce3a59f05..3fdc8fa5cac 100644
--- a/sdks/go/pkg/beam/artifact/server_test.go
+++ b/sdks/go/pkg/beam/artifact/server_test.go
@@ -85,7 +85,7 @@ func (s *server) PutArtifact(ps jobpb.LegacyArtifactStagingService_PutArtifactSe
if header.GetMetadata() == nil {
return errors.Errorf("expected header as first message: %v", header)
}
- key := header.GetMetadata().GetMetadata().GetName()
+ key := header.GetMetadata().GetMetadata().Name
if header.GetMetadata().GetStagingSessionToken() == "" {
return errors.New("missing staging session token")
}
@@ -138,19 +138,19 @@ func (s *server) CommitManifest(ctx context.Context, req *jobpb.CommitManifestRe
artifacts := req.GetManifest().GetArtifact()
for _, md := range artifacts {
- if _, ok := m.m[md.GetName()]; !ok {
- return nil, errors.Errorf("artifact %v not staged", md.GetName())
+ if _, ok := m.m[md.Name]; !ok {
+ return nil, errors.Errorf("artifact %v not staged", md.Name)
}
}
// Update commit. Only one manifest can exist for each staging id.
for _, md := range artifacts {
- m.m[md.GetName()].md = md
+ m.m[md.Name].md = md
}
m.md = req.GetManifest()
- return jobpb.CommitManifestResponse_builder{RetrievalToken: token}.Build(), nil
+ return &jobpb.CommitManifestResponse{RetrievalToken: token}, nil
}
func (s *server) GetManifest(ctx context.Context, req *jobpb.GetManifestRequest) (*jobpb.GetManifestResponse, error) {
@@ -166,7 +166,7 @@ func (s *server) GetManifest(ctx context.Context, req *jobpb.GetManifestRequest)
m.mu.Lock()
defer m.mu.Unlock()
- return jobpb.GetManifestResponse_builder{Manifest: m.md}.Build(), nil
+ return &jobpb.GetManifestResponse{Manifest: m.md}, nil
}
func (s *server) GetArtifact(req *jobpb.LegacyGetArtifactRequest, stream jobpb.LegacyArtifactRetrievalService_GetArtifactServer) error {
@@ -195,7 +195,7 @@ func (s *server) GetArtifact(req *jobpb.LegacyGetArtifactRequest, stream jobpb.L
// Send chunks exactly as we received them.
for _, chunk := range chunks {
- if err := stream.Send(jobpb.ArtifactChunk_builder{Data: chunk}.Build()); err != nil {
+ if err := stream.Send(&jobpb.ArtifactChunk{Data: chunk}); err != nil {
return err
}
}
diff --git a/sdks/go/pkg/beam/artifact/stage.go b/sdks/go/pkg/beam/artifact/stage.go
index d248380c836..de017b4c3c0 100644
--- a/sdks/go/pkg/beam/artifact/stage.go
+++ b/sdks/go/pkg/beam/artifact/stage.go
@@ -31,18 +31,17 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
jobpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1"
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/errorx"
- "google.golang.org/protobuf/proto"
)
// Commit commits a manifest with the given staged artifacts. It returns the
// staging token, if successful.
func Commit(ctx context.Context, client jobpb.LegacyArtifactStagingServiceClient, artifacts []*jobpb.ArtifactMetadata, st string) (string, error) {
- req := jobpb.CommitManifestRequest_builder{
- Manifest: jobpb.Manifest_builder{
+ req := &jobpb.CommitManifestRequest{
+ Manifest: &jobpb.Manifest{
Artifact: artifacts,
- }.Build(),
+ },
StagingSessionToken: st,
- }.Build()
+ }
resp, err := client.CommitManifest(ctx, req)
if err != nil {
return "", err
@@ -128,15 +127,15 @@ func Stage(ctx context.Context, client jobpb.LegacyArtifactStagingServiceClient,
if err != nil {
return nil, err
}
- md := jobpb.ArtifactMetadata_builder{
+ md := &jobpb.ArtifactMetadata{
Name: key,
Permissions: uint32(stat.Mode()),
Sha256: hash,
- }.Build()
- pmd := jobpb.PutArtifactMetadata_builder{
+ }
+ pmd := &jobpb.PutArtifactMetadata{
Metadata: md,
StagingSessionToken: st,
- }.Build()
+ }
fd, err := os.Open(filename)
if err != nil {
@@ -149,9 +148,11 @@ func Stage(ctx context.Context, client jobpb.LegacyArtifactStagingServiceClient,
return nil, err
}
- header := jobpb.PutArtifactRequest_builder{
- Metadata: proto.ValueOrDefault(pmd),
- }.Build()
+ header := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Metadata{
+ Metadata: pmd,
+ },
+ }
if err := stream.Send(header); err != nil {
stream.CloseAndRecv() // ignore error
return nil, errors.Wrapf(err, "failed to send header for %v", filename)
@@ -180,11 +181,13 @@ func stageChunks(stream jobpb.LegacyArtifactStagingService_PutArtifactClient, r
panic(err) // cannot fail
}
- chunk := jobpb.PutArtifactRequest_builder{
- Data: jobpb.ArtifactChunk_builder{
- Data: data[:n],
- }.Build(),
- }.Build()
+ chunk := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Data{
+ Data: &jobpb.ArtifactChunk{
+ Data: data[:n],
+ },
+ },
+ }
err := stream.Send(chunk)
if err == io.EOF {
return "", err
diff --git a/sdks/go/pkg/beam/artifact/stage_test.go b/sdks/go/pkg/beam/artifact/stage_test.go
index af2af73ad75..2355f0a0176 100644
--- a/sdks/go/pkg/beam/artifact/stage_test.go
+++ b/sdks/go/pkg/beam/artifact/stage_test.go
@@ -84,7 +84,7 @@ func validate(ctx context.Context, cc *grpc.ClientConn, t *testing.T, keys, sha2
rcl := jobpb.NewLegacyArtifactRetrievalServiceClient(cc)
for i, key := range keys {
- stream, err := rcl.GetArtifact(ctx, jobpb.LegacyGetArtifactRequest_builder{Name: key, RetrievalToken: rt}.Build())
+ stream, err := rcl.GetArtifact(ctx, &jobpb.LegacyGetArtifactRequest{Name: key, RetrievalToken: rt})
if err != nil {
t.Fatalf("failed to get artifact for %v: %v", key, err)
}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate.go b/sdks/go/pkg/beam/core/runtime/exec/translate.go
index 37b379784ce..b74ede228fd 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/translate.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/translate.go
@@ -349,7 +349,7 @@ func (b *builder) makeCoderForPCollection(id string) (*coder.Coder, *coder.Windo
if !ok {
return nil, nil, errors.Errorf("pcollection %v not found", id)
}
- c, err := b.coders.Coder(col.GetCoderId())
+ c, err := b.coders.Coder(col.CoderId)
if err != nil {
return nil, nil, err
}
@@ -539,11 +539,11 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
var cID string
var kcID string
if rmw := spec.GetReadModifyWriteSpec(); rmw != nil {
- cID = rmw.GetCoderId()
+ cID = rmw.CoderId
} else if bs := spec.GetBagSpec(); bs != nil {
- cID = bs.GetElementCoderId()
+ cID = bs.ElementCoderId
} else if cs := spec.GetCombiningSpec(); cs != nil {
- cID = cs.GetAccumulatorCoderId()
+ cID = cs.AccumulatorCoderId
cmbData := string(cs.GetCombineFn().GetPayload())
var cmbTp v1pb.TransformPayload
if err := protox.DecodeBase64(cmbData, &cmbTp); err != nil {
@@ -559,10 +559,10 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
}
stateIDToCombineFn[key] = cfn
} else if ms := spec.GetMapSpec(); ms != nil {
- cID = ms.GetValueCoderId()
- kcID = ms.GetKeyCoderId()
+ cID = ms.ValueCoderId
+ kcID = ms.KeyCoderId
} else if ss := spec.GetSetSpec(); ss != nil {
- kcID = ss.GetElementCoderId()
+ kcID = ss.ElementCoderId
} else {
return nil, errors.Errorf("Unrecognized state type %v", spec)
}
@@ -801,10 +801,10 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
u = &MapWindows{UID: b.idgen.New(), Fn: mapper, Out: out[0], FnUrn: fn.GetUrn()}
case graphx.URNFlatten:
- u = &Flatten{UID: b.idgen.New(), N: len(transform.GetInputs()), Out: out[0]}
+ u = &Flatten{UID: b.idgen.New(), N: len(transform.Inputs), Out: out[0]}
// Use the same flatten instance for all the inputs links to this transform.
- for i := 0; i < len(transform.GetInputs()); i++ {
+ for i := 0; i < len(transform.Inputs); i++ {
b.links[linkID{id.to, i}] = u
}
@@ -918,5 +918,5 @@ func unmarshalPort(data []byte) (Port, string, error) {
}
return Port{
URL: port.GetApiServiceDescriptor().GetUrl(),
- }, port.GetCoderId(), nil
+ }, port.CoderId, nil
}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate_test.go b/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
index 4203bc5c396..a9917ec456f 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
@@ -17,13 +17,12 @@ package exec
import (
"fmt"
+ fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1"
"reflect"
"strings"
"testing"
"time"
- fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1"
-
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
@@ -72,12 +71,12 @@ func TestUnmarshalKeyedValues(t *testing.T) {
func TestUnmarshalReshuffleCoders(t *testing.T) {
payloads := map[string][]byte{}
encode := func(id, urn string, comps ...string) {
- payloads[id] = protox.MustEncode(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ payloads[id] = protox.MustEncode(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urn,
- }.Build(),
+ },
ComponentCoderIds: comps,
- }.Build())
+ })
}
encode("a", "beam:coder:bytes:v1")
encode("b", "beam:coder:string_utf8:v1")
@@ -268,37 +267,37 @@ func TestUnmarshalWindowMapper(t *testing.T) {
func makeWindowFn(w *window.Fn) (*pipepb.FunctionSpec, error) {
switch w.Kind {
case window.GlobalWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNGlobalWindowsWindowFn,
- }.Build(), nil
+ }, nil
case window.FixedWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNFixedWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.FixedWindowsPayload_builder{
+ &pipepb.FixedWindowsPayload{
Size: durationpb.New(w.Size),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.SlidingWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNSlidingWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.SlidingWindowsPayload_builder{
+ &pipepb.SlidingWindowsPayload{
Size: durationpb.New(w.Size),
Period: durationpb.New(w.Period),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.Sessions:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNSessionsWindowFn,
Payload: protox.MustEncode(
- pipepb.SessionWindowsPayload_builder{
+ &pipepb.SessionWindowsPayload{
GapSize: durationpb.New(w.Gap),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
default:
return nil, errors.Errorf("unexpected windowing strategy: %v", w)
}
@@ -311,11 +310,11 @@ func makeWindowMappingFn(w *window.Fn) (*pipepb.FunctionSpec, error) {
}
switch w.Kind {
case window.GlobalWindows:
- wFn.SetUrn(graphx.URNWindowMappingGlobal)
+ wFn.Urn = graphx.URNWindowMappingGlobal
case window.FixedWindows:
- wFn.SetUrn(graphx.URNWindowMappingFixed)
+ wFn.Urn = graphx.URNWindowMappingFixed
case window.SlidingWindows:
- wFn.SetUrn(graphx.URNWindowMappingSliding)
+ wFn.Urn = graphx.URNWindowMappingSliding
default:
return nil, fmt.Errorf("unknown window fn type %v", w.Kind)
}
@@ -394,7 +393,7 @@ func TestUnmarshalPort(t *testing.T) {
{
inputData: []byte{},
outputPort: Port{URL: port.GetApiServiceDescriptor().GetUrl()},
- outputStr: (&fnpb.RemoteGrpcPort{}).GetCoderId(),
+ outputStr: fnpb.RemoteGrpcPort{}.CoderId,
outputError: nil,
},
}
@@ -414,12 +413,12 @@ func TestUnmarshalPort(t *testing.T) {
}
func TestUnmarshalPlan(t *testing.T) {
- transform := pipepb.PTransform_builder{
- Spec: pipepb.FunctionSpec_builder{
+ transform := pipepb.PTransform{
+ Spec: &pipepb.FunctionSpec{
Urn: urnDataSource,
- }.Build(),
+ },
Outputs: map[string]string{},
- }.Build()
+ }
tests := []struct {
name string
inputDesc *fnpb.ProcessBundleDescriptor
@@ -428,32 +427,32 @@ func TestUnmarshalPlan(t *testing.T) {
}{
{
name: "test_no_root_units",
- inputDesc: fnpb.ProcessBundleDescriptor_builder{
+ inputDesc: &fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{},
- }.Build(),
+ },
outputPlan: nil,
outputError: errors.New("no root units"),
},
{
name: "test_zero_transform",
- inputDesc: fnpb.ProcessBundleDescriptor_builder{
+ inputDesc: &fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{
"": {},
},
- }.Build(),
+ },
outputPlan: nil,
outputError: errors.New("no root units"),
},
{
name: "test_transform_outputs_length_not_one",
- inputDesc: fnpb.ProcessBundleDescriptor_builder{
+ inputDesc: &fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{
- "": transform,
+ "": &transform,
},
- }.Build(),
+ },
outputPlan: nil,
outputError: errors.Errorf("expected one output from DataSource, got %v", transform.GetOutputs()),
},
@@ -474,10 +473,10 @@ func TestUnmarshalPlan(t *testing.T) {
}
func TestNewBuilder(t *testing.T) {
- descriptor := fnpb.ProcessBundleDescriptor_builder{
+ descriptor := fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{},
- }.Build()
+ }
tests := []struct {
name string
inputDesc *fnpb.ProcessBundleDescriptor
@@ -486,9 +485,9 @@ func TestNewBuilder(t *testing.T) {
}{
{
name: "test_1",
- inputDesc: descriptor,
+ inputDesc: &descriptor,
outputBuilder: &builder{
- desc: descriptor,
+ desc: &descriptor,
coders: graphx.NewCoderUnmarshaller(descriptor.GetCoders()),
prev: make(map[string]int),
succ: make(map[string][]linkID),
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/coder.go b/sdks/go/pkg/beam/core/runtime/graphx/coder.go
index 4ea77dee183..99ca5517d3d 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/coder.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/coder.go
@@ -454,12 +454,12 @@ func (b *CoderMarshaller) Add(c *coder.Coder) (string, error) {
if err != nil {
return "", errors.Wrapf(err, "failed to marshal custom coder %v", c)
}
- inner := b.internCoder(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ inner := b.internCoder(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urnCustomCoder,
Payload: []byte(data),
- }.Build(),
- }.Build())
+ },
+ })
return b.internBuiltInCoder(urnLengthPrefixCoder, inner), nil
case coder.KV:
@@ -596,22 +596,22 @@ func (b *CoderMarshaller) Build() map[string]*pipepb.Coder {
}
func (b *CoderMarshaller) internBuiltInCoder(urn string, components ...string) string {
- return b.internCoder(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ return b.internCoder(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urn,
- }.Build(),
+ },
ComponentCoderIds: components,
- }.Build())
+ })
}
func (b *CoderMarshaller) internRowCoder(schema *pipepb.Schema) string {
payload := protox.MustEncode(schema)
- return b.internCoder(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ return b.internCoder(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urnRowCoder,
Payload: payload,
- }.Build(),
- }.Build())
+ },
+ })
}
func (b *CoderMarshaller) internCoder(coder *pipepb.Coder) string {
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
index 2a23e364228..0d44e68285b 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
@@ -375,12 +375,12 @@ func (r *Registry) fromType(ot reflect.Type) (*pipepb.Schema, error) {
schm := ftype.GetRowType().GetSchema()
schm = proto.Clone(schm).(*pipepb.Schema)
if ot.Kind() == reflect.Ptr {
- schm.SetOptions(append(schm.GetOptions(), optGoNillable()))
+ schm.Options = append(schm.Options, optGoNillable())
}
if lID != "" {
- schm.SetOptions(append(schm.GetOptions(), logicalOption(lID)))
+ schm.Options = append(schm.Options, logicalOption(lID))
}
- schm.SetId(getUUID(ot))
+ schm.Id = getUUID(ot)
r.typeToSchema[ot] = schm
r.idToType[schm.GetId()] = ot
return schm, nil
@@ -395,8 +395,8 @@ func (r *Registry) fromType(ot reflect.Type) (*pipepb.Schema, error) {
// Cache the pointer type here with it's own id.
pt := reflect.PtrTo(t)
schm = proto.Clone(schm).(*pipepb.Schema)
- schm.SetId(getUUID(pt))
- schm.SetOptions(append(schm.GetOptions(), optGoNillable()))
+ schm.Id = getUUID(pt)
+ schm.Options = append(schm.Options, optGoNillable())
r.idToType[schm.GetId()] = pt
r.typeToSchema[pt] = schm
@@ -428,17 +428,23 @@ func optGoEmbedded() *pipepb.Option {
// is not set if the toggle isn't true, so the value is always
// true.
func newToggleOption(urn string) *pipepb.Option {
- return pipepb.Option_builder{
+ return &pipepb.Option{
Name: urn,
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BOOLEAN.Enum(),
- }.Build(),
- Value: pipepb.FieldValue_builder{
- AtomicValue: pipepb.AtomicTypeValue_builder{
- Boolean: proto.Bool(true),
- }.Build(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BOOLEAN,
+ },
+ },
+ Value: &pipepb.FieldValue{
+ FieldValue: &pipepb.FieldValue_AtomicValue{
+ AtomicValue: &pipepb.AtomicTypeValue{
+ Value: &pipepb.AtomicTypeValue_Boolean{
+ Boolean: true,
+ },
+ },
+ },
+ },
+ }
}
func checkOptions(opts []*pipepb.Option, urn string) *pipepb.Option {
@@ -459,20 +465,26 @@ func nillableFromOptions(opts []*pipepb.Option, t reflect.Type) reflect.Type {
return nil
}
-var optGoLogicalType = pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
-}.Build()
+var optGoLogicalType = &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+}
func logicalOption(lID string) *pipepb.Option {
- return pipepb.Option_builder{
+ return &pipepb.Option{
Name: optGoLogicalUrn,
Type: optGoLogicalType,
- Value: pipepb.FieldValue_builder{
- AtomicValue: pipepb.AtomicTypeValue_builder{
- String: proto.String(lID),
- }.Build(),
- }.Build(),
- }.Build()
+ Value: &pipepb.FieldValue{
+ FieldValue: &pipepb.FieldValue_AtomicValue{
+ AtomicValue: &pipepb.AtomicTypeValue{
+ Value: &pipepb.AtomicTypeValue_String_{
+ String_: lID,
+ },
+ },
+ },
+ },
+ }
}
// fromLogicalOption returns the logical type id of this top
@@ -501,8 +513,8 @@ func (r *Registry) structToSchema(t reflect.Type) (*pipepb.Schema, error) {
if ftype != nil {
schm := ftype.GetRowType().GetSchema()
schm = proto.Clone(schm).(*pipepb.Schema)
- schm.SetOptions(append(schm.GetOptions(), logicalOption(lID)))
- schm.SetId(getUUID(t))
+ schm.Options = append(schm.Options, logicalOption(lID))
+ schm.Id = getUUID(t)
r.typeToSchema[t] = schm
r.idToType[schm.GetId()] = t
return schm, nil
@@ -524,15 +536,15 @@ func (r *Registry) structToSchema(t reflect.Type) (*pipepb.Schema, error) {
}
if isAnon {
f = proto.Clone(f).(*pipepb.Field)
- f.SetOptions(append(f.GetOptions(), optGoEmbedded()))
+ f.Options = append(f.Options, optGoEmbedded())
}
fields = append(fields, f)
}
- schm := pipepb.Schema_builder{
+ schm := &pipepb.Schema{
Fields: fields,
Id: getUUID(t),
- }.Build()
+ }
r.idToType[schm.GetId()] = t
r.typeToSchema[t] = schm
return schm, nil
@@ -547,10 +559,10 @@ func (r *Registry) structFieldToField(sf reflect.StructField) (*pipepb.Field, er
if err != nil {
return nil, err
}
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
Type: ftype,
- }.Build(), nil
+ }, nil
}
func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, error) {
@@ -559,13 +571,15 @@ func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, e
return nil, err
}
if ftype != nil {
- return pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: lID,
- Representation: ftype,
- // TODO(BEAM-9615): Handle type Arguments.
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: lID,
+ Representation: ftype,
+ // TODO(BEAM-9615): Handle type Arguments.
+ },
+ },
+ }, nil
}
t := ot
@@ -575,7 +589,7 @@ func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, e
if err != nil {
return nil, errors.Wrapf(err, "unable to convert key of %v to schema field", ot)
}
- vt.SetNullable(true)
+ vt.Nullable = true
return vt, nil
case reflect.Map:
kt, err := r.reflectTypeToFieldType(t.Key())
@@ -586,45 +600,55 @@ func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, e
if err != nil {
return nil, errors.Wrapf(err, "unable to convert value of %v to schema field", ot)
}
- return pipepb.FieldType_builder{
- MapType: pipepb.MapType_builder{
- KeyType: kt,
- ValueType: vt,
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_MapType{
+ MapType: &pipepb.MapType{
+ KeyType: kt,
+ ValueType: vt,
+ },
+ },
+ }, nil
case reflect.Struct:
sch, err := r.structToSchema(t)
if err != nil {
return nil, errors.Wrapf(err, "unable to convert %v to schema field", ot)
}
- return pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: sch,
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: sch,
+ },
+ },
+ }, nil
case reflect.Slice, reflect.Array:
// Special handling for []byte
if t == reflectx.ByteSlice {
- return pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BYTES.Enum(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BYTES,
+ },
+ }, nil
}
vt, err := r.reflectTypeToFieldType(t.Elem())
if err != nil {
return nil, errors.Wrapf(err, "unable to convert element type of %v to schema field", ot)
}
- return pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: vt,
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: vt,
+ },
+ },
+ }, nil
case reflect.Interface, reflect.Func, reflect.Chan, reflect.UnsafePointer, reflect.Complex128, reflect.Complex64, reflect.Invalid:
return nil, errors.Errorf("unable to convert unsupported type %v to schema", ot)
default: // must be an atomic type
if enum, ok := reflectTypeToAtomicTypeMap[t.Kind()]; ok {
- return pipepb.FieldType_builder{
- AtomicType: enum.Enum(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: enum,
+ },
+ }, nil
}
return nil, errors.Errorf("unable to map %v to pipepb.AtomicType", t)
}
@@ -689,7 +713,7 @@ func (r *Registry) toType(s *pipepb.Schema) (reflect.Type, error) {
if err != nil {
return nil, errors.Wrapf(err, "cannot convert schema field %v to field", sf.GetName())
}
- if checkOptions(sf.GetOptions(), optGoEmbeddedUrn) != nil {
+ if checkOptions(sf.Options, optGoEmbeddedUrn) != nil {
rf.Anonymous = true
}
fields = append(fields, rf)
@@ -706,7 +730,7 @@ func (r *Registry) toType(s *pipepb.Schema) (reflect.Type, error) {
func (r *Registry) fieldToStructField(sf *pipepb.Field) (reflect.StructField, error) {
name := sf.GetName()
- rt, err := r.fieldTypeToReflectType(sf.GetType(), sf.GetOptions())
+ rt, err := r.fieldTypeToReflectType(sf.GetType(), sf.Options)
if err != nil {
return reflect.StructField{}, err
}
@@ -736,19 +760,19 @@ var atomicTypeToReflectType = map[pipepb.AtomicType]reflect.Type{
func (r *Registry) fieldTypeToReflectType(sft *pipepb.FieldType, opts []*pipepb.Option) (reflect.Type, error) {
var t reflect.Type
- switch sft.WhichTypeInfo() {
- case pipepb.FieldType_AtomicType_case:
+ switch sft.GetTypeInfo().(type) {
+ case *pipepb.FieldType_AtomicType:
var ok bool
if t, ok = atomicTypeToReflectType[sft.GetAtomicType()]; !ok {
return nil, errors.Errorf("unknown atomic type: %v", sft.GetAtomicType())
}
- case pipepb.FieldType_ArrayType_case:
+ case *pipepb.FieldType_ArrayType:
rt, err := r.fieldTypeToReflectType(sft.GetArrayType().GetElementType(), nil)
if err != nil {
return nil, errors.Wrap(err, "unable to convert array element type")
}
t = reflect.SliceOf(rt)
- case pipepb.FieldType_MapType_case:
+ case *pipepb.FieldType_MapType:
kt, err := r.fieldTypeToReflectType(sft.GetMapType().GetKeyType(), nil)
if err != nil {
return nil, errors.Wrap(err, "unable to convert map key type")
@@ -758,7 +782,7 @@ func (r *Registry) fieldTypeToReflectType(sft *pipepb.FieldType, opts []*pipepb.
return nil, errors.Wrap(err, "unable to convert map value type")
}
t = reflect.MapOf(kt, vt) // Panics for invalid map keys (slices/iterables)
- case pipepb.FieldType_RowType_case:
+ case *pipepb.FieldType_RowType:
rt, err := r.toType(sft.GetRowType().GetSchema())
if err != nil {
return nil, errors.Wrapf(err, "unable to convert row type: %v", sft.GetRowType().GetSchema().GetId())
@@ -767,7 +791,7 @@ func (r *Registry) fieldTypeToReflectType(sft *pipepb.FieldType, opts []*pipepb.
// case *pipepb.FieldType_IterableType:
// TODO(BEAM-9615): handle IterableTypes (eg. CoGBK values)
- case pipepb.FieldType_LogicalType_case:
+ case *pipepb.FieldType_LogicalType:
lst := sft.GetLogicalType()
identifier := lst.GetUrn()
lt, ok := r.logicalTypes[identifier]
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go
index ebe3d3898ab..367d70e81d1 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go
@@ -107,532 +107,650 @@ func TestSchemaConversion(t *testing.T) {
rt reflect.Type
}{
{
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "firstField",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
FirstField int32 `beam:"firstField"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "stringField",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "intPtrField",
- Type: pipepb.FieldType_builder{
- Nullable: true,
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ Nullable: true,
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
StringField string `beam:"stringField"`
IntPtrField *int32 `beam:"intPtrField"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "cypher",
- Type: pipepb.FieldType_builder{
- MapType: pipepb.MapType_builder{
- KeyType: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BOOLEAN.Enum(),
- }.Build(),
- ValueType: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_FLOAT.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_MapType{
+ MapType: &pipepb.MapType{
+ KeyType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BOOLEAN,
+ },
+ },
+ ValueType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_FLOAT,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: hasMapType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "wrapper",
- Type: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{pipepb.Field_builder{
- Name: "threshold",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_DOUBLE.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{{
+ Name: "threshold",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_DOUBLE,
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
Wrapper struct {
Threshold float64 `beam:"threshold"`
} `beam:"wrapper"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "payloads",
- Type: pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BYTES.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BYTES,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
Payloads [][]byte `beam:"payloads"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "AString",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "AnIntPtr",
- Type: pipepb.FieldType_builder{
- Nullable: true,
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ Nullable: true,
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
AString string
AnIntPtr *int32
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "A",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "B",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "C",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BOOLEAN.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BOOLEAN,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(registeredType{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "D",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(sRegisteredType{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "A",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "B",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "C",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(justAType{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Q",
- Type: pipepb.FieldType_builder{
- MapType: pipepb.MapType_builder{
- KeyType: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- ValueType: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(), pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_MapType{
+ MapType: &pipepb.MapType{
+ KeyType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ ValueType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }, {
Name: "T",
- Type: pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
Q map[int]int
T []int
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "SuperNES",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
},
Options: []*pipepb.Option{optGoNillable()},
- }.Build(),
+ },
rt: reflect.TypeOf(&struct {
SuperNES int16
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Options: []*pipepb.Option{
logicalOption("schema.unexportedFields"),
},
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "D",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "uint64",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "uint64",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: unexportedFieldsType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "G",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.unexportedFields",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "D",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "uint64",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.unexportedFields",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "D",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "uint64",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct{ G unexportedFields }{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "H",
- Type: pipepb.FieldType_builder{
+ Type: &pipepb.FieldType{
Nullable: true,
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.unexportedFields",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "D",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "uint64",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.unexportedFields",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "D",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "uint64",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct{ H *unexportedFields }{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "E",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
},
Options: []*pipepb.Option{optGoNillable(), logicalOption("*schema.exportedFunc")},
- }.Build(),
+ },
rt: exportedFuncType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Q",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.myInt",
- Representation: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.myInt",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: anotherType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Exported",
Options: []*pipepb.Option{optGoEmbedded()},
- Type: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "G",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.myInt",
- Representation: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "G",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.myInt",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: hasEmbeddedType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Exported",
Options: []*pipepb.Option{optGoEmbedded()},
- Type: pipepb.FieldType_builder{
+ Type: &pipepb.FieldType{
Nullable: true,
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "G",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.myInt",
- Representation: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "G",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.myInt",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: hasEmbeddedPtrType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "T",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
},
Options: []*pipepb.Option{optGoNillable()},
- }.Build(),
+ },
rt: reflect.TypeOf(&struct {
myInt
T string
i int
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Options: []*pipepb.Option{
logicalOption("schema.exportedFunc"),
},
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "V",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(exportedFunc{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "U",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.exportedFunc",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "V",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.exportedFunc",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "V",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
U exportedFunc
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "U",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.nonRegisteredLogical",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "K",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.nonRegisteredLogical",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "K",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
U nonRegisteredLogical
}{}),
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/translate.go b/sdks/go/pkg/beam/core/runtime/graphx/translate.go
index 41dd4f0b351..1e30d425850 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/translate.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/translate.go
@@ -126,27 +126,27 @@ func CreateEnvironment(ctx context.Context, urn string, extractEnvironmentConfig
return nil, errors.Errorf("unsupported environment %v", urn)
case URNEnvExternal:
config := extractEnvironmentConfig(ctx)
- payload := pipepb.ExternalPayload_builder{Endpoint: pipepb.ApiServiceDescriptor_builder{Url: config}.Build()}.Build()
+ payload := &pipepb.ExternalPayload{Endpoint: &pipepb.ApiServiceDescriptor{Url: config}}
serializedPayload = protox.MustEncode(payload)
case URNEnvDocker:
fallthrough
default:
config := extractEnvironmentConfig(ctx)
- payload := pipepb.DockerPayload_builder{ContainerImage: config}.Build()
+ payload := &pipepb.DockerPayload{ContainerImage: config}
serializedPayload = protox.MustEncode(payload)
}
- return pipepb.Environment_builder{
+ return &pipepb.Environment{
Urn: urn,
Payload: serializedPayload,
Capabilities: goCapabilities(),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: URNArtifactFileType,
TypePayload: protox.MustEncode(&pipepb.ArtifactFilePayload{}),
RoleUrn: URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build(), nil
+ }, nil
}
// TODO(https://github.com/apache/beam/issues/23893): Along with scoped resource hints,
@@ -195,10 +195,10 @@ func Marshal(edges []*graph.MultiEdge, opt *Options) (*pipepb.Pipeline, error) {
}
}
- p := pipepb.Pipeline_builder{
+ p := &pipepb.Pipeline{
Components: m.build(),
Requirements: m.getRequirements(),
- }.Build()
+ }
p, err := pipelinex.Normalize(p)
if err != nil {
@@ -248,13 +248,13 @@ func newMarshaller(opt *Options) *marshaller {
}
func (m *marshaller) build() *pipepb.Components {
- return pipepb.Components_builder{
+ return &pipepb.Components{
Transforms: m.transforms,
Pcollections: m.pcollections,
WindowingStrategies: m.windowing,
Environments: m.environments,
Coders: m.coders.Build(),
- }.Build()
+ }
}
func (m *marshaller) getRequirements() []string {
@@ -292,13 +292,13 @@ func (m *marshaller) addScopeTree(s *ScopeTree) (string, error) {
metadata := m.opt.GetContextReg().ExtractTransformMetadata(s.Scope.Scope.Context)
- transform := pipepb.PTransform_builder{
+ transform := &pipepb.PTransform{
UniqueName: s.Scope.Name,
Subtransforms: subtransforms,
EnvironmentId: m.addDefaultEnv(),
Annotations: metadata.Annotations,
// DisplayData: metadata.DisplayData,
- }.Build()
+ }
if err := m.updateIfCombineComposite(s, transform); err != nil {
return "", errors.Wrapf(err, "failed to add scope tree: %v", s)
@@ -330,14 +330,14 @@ func (m *marshaller) updateIfCombineComposite(s *ScopeTree, transform *pipepb.PT
if err != nil {
return errors.Wrapf(err, "failed to update PTransform spec: %v", transform)
}
- payload := pipepb.CombinePayload_builder{
- CombineFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.CombinePayload{
+ CombineFn: &pipepb.FunctionSpec{
Urn: URNDoFn,
Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
+ },
AccumulatorCoderId: acID,
- }.Build()
- transform.SetSpec(pipepb.FunctionSpec_builder{Urn: URNCombinePerKey, Payload: protox.MustEncode(payload)}.Build())
+ }
+ transform.Spec = &pipepb.FunctionSpec{Urn: URNCombinePerKey, Payload: protox.MustEncode(payload)}
return nil
}
@@ -414,7 +414,7 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
var spec *pipepb.FunctionSpec
switch edge.Edge.Op {
case graph.Impulse:
- spec = pipepb.FunctionSpec_builder{Urn: URNImpulse}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNImpulse}
case graph.ParDo:
si := make(map[string]*pipepb.SideInput)
@@ -432,18 +432,18 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
return nil, err
}
- si[fmt.Sprintf("i%v", i)] = pipepb.SideInput_builder{
- AccessPattern: pipepb.FunctionSpec_builder{
+ si[fmt.Sprintf("i%v", i)] = &pipepb.SideInput{
+ AccessPattern: &pipepb.FunctionSpec{
Urn: URNIterableSideInput,
- }.Build(),
- ViewFn: pipepb.FunctionSpec_builder{
+ },
+ ViewFn: &pipepb.FunctionSpec{
Urn: "foo",
- }.Build(),
- WindowMappingFn: pipepb.FunctionSpec_builder{
+ },
+ WindowMappingFn: &pipepb.FunctionSpec{
Urn: mappingUrn,
- Payload: siWSpec.GetPayload(),
- }.Build(),
- }.Build()
+ Payload: siWSpec.Payload,
+ },
+ }
case graph.Map, graph.MultiMap:
// Already in a MultiMap form, don't need to add a fixed key.
@@ -456,18 +456,18 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
return nil, err
}
- si[fmt.Sprintf("i%v", i)] = pipepb.SideInput_builder{
- AccessPattern: pipepb.FunctionSpec_builder{
+ si[fmt.Sprintf("i%v", i)] = &pipepb.SideInput{
+ AccessPattern: &pipepb.FunctionSpec{
Urn: URNMultimapSideInput,
- }.Build(),
- ViewFn: pipepb.FunctionSpec_builder{
+ },
+ ViewFn: &pipepb.FunctionSpec{
Urn: "foo",
- }.Build(),
- WindowMappingFn: pipepb.FunctionSpec_builder{
+ },
+ WindowMappingFn: &pipepb.FunctionSpec{
Urn: mappingUrn,
- Payload: siWSpec.GetPayload(),
- }.Build(),
- }.Build()
+ Payload: siWSpec.Payload,
+ },
+ }
default:
return nil, errors.Errorf("unexpected input kind: %v", edge)
}
@@ -478,23 +478,23 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
return handleErr(err)
}
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNDoFn,
Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
+ },
SideInputs: si,
- }.Build()
+ }
if edge.Edge.DoFn.IsSplittable() {
coderID, err := m.coders.Add(edge.Edge.RestrictionCoder)
if err != nil {
return handleErr(err)
}
- payload.SetRestrictionCoderId(coderID)
+ payload.RestrictionCoderId = coderID
m.requirements[URNRequiresSplittableDoFn] = true
}
if _, ok := edge.Edge.DoFn.ProcessElementFn().BundleFinalization(); ok {
- payload.SetRequestsFinalization(true)
+ payload.RequestsFinalization = true
m.requirements[URNRequiresBundleFinalization] = true
}
if _, ok := edge.Edge.DoFn.ProcessElementFn().StateProvider(); ok {
@@ -520,23 +520,27 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
}
switch ps.StateType() {
case state.TypeValue:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- ReadModifyWriteSpec: pipepb.ReadModifyWriteStateSpec_builder{
- CoderId: coderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_ReadModifyWriteSpec{
+ ReadModifyWriteSpec: &pipepb.ReadModifyWriteStateSpec{
+ CoderId: coderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNBagUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeBag:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- BagSpec: pipepb.BagStateSpec_builder{
- ElementCoderId: coderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_BagSpec{
+ BagSpec: &pipepb.BagStateSpec{
+ ElementCoderId: coderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNBagUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeCombining:
cps := ps.(state.CombiningPipelineState).GetCombineFn()
f, err := graph.NewFn(cps)
@@ -555,42 +559,48 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
if err != nil {
return handleErr(err)
}
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- CombiningSpec: pipepb.CombiningStateSpec_builder{
- AccumulatorCoderId: coderID,
- CombineFn: pipepb.FunctionSpec_builder{
- Urn: "beam:combinefn:gosdk:v1",
- Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_CombiningSpec{
+ CombiningSpec: &pipepb.CombiningStateSpec{
+ AccumulatorCoderId: coderID,
+ CombineFn: &pipepb.FunctionSpec{
+ Urn: "beam:combinefn:gosdk:v1",
+ Payload: []byte(mustEncodeMultiEdge),
+ },
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNBagUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeMap:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- MapSpec: pipepb.MapStateSpec_builder{
- KeyCoderId: keyCoderID,
- ValueCoderId: coderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_MapSpec{
+ MapSpec: &pipepb.MapStateSpec{
+ KeyCoderId: keyCoderID,
+ ValueCoderId: coderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNMultiMapUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeSet:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- SetSpec: pipepb.SetStateSpec_builder{
- ElementCoderId: keyCoderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_SetSpec{
+ SetSpec: &pipepb.SetStateSpec{
+ ElementCoderId: keyCoderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNMultiMapUserState,
- }.Build(),
- }.Build()
+ },
+ }
default:
return nil, errors.Errorf("State type %v not recognized for state %v", ps.StateKey(), ps)
}
}
- payload.SetStateSpecs(stateSpecs)
+ payload.StateSpecs = stateSpecs
}
if _, ok := edge.Edge.DoFn.ProcessElementFn().TimerProvider(); ok {
m.requirements[URNRequiresStatefulProcessing] = true
@@ -600,7 +610,7 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
// All timers for a single DoFn have the same key and window coders, that match the input PCollection.
mainInputID := inputs["i0"]
pCol := m.pcollections[mainInputID]
- kvCoder := m.coders.coders[pCol.GetCoderId()]
+ kvCoder := m.coders.coders[pCol.CoderId]
if kvCoder.GetSpec().GetUrn() != urnKVCoder {
return nil, errors.Errorf("timer using DoFn %v doesn't use a KV as PCollection input. Unable to extract key coder for timers, got %v", edge.Name, kvCoder.GetSpec().GetUrn())
}
@@ -614,15 +624,15 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
for _, pt := range pipelineTimers {
for timerFamilyID, timeDomain := range pt.Timers() {
- timerSpecs[timerFamilyID] = pipepb.TimerFamilySpec_builder{
+ timerSpecs[timerFamilyID] = &pipepb.TimerFamilySpec{
TimeDomain: pipepb.TimeDomain_Enum(timeDomain),
TimerFamilyCoderId: timerCoderID,
- }.Build()
+ }
}
}
- payload.SetTimerFamilySpecs(timerSpecs)
+ payload.TimerFamilySpecs = timerSpecs
}
- spec = pipepb.FunctionSpec_builder{Urn: URNParDo, Payload: protox.MustEncode(payload)}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNParDo, Payload: protox.MustEncode(payload)}
annotations = edge.Edge.DoFn.Annotations()
case graph.Combine:
@@ -630,33 +640,33 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
if err != nil {
return handleErr(err)
}
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNDoFn,
Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
- }.Build()
- spec = pipepb.FunctionSpec_builder{Urn: URNParDo, Payload: protox.MustEncode(payload)}.Build()
+ },
+ }
+ spec = &pipepb.FunctionSpec{Urn: URNParDo, Payload: protox.MustEncode(payload)}
case graph.Flatten:
- spec = pipepb.FunctionSpec_builder{Urn: URNFlatten}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNFlatten}
case graph.CoGBK:
- spec = pipepb.FunctionSpec_builder{Urn: URNGBK}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNGBK}
case graph.WindowInto:
windowFn, err := makeWindowFn(edge.Edge.WindowFn)
if err != nil {
return handleErr(err)
}
- payload := pipepb.WindowIntoPayload_builder{
+ payload := &pipepb.WindowIntoPayload{
WindowFn: windowFn,
- }.Build()
- spec = pipepb.FunctionSpec_builder{Urn: URNWindow, Payload: protox.MustEncode(payload)}.Build()
+ }
+ spec = &pipepb.FunctionSpec{Urn: URNWindow, Payload: protox.MustEncode(payload)}
case graph.External:
pyld := edge.Edge.Payload
- spec = pipepb.FunctionSpec_builder{Urn: pyld.URN, Payload: pyld.Data}.Build()
+ spec = &pipepb.FunctionSpec{Urn: pyld.URN, Payload: pyld.Data}
if len(pyld.InputsMap) != 0 {
if got, want := len(pyld.InputsMap), len(edge.Edge.Input); got != want {
@@ -690,18 +700,18 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
}
var transformEnvID = ""
- if !(spec.GetUrn() == URNGBK || spec.GetUrn() == URNImpulse) {
+ if !(spec.Urn == URNGBK || spec.Urn == URNImpulse) {
transformEnvID = m.addDefaultEnv()
}
- transform := pipepb.PTransform_builder{
+ transform := &pipepb.PTransform{
UniqueName: edge.Name,
Spec: spec,
Inputs: inputs,
Outputs: outputs,
EnvironmentId: transformEnvID,
Annotations: annotations,
- }.Build()
+ }
m.transforms[id] = transform
allPIds = append(allPIds, id)
return allPIds, nil
@@ -724,22 +734,22 @@ func (m *marshaller) expandCrossLanguage(namedEdge NamedEdge) (string, error) {
inputs[tag] = nodeID(n)
}
- spec := pipepb.FunctionSpec_builder{
+ spec := &pipepb.FunctionSpec{
Urn: edge.External.Urn,
Payload: edge.External.Payload,
- }.Build()
+ }
- transform := pipepb.PTransform_builder{
+ transform := &pipepb.PTransform{
UniqueName: namedEdge.Name,
Spec: spec,
Inputs: inputs,
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
// Add the coders for output in the marshaller even if expanded is nil
// for output coder field in expansion request.
// We need this specifically for Python External Transforms.
- names := strings.Split(spec.GetUrn(), ":")
+ names := strings.Split(spec.Urn, ":")
if len(names) > 2 && names[2] == "python" {
for _, out := range edge.Output {
id, err := m.coders.Add(out.To.Coder)
@@ -761,12 +771,12 @@ func (m *marshaller) expandCrossLanguage(namedEdge NamedEdge) (string, error) {
}
outputs[fmt.Sprintf("i%v", i)] = nodeID(out.To)
}
- transform.SetOutputs(outputs)
+ transform.Outputs = outputs
environment, err := ExpandedTransform(edge.External.Expanded)
if err != nil {
return "", errors.Wrapf(err, "failed to expand cross language transform for edge: %v", namedEdge)
}
- transform.SetEnvironmentId(environment.GetEnvironmentId())
+ transform.EnvironmentId = environment.EnvironmentId
}
m.transforms[id] = transform
@@ -815,25 +825,25 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
// Inject(i)
injectID := fmt.Sprintf("%v_inject%v", id, i)
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNInject,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNInject,
Inject: &v1pb.InjectPayload{N: (int32)(i)},
})),
- }.Build(),
- }.Build()
- inject := pipepb.PTransform_builder{
+ },
+ }
+ inject := &pipepb.PTransform{
UniqueName: injectID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(payload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": nodeID(in.From)},
Outputs: map[string]string{"i0": out},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[injectID] = inject
subtransforms = append(subtransforms, injectID)
@@ -850,13 +860,13 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
}
flattenID := fmt.Sprintf("%v_flatten", id)
- flatten := pipepb.PTransform_builder{
+ flatten := &pipepb.PTransform{
UniqueName: flattenID,
- Spec: pipepb.FunctionSpec_builder{Urn: URNFlatten}.Build(),
+ Spec: &pipepb.FunctionSpec{Urn: URNFlatten},
Inputs: inputs,
Outputs: map[string]string{"i0": out},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[flattenID] = flatten
subtransforms = append(subtransforms, flattenID)
@@ -868,12 +878,12 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
}
gbkID := fmt.Sprintf("%v_gbk", id)
- gbk := pipepb.PTransform_builder{
+ gbk := &pipepb.PTransform{
UniqueName: gbkID,
- Spec: pipepb.FunctionSpec_builder{Urn: URNGBK}.Build(),
+ Spec: &pipepb.FunctionSpec{Urn: URNGBK},
Inputs: map[string]string{"i0": out},
Outputs: map[string]string{"i0": gbkOut},
- }.Build()
+ }
m.transforms[gbkID] = gbk
subtransforms = append(subtransforms, gbkID)
@@ -884,35 +894,35 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
}
expandID := fmt.Sprintf("%v_expand", id)
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNExpand,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNExpand,
})),
- }.Build(),
- }.Build()
- expand := pipepb.PTransform_builder{
+ },
+ }
+ expand := &pipepb.PTransform{
UniqueName: expandID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(payload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": gbkOut},
Outputs: map[string]string{"i0": nodeID(outNode)},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[id] = expand
subtransforms = append(subtransforms, id)
// Add composite for visualization
cogbkID := fmt.Sprintf("%v_cogbk", id)
- m.transforms[cogbkID] = pipepb.PTransform_builder{
+ m.transforms[cogbkID] = &pipepb.PTransform{
UniqueName: edge.Name,
Subtransforms: subtransforms,
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
return cogbkID, nil
}
@@ -992,14 +1002,16 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
if err != nil {
return handleErr(err)
}
- m.pcollections[postReify].SetWindowingStrategyId(
- m.internWindowingStrategy(pipepb.WindowingStrategy_builder{
+ m.pcollections[postReify].WindowingStrategyId =
+ m.internWindowingStrategy(&pipepb.WindowingStrategy{
// Not segregated by time...
WindowFn: windowFn,
// ...output after every element is received...
- Trigger: pipepb.Trigger_builder{
- Always: &pipepb.Trigger_Always{},
- }.Build(),
+ Trigger: &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Always_{
+ Always: &pipepb.Trigger_Always{},
+ },
+ },
// ...and after outputing, discard the output elements...
AccumulationMode: pipepb.AccumulationMode_DISCARDING,
// ...and since every pane should have 1 element,
@@ -1013,14 +1025,14 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
ClosingBehavior: pipepb.ClosingBehavior_EMIT_IF_NONEMPTY,
AllowedLateness: int64(in.From.WindowingStrategy().AllowedLateness),
OnTimeBehavior: pipepb.OnTimeBehavior_FIRE_ALWAYS,
- }.Build()))
+ })
}
// Inputs (i)
inputID := fmt.Sprintf("%v_reifyts", id)
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNReshuffleInput,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNReshuffleInput,
@@ -1029,18 +1041,18 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
CoderPayloads: coderPayloads,
},
})),
- }.Build(),
- }.Build()
- input := pipepb.PTransform_builder{
+ },
+ }
+ input := &pipepb.PTransform{
UniqueName: inputID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(payload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": nodeID(in.From)},
Outputs: map[string]string{"i0": postReify},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[inputID] = input
subtransforms = append(subtransforms, inputID)
@@ -1054,12 +1066,12 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
}
gbkID := fmt.Sprintf("%v_gbk", id)
- gbk := pipepb.PTransform_builder{
+ gbk := &pipepb.PTransform{
UniqueName: gbkID,
- Spec: pipepb.FunctionSpec_builder{Urn: URNGBK}.Build(),
+ Spec: &pipepb.FunctionSpec{Urn: URNGBK},
Inputs: map[string]string{"i0": postReify},
Outputs: map[string]string{"i0": gbkOut},
- }.Build()
+ }
m.transforms[gbkID] = gbk
subtransforms = append(subtransforms, gbkID)
@@ -1069,11 +1081,11 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
if err != nil {
return handleErr(err)
}
- m.pcollections[outPCol].SetWindowingStrategyId(preservedWSID)
+ m.pcollections[outPCol].WindowingStrategyId = preservedWSID
outputID := fmt.Sprintf("%v_unreify", id)
- outputPayload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ outputPayload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNReshuffleOutput,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNReshuffleOutput,
@@ -1082,31 +1094,31 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
CoderPayloads: coderPayloads,
},
})),
- }.Build(),
- }.Build()
- output := pipepb.PTransform_builder{
+ },
+ }
+ output := &pipepb.PTransform{
UniqueName: outputID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(outputPayload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": gbkOut},
Outputs: map[string]string{"i0": nodeID(outNode)},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[id] = output
subtransforms = append(subtransforms, id)
// Add composite for visualization, or runner optimization
reshuffleID := fmt.Sprintf("%v_reshuffle", id)
- m.transforms[reshuffleID] = pipepb.PTransform_builder{
+ m.transforms[reshuffleID] = &pipepb.PTransform{
UniqueName: edge.Name,
Subtransforms: subtransforms,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNReshuffle,
- }.Build(),
+ },
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
return reshuffleID, nil
}
@@ -1152,12 +1164,12 @@ func (m *marshaller) makeNode(id, cid string, n *graph.Node) (string, error) {
return "", errors.Wrapf(err, "failed to make node %v with node id %v", n, id)
}
- col := pipepb.PCollection_builder{
+ col := &pipepb.PCollection{
UniqueName: id,
CoderId: cid,
IsBounded: boolToBounded(n.Bounded()),
WindowingStrategyId: windowingStrategyId,
- }.Build()
+ }
m.pcollections[id] = col
return id, nil
}
@@ -1182,7 +1194,7 @@ func (m *marshaller) addDefaultEnv() string {
// Add the pipeline level resource hints here for now.
// TODO(https://github.com/apache/beam/issues/23893) move to a better place for
// scoped hints in next pass, which affect number of environments set by Go pipelines.
- env.SetResourceHints(m.opt.PipelineResourceHints.Payloads())
+ env.ResourceHints = m.opt.PipelineResourceHints.Payloads()
m.environments[defaultEnvId] = env
}
return defaultEnvId
@@ -1193,7 +1205,7 @@ func (m *marshaller) addWindowingStrategy(w *window.WindowingStrategy) (string,
if err != nil {
return "", errors.Wrapf(err, "failed to add window strategy %v", w)
}
- ws.SetEnvironmentId(m.addDefaultEnv())
+ ws.EnvironmentId = m.addDefaultEnv()
return m.internWindowingStrategy(ws), nil
}
@@ -1231,7 +1243,7 @@ func MarshalWindowingStrategy(c *CoderMarshaller, w *window.WindowingStrategy) (
mergeStat = pipepb.MergeStatus_NON_MERGING
}
- ws := pipepb.WindowingStrategy_builder{
+ ws := &pipepb.WindowingStrategy{
WindowFn: windowFn,
MergeStatus: mergeStat,
WindowCoderId: windowCoderID,
@@ -1241,7 +1253,7 @@ func MarshalWindowingStrategy(c *CoderMarshaller, w *window.WindowingStrategy) (
ClosingBehavior: pipepb.ClosingBehavior_EMIT_IF_NONEMPTY,
AllowedLateness: int64(w.AllowedLateness),
OnTimeBehavior: pipepb.OnTimeBehavior_FIRE_IF_NONEMPTY,
- }.Build()
+ }
return ws, nil
}
@@ -1263,25 +1275,33 @@ func makeAccumulationMode(m window.AccumulationMode) pipepb.AccumulationMode_Enu
func makeTrigger(t trigger.Trigger) *pipepb.Trigger {
switch t := t.(type) {
case *trigger.DefaultTrigger:
- return pipepb.Trigger_builder{
- Default: &pipepb.Trigger_Default{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Default_{
+ Default: &pipepb.Trigger_Default{},
+ },
+ }
case *trigger.AlwaysTrigger:
- return pipepb.Trigger_builder{
- Always: &pipepb.Trigger_Always{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Always_{
+ Always: &pipepb.Trigger_Always{},
+ },
+ }
case *trigger.AfterAnyTrigger:
- return pipepb.Trigger_builder{
- AfterAny: pipepb.Trigger_AfterAny_builder{
- Subtriggers: extractSubtriggers(t.SubTriggers()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterAny_{
+ AfterAny: &pipepb.Trigger_AfterAny{
+ Subtriggers: extractSubtriggers(t.SubTriggers()),
+ },
+ },
+ }
case *trigger.AfterAllTrigger:
- return pipepb.Trigger_builder{
- AfterAll: pipepb.Trigger_AfterAll_builder{
- Subtriggers: extractSubtriggers(t.SubTriggers()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterAll_{
+ AfterAll: &pipepb.Trigger_AfterAll{
+ Subtriggers: extractSubtriggers(t.SubTriggers()),
+ },
+ },
+ }
case *trigger.AfterProcessingTimeTrigger:
if len(t.TimestampTransforms()) == 0 {
panic("AfterProcessingTime trigger set without a delay or alignment.")
@@ -1291,66 +1311,88 @@ func makeTrigger(t trigger.Trigger) *pipepb.Trigger {
var ttp *pipepb.TimestampTransform
switch tt := tt.(type) {
case trigger.DelayTransform:
- ttp = pipepb.TimestampTransform_builder{
- Delay: pipepb.TimestampTransform_Delay_builder{DelayMillis: tt.Delay}.Build()}.Build()
+ ttp = &pipepb.TimestampTransform{
+ TimestampTransform: &pipepb.TimestampTransform_Delay_{
+ Delay: &pipepb.TimestampTransform_Delay{DelayMillis: tt.Delay},
+ }}
case trigger.AlignToTransform:
- ttp = pipepb.TimestampTransform_builder{
- AlignTo: pipepb.TimestampTransform_AlignTo_builder{
- Period: tt.Period,
- Offset: tt.Offset,
- }.Build()}.Build()
+ ttp = &pipepb.TimestampTransform{
+ TimestampTransform: &pipepb.TimestampTransform_AlignTo_{
+ AlignTo: &pipepb.TimestampTransform_AlignTo{
+ Period: tt.Period,
+ Offset: tt.Offset,
+ },
+ }}
}
tts = append(tts, ttp)
}
- return pipepb.Trigger_builder{
- AfterProcessingTime: pipepb.Trigger_AfterProcessingTime_builder{
- TimestampTransforms: tts,
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterProcessingTime_{
+ AfterProcessingTime: &pipepb.Trigger_AfterProcessingTime{
+ TimestampTransforms: tts,
+ },
+ },
+ }
case *trigger.AfterCountTrigger:
- return pipepb.Trigger_builder{
- ElementCount: pipepb.Trigger_ElementCount_builder{ElementCount: t.ElementCount()}.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_ElementCount_{
+ ElementCount: &pipepb.Trigger_ElementCount{ElementCount: t.ElementCount()},
+ },
+ }
case *trigger.AfterEndOfWindowTrigger:
var lateTrigger *pipepb.Trigger
if t.Late() != nil {
lateTrigger = makeTrigger(t.Late())
}
- return pipepb.Trigger_builder{
- AfterEndOfWindow: pipepb.Trigger_AfterEndOfWindow_builder{
- EarlyFirings: makeTrigger(t.Early()),
- LateFirings: lateTrigger,
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterEndOfWindow_{
+ AfterEndOfWindow: &pipepb.Trigger_AfterEndOfWindow{
+ EarlyFirings: makeTrigger(t.Early()),
+ LateFirings: lateTrigger,
+ },
+ },
+ }
case *trigger.RepeatTrigger:
- return pipepb.Trigger_builder{
- Repeat: pipepb.Trigger_Repeat_builder{Subtrigger: makeTrigger(t.SubTrigger())}.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Repeat_{
+ Repeat: &pipepb.Trigger_Repeat{Subtrigger: makeTrigger(t.SubTrigger())},
+ },
+ }
case *trigger.NeverTrigger:
- return pipepb.Trigger_builder{
- Never: &pipepb.Trigger_Never{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Never_{
+ Never: &pipepb.Trigger_Never{},
+ },
+ }
case *trigger.AfterSynchronizedProcessingTimeTrigger:
- return pipepb.Trigger_builder{
- AfterSynchronizedProcessingTime: &pipepb.Trigger_AfterSynchronizedProcessingTime{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterSynchronizedProcessingTime_{
+ AfterSynchronizedProcessingTime: &pipepb.Trigger_AfterSynchronizedProcessingTime{},
+ },
+ }
case *trigger.OrFinallyTrigger:
- return pipepb.Trigger_builder{
- OrFinally: pipepb.Trigger_OrFinally_builder{
- Main: makeTrigger(t.Main()),
- Finally: makeTrigger(t.Finally()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_OrFinally_{
+ OrFinally: &pipepb.Trigger_OrFinally{
+ Main: makeTrigger(t.Main()),
+ Finally: makeTrigger(t.Finally()),
+ },
+ },
+ }
case *trigger.AfterEachTrigger:
- return pipepb.Trigger_builder{
- AfterEach: pipepb.Trigger_AfterEach_builder{
- Subtriggers: extractSubtriggers(t.Subtriggers()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterEach_{
+ AfterEach: &pipepb.Trigger_AfterEach{
+ Subtriggers: extractSubtriggers(t.Subtriggers()),
+ },
+ },
+ }
default:
- return pipepb.Trigger_builder{
- Default: &pipepb.Trigger_Default{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Default_{
+ Default: &pipepb.Trigger_Default{},
+ },
+ }
}
}
@@ -1369,37 +1411,37 @@ func extractSubtriggers(t []trigger.Trigger) []*pipepb.Trigger {
func makeWindowFn(w *window.Fn) (*pipepb.FunctionSpec, error) {
switch w.Kind {
case window.GlobalWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNGlobalWindowsWindowFn,
- }.Build(), nil
+ }, nil
case window.FixedWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNFixedWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.FixedWindowsPayload_builder{
+ &pipepb.FixedWindowsPayload{
Size: durationpb.New(w.Size),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.SlidingWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNSlidingWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.SlidingWindowsPayload_builder{
+ &pipepb.SlidingWindowsPayload{
Size: durationpb.New(w.Size),
Period: durationpb.New(w.Period),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.Sessions:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNSessionsWindowFn,
Payload: protox.MustEncode(
- pipepb.SessionWindowsPayload_builder{
+ &pipepb.SessionWindowsPayload{
GapSize: durationpb.New(w.Gap),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
default:
return nil, errors.Errorf("unexpected windowing strategy: %v", w)
}
@@ -1449,11 +1491,11 @@ func UpdateDefaultEnvWorkerType(typeUrn string, pyld []byte, p *pipepb.Pipeline)
return errors.Errorf("unable to find default Go environment with ID %q", defaultEnvId)
}
for _, dep := range env.GetDependencies() {
- if dep.GetRoleUrn() != URNArtifactGoWorkerRole {
+ if dep.RoleUrn != URNArtifactGoWorkerRole {
continue
}
- dep.SetTypeUrn(typeUrn)
- dep.SetTypePayload(pyld)
+ dep.TypeUrn = typeUrn
+ dep.TypePayload = pyld
return nil
}
return errors.Errorf("unable to find dependency with %q role in environment with ID %q,", URNArtifactGoWorkerRole, defaultEnvId)
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go b/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go
index 2f52818200d..e18a5f97796 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go
@@ -170,12 +170,12 @@ func TestMarshal(t *testing.T) {
t.Fatalf("got %v edges, want %v", got, want)
}
- payload, err := proto.Marshal(pipepb.DockerPayload_builder{ContainerImage: "foo"}.Build())
+ payload, err := proto.Marshal(&pipepb.DockerPayload{ContainerImage: "foo"})
if err != nil {
t.Fatal(err)
}
p, err := graphx.Marshal(edges,
- &graphx.Options{Environment: pipepb.Environment_builder{Urn: "beam:env:docker:v1", Payload: payload}.Build()})
+ &graphx.Options{Environment: &pipepb.Environment{Urn: "beam:env:docker:v1", Payload: payload}})
if err != nil {
t.Fatal(err)
}
@@ -236,12 +236,12 @@ func TestMarshal_PTransformAnnotations(t *testing.T) {
t.Fatal(err)
}
- payload, err := proto.Marshal(pipepb.DockerPayload_builder{ContainerImage: "foo"}.Build())
+ payload, err := proto.Marshal(&pipepb.DockerPayload{ContainerImage: "foo"})
if err != nil {
t.Fatal(err)
}
p, err := graphx.Marshal(edges,
- &graphx.Options{Environment: pipepb.Environment_builder{Urn: "beam:env:docker:v1", Payload: payload}.Build(), ContextReg: &creg})
+ &graphx.Options{Environment: &pipepb.Environment{Urn: "beam:env:docker:v1", Payload: payload}, ContextReg: &creg})
if err != nil {
t.Fatal(err)
}
@@ -317,19 +317,19 @@ func TestCreateEnvironment(t *testing.T) {
name: "external",
urn: graphx.URNEnvExternal,
payload: func(name string) []byte {
- return protox.MustEncode(pipepb.ExternalPayload_builder{
- Endpoint: pipepb.ApiServiceDescriptor_builder{
+ return protox.MustEncode(&pipepb.ExternalPayload{
+ Endpoint: &pipepb.ApiServiceDescriptor{
Url: name,
- }.Build(),
- }.Build())
+ },
+ })
},
}, {
name: "docker",
urn: graphx.URNEnvDocker,
payload: func(name string) []byte {
- return protox.MustEncode(pipepb.DockerPayload_builder{
+ return protox.MustEncode(&pipepb.DockerPayload{
ContainerImage: name,
- }.Build())
+ })
},
},
}
@@ -340,17 +340,17 @@ func TestCreateEnvironment(t *testing.T) {
if err != nil {
t.Errorf("CreateEnvironment(%v) = %v error, want nil", test.urn, err)
}
- want := pipepb.Environment_builder{
+ want := &pipepb.Environment{
Urn: test.urn,
Payload: test.payload(test.name),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: graphx.URNArtifactFileType,
TypePayload: protox.MustEncode(&pipepb.ArtifactFilePayload{}),
RoleUrn: graphx.URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build()
+ }
opts := []cmp.Option{
protocmp.Transform(),
// Ignore the capabilities field, since we can't access that method here.
@@ -365,87 +365,87 @@ func TestCreateEnvironment(t *testing.T) {
func TestUpdateDefaultEnvWorkerType(t *testing.T) {
t.Run("noEnvs", func(t *testing.T) {
- if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, pipepb.Pipeline_builder{
+ if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, &pipepb.Pipeline{
Components: &pipepb.Components{},
- }.Build()); err == nil {
+ }); err == nil {
t.Error("UpdateDefaultEnvWorkerType(<emptyEnv>) no error, want err")
}
})
t.Run("noGoEnvs", func(t *testing.T) {
- if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: map[string]*pipepb.Environment{
- "java": pipepb.Environment_builder{Urn: "java"}.Build(),
- "python": pipepb.Environment_builder{Urn: "python"}.Build(),
- "typescript": pipepb.Environment_builder{Urn: "typescript"}.Build(),
+ "java": {Urn: "java"},
+ "python": {Urn: "python"},
+ "typescript": {Urn: "typescript"},
},
- }.Build(),
- }.Build()); err == nil {
+ },
+ }); err == nil {
t.Error("UpdateDefaultEnvWorkerType(<noGoEnvs>) no error, want err")
}
})
t.Run("badGoEnv", func(t *testing.T) {
- if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: map[string]*pipepb.Environment{
- "java": pipepb.Environment_builder{Urn: "java"}.Build(),
- "python": pipepb.Environment_builder{Urn: "python"}.Build(),
- "typescript": pipepb.Environment_builder{Urn: "typescript"}.Build(),
- "go": pipepb.Environment_builder{
+ "java": {Urn: "java"},
+ "python": {Urn: "python"},
+ "typescript": {Urn: "typescript"},
+ "go": {
Urn: "test",
Payload: []byte("test"),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
RoleUrn: "unset",
- }.Build(),
+ },
},
- }.Build(),
+ },
},
- }.Build(),
- }.Build()); err == nil {
+ },
+ }); err == nil {
t.Error("UpdateDefaultEnvWorkerType(<badGoEnv>) no error, want err")
}
})
t.Run("goEnv", func(t *testing.T) {
wantUrn := graphx.URNArtifactFileType
- wantPyld := protox.MustEncode(pipepb.ArtifactFilePayload_builder{
+ wantPyld := protox.MustEncode(&pipepb.ArtifactFilePayload{
Path: "good",
- }.Build())
- p := pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ })
+ p := &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: map[string]*pipepb.Environment{
- "java": pipepb.Environment_builder{Urn: "java"}.Build(),
- "python": pipepb.Environment_builder{Urn: "python"}.Build(),
- "typescript": pipepb.Environment_builder{Urn: "typescript"}.Build(),
- "go": pipepb.Environment_builder{
+ "java": {Urn: "java"},
+ "python": {Urn: "python"},
+ "typescript": {Urn: "typescript"},
+ "go": {
Urn: "test",
Payload: []byte("test"),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: "to be removed",
TypePayload: nil,
RoleUrn: graphx.URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build(),
+ },
},
- }.Build(),
- }.Build()
+ },
+ }
if err := graphx.UpdateDefaultEnvWorkerType(wantUrn, wantPyld, p); err != nil {
t.Errorf("UpdateDefaultEnvWorkerType(<goEnv>) = %v, want nil", err)
}
got := p.GetComponents().GetEnvironments()["go"]
- want := pipepb.Environment_builder{
+ want := &pipepb.Environment{
Urn: "test",
Payload: []byte("test"),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: wantUrn,
TypePayload: wantPyld,
RoleUrn: graphx.URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build()
+ }
opts := []cmp.Option{
protocmp.Transform(),
}
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/xlang.go b/sdks/go/pkg/beam/core/runtime/graphx/xlang.go
index b02af765c37..baee3c00093 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/xlang.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/xlang.go
@@ -34,7 +34,7 @@ func mergeExpandedWithPipeline(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
exp := e.External.Expanded
id := fmt.Sprintf("e%v", e.ID())
- p.SetRequirements(append(p.GetRequirements(), exp.Requirements...))
+ p.Requirements = append(p.Requirements, exp.Requirements...)
// Adding components of the Expanded Transforms to the current Pipeline
components, err := ExpandedComponents(exp)
@@ -42,16 +42,16 @@ func mergeExpandedWithPipeline(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
panic(err)
}
for k, v := range components.GetTransforms() {
- p.GetComponents().GetTransforms()[k] = v
+ p.Components.Transforms[k] = v
}
for k, v := range components.GetPcollections() {
- p.GetComponents().GetPcollections()[k] = v
+ p.Components.Pcollections[k] = v
}
for k, v := range components.GetWindowingStrategies() {
- p.GetComponents().GetWindowingStrategies()[k] = v
+ p.Components.WindowingStrategies[k] = v
}
for k, v := range components.GetCoders() {
- p.GetComponents().GetCoders()[k] = v
+ p.Components.Coders[k] = v
}
for k, v := range components.GetEnvironments() {
if k == defaultEnvId {
@@ -61,14 +61,14 @@ func mergeExpandedWithPipeline(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
// using unique namespace prevents collision.
continue
}
- p.GetComponents().GetEnvironments()[k] = v
+ p.Components.Environments[k] = v
}
transform, err := ExpandedTransform(exp)
if err != nil {
panic(err)
}
- p.GetComponents().GetTransforms()[id] = transform
+ p.Components.Transforms[id] = transform
}
}
@@ -102,7 +102,7 @@ func purgeOutputInput(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
}
idxMap[nodeID] = pcolID
- delete(components.GetPcollections(), nodeID)
+ delete(components.Pcollections, nodeID)
}
}
@@ -200,13 +200,13 @@ func AddFakeImpulses(p *pipepb.Pipeline) {
key := fmt.Sprintf("%s_%s", "impulse", tag)
output := map[string]string{"out": id}
- impulse := pipepb.PTransform_builder{
+ impulse := &pipepb.PTransform{
UniqueName: key,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNImpulse,
- }.Build(),
+ },
Outputs: output,
- }.Build()
+ }
transforms[key] = impulse
}
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go b/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go
index 283e49e691c..7b68a770768 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go
@@ -129,17 +129,17 @@ func TestExternalOutputs(t *testing.T) {
}
func newTransform(name string) *pipepb.PTransform {
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
UniqueName: name,
- }.Build()
+ }
}
func newComponents(ts []string) *pipepb.Components {
components := &pipepb.Components{}
- components.SetTransforms(make(map[string]*pipepb.PTransform))
+ components.Transforms = make(map[string]*pipepb.PTransform)
for id, t := range ts {
- components.GetTransforms()[fmt.Sprint(id)] = newTransform(t)
+ components.Transforms[fmt.Sprint(id)] = newTransform(t)
}
return components
@@ -205,23 +205,23 @@ var testExternal = graph.ExternalTransform{
Expanded: &testExpanded,
}
-var testComponents = pipepb.Components_builder{
- Transforms: map[string]*pipepb.PTransform{"transform_id1": pipepb.PTransform_builder{UniqueName: "test_components_transform"}.Build()},
- Pcollections: map[string]*pipepb.PCollection{"pcollection_id1": pipepb.PCollection_builder{UniqueName: "test_components_pcollection"}.Build()},
- WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id1": pipepb.WindowingStrategy_builder{WindowCoderId: "test_components_windowing"}.Build()},
- Coders: map[string]*pipepb.Coder{"coder_id1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "test_components_coder"}.Build()}.Build()},
- Environments: map[string]*pipepb.Environment{"environment_id1": pipepb.Environment_builder{Urn: "test_components_environment"}.Build()},
-}.Build()
+var testComponents = pipepb.Components{
+ Transforms: map[string]*pipepb.PTransform{"transform_id1": {UniqueName: "test_components_transform"}},
+ Pcollections: map[string]*pipepb.PCollection{"pcollection_id1": {UniqueName: "test_components_pcollection"}},
+ WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id1": {WindowCoderId: "test_components_windowing"}},
+ Coders: map[string]*pipepb.Coder{"coder_id1": {Spec: &pipepb.FunctionSpec{Urn: "test_components_coder"}}},
+ Environments: map[string]*pipepb.Environment{"environment_id1": {Urn: "test_components_environment"}},
+}
var testRequirements = []string{"test_requirement1", "test_requirement2"}
-var testTransform = pipepb.PTransform_builder{
+var testTransform = pipepb.PTransform{
UniqueName: "test_transform",
-}.Build()
+}
var testExpanded = graph.ExpandedTransform{
- Components: testComponents,
- Transform: testTransform,
+ Components: &testComponents,
+ Transform: &testTransform,
Requirements: testRequirements,
}
@@ -232,23 +232,23 @@ var testExternal2 = graph.ExternalTransform{
Expanded: &testExpanded2,
}
-var testComponents2 = pipepb.Components_builder{
- Transforms: map[string]*pipepb.PTransform{"transform_id2": pipepb.PTransform_builder{UniqueName: "test_components2_transform"}.Build()},
- Pcollections: map[string]*pipepb.PCollection{"pcollection_id2": pipepb.PCollection_builder{UniqueName: "test_components2_pcollection"}.Build()},
- WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id2": pipepb.WindowingStrategy_builder{WindowCoderId: "test_components2_windowing"}.Build()},
- Coders: map[string]*pipepb.Coder{"coder_id2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "test_components2_coder"}.Build()}.Build()},
- Environments: map[string]*pipepb.Environment{"environment_id2": pipepb.Environment_builder{Urn: "test_components2_environment"}.Build()},
-}.Build()
+var testComponents2 = pipepb.Components{
+ Transforms: map[string]*pipepb.PTransform{"transform_id2": {UniqueName: "test_components2_transform"}},
+ Pcollections: map[string]*pipepb.PCollection{"pcollection_id2": {UniqueName: "test_components2_pcollection"}},
+ WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id2": {WindowCoderId: "test_components2_windowing"}},
+ Coders: map[string]*pipepb.Coder{"coder_id2": {Spec: &pipepb.FunctionSpec{Urn: "test_components2_coder"}}},
+ Environments: map[string]*pipepb.Environment{"environment_id2": {Urn: "test_components2_environment"}},
+}
var testRequirements2 = []string{"test_requirement2", "test_requirement3"}
-var testTransform2 = pipepb.PTransform_builder{
+var testTransform2 = pipepb.PTransform{
UniqueName: "test_transform2",
-}.Build()
+}
var testExpanded2 = graph.ExpandedTransform{
- Components: testComponents2,
- Transform: testTransform2,
+ Components: &testComponents2,
+ Transform: &testTransform2,
Requirements: testRequirements2,
}
@@ -314,13 +314,13 @@ func TestMergeExpandedWithPipeline(t *testing.T) {
g := graph.New()
edges := test.makeEdges(g)
var p pipepb.Pipeline
- p.SetComponents(pipepb.Components_builder{
+ p.Components = &pipepb.Components{
Transforms: make(map[string]*pipepb.PTransform),
Pcollections: make(map[string]*pipepb.PCollection),
WindowingStrategies: make(map[string]*pipepb.WindowingStrategy),
Coders: make(map[string]*pipepb.Coder),
Environments: make(map[string]*pipepb.Environment),
- }.Build())
+ }
mergeExpandedWithPipeline(edges, &p)
// Check that all wanted expanded components have been added to
@@ -371,7 +371,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
// Transforms
for k, wantVal := range wantComps.GetTransforms() {
- gotVal, ok := gotComps.GetTransforms()[k]
+ gotVal, ok := gotComps.Transforms[k]
if !ok {
t.Errorf("Pipeline components missing expected transform with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -380,7 +380,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// PCollections
for k, wantVal := range wantComps.GetPcollections() {
- gotVal, ok := gotComps.GetPcollections()[k]
+ gotVal, ok := gotComps.Pcollections[k]
if !ok {
t.Errorf("Pipeline components missing expected PCollection with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -389,7 +389,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// Windowing Strategies
for k, wantVal := range wantComps.GetWindowingStrategies() {
- gotVal, ok := gotComps.GetWindowingStrategies()[k]
+ gotVal, ok := gotComps.WindowingStrategies[k]
if !ok {
t.Errorf("Pipeline components missing expected windowing strategy with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -398,7 +398,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// Coders
for k, wantVal := range wantComps.GetCoders() {
- gotVal, ok := gotComps.GetCoders()[k]
+ gotVal, ok := gotComps.Coders[k]
if !ok {
t.Errorf("Pipeline components missing expected coder with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -407,7 +407,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// Environments
for k, wantVal := range wantComps.GetEnvironments() {
- gotVal, ok := gotComps.GetEnvironments()[k]
+ gotVal, ok := gotComps.Environments[k]
if !ok {
t.Errorf("Pipeline components missing expected environment with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
index dc7733a3111..6385098d013 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
@@ -594,17 +594,17 @@ func (w *dataWriter) Close() error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
delete(w.ch.writers[w.id.instID], w.id.ptransformID)
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Data: []*fnpb.Elements_Data{
- fnpb.Elements_Data_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
// TODO(https://github.com/apache/beam/issues/21164): Set IsLast true on final flush instead of w/empty sentinel?
// Empty data == sentinel
IsLast: true,
- }.Build(),
+ },
},
- }.Build()
+ }
return w.send(msg)
}
@@ -617,15 +617,15 @@ func (w *dataWriter) Flush() error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Data: []*fnpb.Elements_Data{
- fnpb.Elements_Data_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
Data: w.buf,
- }.Build(),
+ },
},
- }.Build()
+ }
if l := len(w.buf); l > largeBufferNotificationThreshold {
log.Infof(context.TODO(), "dataWriter[%v;%v].Flush flushed large buffer of length %d", w.id, w.ch.id, l)
}
@@ -707,16 +707,16 @@ func (w *timerWriter) Close() error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
delete(w.ch.timerWriters[w.id.instID], timerKey{w.id.ptransformID, w.timerFamilyID})
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
- fnpb.Elements_Timers_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
TimerFamilyId: w.timerFamilyID,
IsLast: true,
- }.Build(),
+ },
},
- }.Build()
+ }
return w.send(msg)
}
@@ -724,16 +724,16 @@ func (w *timerWriter) writeTimers(p []byte) error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
- fnpb.Elements_Timers_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
TimerFamilyId: w.timerFamilyID,
Timers: p,
- }.Build(),
+ },
},
- }.Build()
+ }
return w.send(msg)
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
index dc197e00fd0..9f6f8a986a3 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
@@ -52,20 +52,20 @@ func (f *fakeDataClient) Recv() (*fnpb.Elements, error) {
f.calls++
data := []byte{1, 2, 3, 4, 1, 2, 3, 4}
- elemData := fnpb.Elements_Data_builder{
+ elemData := fnpb.Elements_Data{
InstructionId: "inst_ref",
Data: data,
TransformId: "ptr",
- }.Build()
+ }
if f.isLastCall == f.calls {
- elemData.SetIsLast(true)
+ elemData.IsLast = true
}
- msg := &fnpb.Elements{}
+ msg := fnpb.Elements{}
// Send extraData more than the number of elements buffered in the channel.
for i := 0; i < bufElements+extraData; i++ {
- msg.SetData(append(msg.GetData(), elemData))
+ msg.Data = append(msg.Data, &elemData)
}
// The first two calls fill up the buffer completely to stimulate the deadlock
@@ -73,16 +73,16 @@ func (f *fakeDataClient) Recv() (*fnpb.Elements, error) {
// Subsequent calls return no data.
switch f.calls {
case 1:
- return msg, f.err
+ return &msg, f.err
case 2:
- return msg, f.err
+ return &msg, f.err
case 3:
- elemData.SetData([]byte{})
- msg.SetData([]*fnpb.Elements_Data{elemData})
+ elemData.Data = []byte{}
+ msg.Data = []*fnpb.Elements_Data{&elemData}
// Broadcasting done here means that this code providing messages
// has not been blocked by the bug blocking the dataReader
// from getting more messages.
- return msg, f.err
+ return &msg, f.err
default:
f.done <- true
return nil, io.EOF
@@ -165,16 +165,16 @@ func TestElementChan(t *testing.T) {
}
timerElm := func(val byte, isLast bool) *fnpb.Elements_Timers {
- return fnpb.Elements_Timers_builder{InstructionId: instID, TransformId: timerID, Timers: []byte{val}, IsLast: isLast, TimerFamilyId: timerFamily}.Build()
+ return &fnpb.Elements_Timers{InstructionId: instID, TransformId: timerID, Timers: []byte{val}, IsLast: isLast, TimerFamilyId: timerFamily}
}
dataElm := func(val byte, isLast bool) *fnpb.Elements_Data {
- return fnpb.Elements_Data_builder{InstructionId: instID, TransformId: dataID, Data: []byte{val}, IsLast: isLast}.Build()
+ return &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{val}, IsLast: isLast}
}
noTimerElm := func() *fnpb.Elements_Timers {
- return fnpb.Elements_Timers_builder{InstructionId: instID, TransformId: timerID, Timers: []byte{}, IsLast: true}.Build()
+ return &fnpb.Elements_Timers{InstructionId: instID, TransformId: timerID, Timers: []byte{}, IsLast: true}
}
noDataElm := func() *fnpb.Elements_Data {
- return fnpb.Elements_Data_builder{InstructionId: instID, TransformId: dataID, Data: []byte{}, IsLast: true}.Build()
+ return &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{}, IsLast: true}
}
openChan := func(ctx context.Context, t *testing.T, c *DataChannel, timers ...string) <-chan exec.Elements {
t.Helper()
@@ -196,13 +196,13 @@ func TestElementChan(t *testing.T) {
name: "ReadThenData_singleRecv",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(1, false),
dataElm(2, false),
dataElm(3, true),
},
- }.Build())
+ })
return elms
},
wantSum: 6, wantCount: 3,
@@ -210,9 +210,9 @@ func TestElementChan(t *testing.T) {
name: "ReadThenData_multipleRecv",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(1, false)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(2, false)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(2, false)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
@@ -220,14 +220,14 @@ func TestElementChan(t *testing.T) {
name: "ReadThenNoData",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
return elms
},
wantSum: 0, wantCount: 0,
}, {
name: "NoDataThenRead",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
elms := openChan(ctx, t, c)
return elms
},
@@ -235,7 +235,7 @@ func TestElementChan(t *testing.T) {
}, {
name: "NoDataInstEndsThenRead",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
c.removeInstruction(instID)
elms := openChan(ctx, t, c)
return elms
@@ -245,18 +245,18 @@ func TestElementChan(t *testing.T) {
name: "ReadThenDataAndTimers",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(1, false)}}.Build())
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
}, {
name: "AllDataAndTimersThenRead",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(1, false)}}.Build())
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
elms := openChan(ctx, t, c, timerID)
return elms
},
@@ -264,30 +264,30 @@ func TestElementChan(t *testing.T) {
}, {
name: "DataThenReaderThenLast",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(1, false),
dataElm(2, false),
},
- }.Build())
+ })
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
}, {
name: "PartialTimersAllDataReadThenLastTimer",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
timerElm(1, false),
timerElm(2, false),
},
- }.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ })
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(3, true)}})
return elms
},
@@ -295,16 +295,16 @@ func TestElementChan(t *testing.T) {
}, {
name: "AllTimerThenReaderThenDataClose",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
timerElm(1, false),
timerElm(2, false),
timerElm(3, true),
},
- }.Build())
+ })
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
return elms
},
@@ -312,28 +312,28 @@ func TestElementChan(t *testing.T) {
}, {
name: "NoTimersThenReaderThenNoData",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{noTimerElm()}}.Build())
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{noTimerElm()}})
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
return elms
},
wantSum: 0, wantCount: 0,
}, {
name: "SomeTimersThenReaderThenAData",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)}}.Build())
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)}})
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
}, {
name: "SomeTimersAndADataThenReader",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)},
Data: []*fnpb.Elements_Data{dataElm(3, true)},
- }.Build())
+ })
elms := openChan(ctx, t, c, timerID)
return elms
},
@@ -341,12 +341,12 @@ func TestElementChan(t *testing.T) {
}, {
name: "PartialReadThenEndInstruction",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(1, false),
dataElm(2, false),
},
- }.Build())
+ })
elms := openChan(ctx, t, c)
var sum int
e := <-elms
@@ -362,12 +362,12 @@ func TestElementChan(t *testing.T) {
c.removeInstruction(instID)
// Instruction is ended, so further data for this instruction is ignored.
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(3, false),
dataElm(4, true),
},
- }.Build())
+ })
elms = openChan(ctx, t, c)
return elms
@@ -410,12 +410,12 @@ func BenchmarkElementChan(b *testing.B) {
if err != nil {
b.Errorf("Unexpected error from OpenElementChan(%v, %v, nil): %v", dataID, instID, err)
}
- e := fnpb.Elements_Data_builder{InstructionId: instID, TransformId: dataID, Data: []byte{1}, IsLast: false}.Build()
+ e := &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{1}, IsLast: false}
es := make([]*fnpb.Elements_Data, 0, bench.size)
for i := 0; i < bench.size; i++ {
es = append(es, e)
}
- batch := fnpb.Elements_builder{Data: es}.Build()
+ batch := &fnpb.Elements{Data: es}
var wg sync.WaitGroup
wg.Add(1)
go func() {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness.go b/sdks/go/pkg/beam/core/runtime/harness/harness.go
index 2b32efc02eb..20aad81c123 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/harness.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/harness.go
@@ -40,7 +40,6 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/grpcx"
"golang.org/x/sync/singleflight"
"google.golang.org/grpc"
- "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -110,7 +109,7 @@ func MainWithOptions(ctx context.Context, loggingEndpoint, controlEndpoint strin
client := fnpb.NewBeamFnControlClient(conn)
lookupDesc := func(id bundleDescriptorID) (*fnpb.ProcessBundleDescriptor, error) {
- return client.GetProcessBundleDescriptor(ctx, fnpb.GetProcessBundleDescriptorRequest_builder{ProcessBundleDescriptorId: string(id)}.Build())
+ return client.GetProcessBundleDescriptor(ctx, &fnpb.GetProcessBundleDescriptorRequest{ProcessBundleDescriptorId: string(id)})
}
stub, err := client.Control(ctx)
@@ -375,10 +374,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
}
c.mu.Unlock()
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- Register: &fnpb.RegisterResponse{},
- }.Build()
+ Response: &fnpb.InstructionResponse_Register{
+ Register: &fnpb.RegisterResponse{},
+ },
+ }
case req.GetProcessBundle() != nil:
msg := req.GetProcessBundle()
@@ -466,15 +467,15 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
if len(checkpoints) > 0 {
for _, cp := range checkpoints {
for _, r := range cp.SR.RS {
- rRoots = append(rRoots, fnpb.DelayedBundleApplication_builder{
- Application: fnpb.BundleApplication_builder{
+ rRoots = append(rRoots, &fnpb.DelayedBundleApplication{
+ Application: &fnpb.BundleApplication{
TransformId: cp.SR.TId,
InputId: cp.SR.InId,
Element: r,
OutputWatermarks: cp.SR.OW,
- }.Build(),
+ },
RequestedTimeDelay: durationpb.New(cp.Reapply),
- }.Build())
+ })
}
}
}
@@ -490,15 +491,17 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
if err != nil {
return fail(ctx, instID, "process bundle failed for instruction %v using plan %v : %v", instID, bdID, err)
}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- ProcessBundle: fnpb.ProcessBundleResponse_builder{
- ResidualRoots: rRoots,
- MonitoringData: pylds,
- MonitoringInfos: mons,
- RequiresFinalization: requiresFinalization,
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_ProcessBundle{
+ ProcessBundle: &fnpb.ProcessBundleResponse{
+ ResidualRoots: rRoots,
+ MonitoringData: pylds,
+ MonitoringInfos: mons,
+ RequiresFinalization: requiresFinalization,
+ },
+ },
+ }
case req.GetFinalizeBundle() != nil:
msg := req.GetFinalizeBundle()
@@ -518,10 +521,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
c.plans[af.bdID] = append(c.plans[af.bdID], af.plan)
delete(c.awaitingFinalization, ref)
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- FinalizeBundle: &fnpb.FinalizeBundleResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_FinalizeBundle{
+ FinalizeBundle: &fnpb.FinalizeBundleResponse{},
+ },
+ }
case req.GetProcessBundleProgress() != nil:
msg := req.GetProcessBundleProgress()
@@ -533,22 +538,26 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
return resp
}
if plan == nil && resp == nil {
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- ProcessBundleProgress: &fnpb.ProcessBundleProgressResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_ProcessBundleProgress{
+ ProcessBundleProgress: &fnpb.ProcessBundleProgressResponse{},
+ },
+ }
}
mons, pylds, consumingReceivedData := monitoring(plan, store, c.runnerCapabilities[URNMonitoringInfoShortID])
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- ProcessBundleProgress: fnpb.ProcessBundleProgressResponse_builder{
- MonitoringData: pylds,
- MonitoringInfos: mons,
- ConsumingReceivedData: &consumingReceivedData,
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_ProcessBundleProgress{
+ ProcessBundleProgress: &fnpb.ProcessBundleProgressResponse{
+ MonitoringData: pylds,
+ MonitoringInfos: mons,
+ ConsumingReceivedData: &consumingReceivedData,
+ },
+ },
+ }
case req.GetProcessBundleSplit() != nil:
msg := req.GetProcessBundleSplit()
@@ -562,10 +571,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
return resp
}
if plan == nil {
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_ProcessBundleSplit{
+ ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
+ },
+ }
}
// Get the desired splits for the root FnAPI read operation.
@@ -586,10 +597,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
// Unsuccessful splits without errors indicate we should return an empty response,
// as processing can continue.
if sr.Unsuccessful {
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_ProcessBundleSplit{
+ ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
+ },
+ }
}
var pRoots []*fnpb.BundleApplication
@@ -597,53 +610,59 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
if len(sr.PS) > 0 && len(sr.RS) > 0 {
pRoots = make([]*fnpb.BundleApplication, len(sr.PS))
for i, p := range sr.PS {
- pRoots[i] = fnpb.BundleApplication_builder{
+ pRoots[i] = &fnpb.BundleApplication{
TransformId: sr.TId,
InputId: sr.InId,
Element: p,
- }.Build()
+ }
}
rRoots = make([]*fnpb.DelayedBundleApplication, len(sr.RS))
for i, r := range sr.RS {
- rRoots[i] = fnpb.DelayedBundleApplication_builder{
- Application: fnpb.BundleApplication_builder{
+ rRoots[i] = &fnpb.DelayedBundleApplication{
+ Application: &fnpb.BundleApplication{
TransformId: sr.TId,
InputId: sr.InId,
Element: r,
OutputWatermarks: sr.OW,
- }.Build(),
- }.Build()
+ },
+ }
}
}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- ProcessBundleSplit: fnpb.ProcessBundleSplitResponse_builder{
- ChannelSplits: []*fnpb.ProcessBundleSplitResponse_ChannelSplit{fnpb.ProcessBundleSplitResponse_ChannelSplit_builder{
- TransformId: plan.SourcePTransformID(),
- LastPrimaryElement: sr.PI,
- FirstResidualElement: sr.RI,
- }.Build()},
- PrimaryRoots: pRoots,
- ResidualRoots: rRoots,
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_ProcessBundleSplit{
+ ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{
+ ChannelSplits: []*fnpb.ProcessBundleSplitResponse_ChannelSplit{{
+ TransformId: plan.SourcePTransformID(),
+ LastPrimaryElement: sr.PI,
+ FirstResidualElement: sr.RI,
+ }},
+ PrimaryRoots: pRoots,
+ ResidualRoots: rRoots,
+ },
+ },
+ }
case req.GetMonitoringInfos() != nil:
msg := req.GetMonitoringInfos()
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- MonitoringInfos: fnpb.MonitoringInfosMetadataResponse_builder{
- MonitoringInfo: shortIdsToInfos(msg.GetMonitoringInfoId()),
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_MonitoringInfos{
+ MonitoringInfos: &fnpb.MonitoringInfosMetadataResponse{
+ MonitoringInfo: shortIdsToInfos(msg.GetMonitoringInfoId()),
+ },
+ },
+ }
case req.GetHarnessMonitoringInfos() != nil:
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- HarnessMonitoringInfos: fnpb.HarnessMonitoringInfosResponse_builder{
- // TODO(BEAM-11092): Populate with non-bundle metrics data.
- MonitoringData: map[string][]byte{},
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_HarnessMonitoringInfos{
+ HarnessMonitoringInfos: &fnpb.HarnessMonitoringInfosResponse{
+ // TODO(BEAM-11092): Populate with non-bundle metrics data.
+ MonitoringData: map[string][]byte{},
+ },
+ },
+ }
case req.GetSampleData() != nil:
msg := req.GetSampleData()
var samples = make(map[string]*fnpb.SampleDataResponse_ElementList)
@@ -652,20 +671,22 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
for pid, elements := range elementsMap {
var elementList fnpb.SampleDataResponse_ElementList
for i := range elements {
- var sampledElement = fnpb.SampledElement_builder{
+ var sampledElement = &fnpb.SampledElement{
Element: elements[i].Element,
SampleTimestamp: timestamppb.New(elements[i].Timestamp),
- }.Build()
- elementList.SetElements(append(elementList.GetElements(), sampledElement))
+ }
+ elementList.Elements = append(elementList.Elements, sampledElement)
}
samples[pid] = &elementList
}
}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- SampleData: fnpb.SampleDataResponse_builder{ElementSamples: samples}.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_SampleData{
+ SampleData: &fnpb.SampleDataResponse{ElementSamples: samples},
+ },
+ }
default:
return fail(ctx, instID, "Unexpected request: %v", req)
}
@@ -709,11 +730,11 @@ func fail(ctx context.Context, id instructionID, format string, args ...any) *fn
log.Output(ctx, log.SevError, 1, fmt.Sprintf(format, args...))
dummy := &fnpb.InstructionResponse_Register{Register: &fnpb.RegisterResponse{}}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(id),
Error: fmt.Sprintf(format, args...),
- Register: proto.ValueOrDefault(dummy.Register),
- }.Build()
+ Response: dummy,
+ }
}
// dial to the specified endpoint. if timeout <=0, call blocks until
diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness_test.go b/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
index e52ca9b1c11..8c25db613eb 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
@@ -29,62 +29,62 @@ import (
// validDescriptor describes a valid pipeline with a source and a sink, but doesn't do anything else.
func validDescriptor(t *testing.T) *fnpb.ProcessBundleDescriptor {
t.Helper()
- port := fnpb.RemoteGrpcPort_builder{
+ port := &fnpb.RemoteGrpcPort{
CoderId: "c1",
- ApiServiceDescriptor: pipepb.ApiServiceDescriptor_builder{
+ ApiServiceDescriptor: &pipepb.ApiServiceDescriptor{
Url: "hostname:port",
- }.Build(),
- }.Build()
+ },
+ }
portBytes, err := proto.Marshal(port)
if err != nil {
t.Fatalf("bad port: %v", err)
}
- return fnpb.ProcessBundleDescriptor_builder{
+ return &fnpb.ProcessBundleDescriptor{
Id: "test",
Transforms: map[string]*pipepb.PTransform{
- "source": pipepb.PTransform_builder{
- Spec: pipepb.FunctionSpec_builder{
+ "source": &pipepb.PTransform{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:runner:source:v1",
Payload: portBytes,
- }.Build(),
+ },
Outputs: map[string]string{
"o1": "p1",
},
- }.Build(),
- "sink": pipepb.PTransform_builder{
- Spec: pipepb.FunctionSpec_builder{
+ },
+ "sink": &pipepb.PTransform{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:runner:sink:v1",
Payload: portBytes,
- }.Build(),
+ },
Inputs: map[string]string{
"i1": "p1",
},
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p1": pipepb.PCollection_builder{
+ "p1": &pipepb.PCollection{
CoderId: "c1",
- }.Build(),
+ },
},
Coders: map[string]*pipepb.Coder{
- "c1": pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ "c1": &pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:coder:windowed_value:v1",
- }.Build(),
+ },
ComponentCoderIds: []string{"c2", "c3"},
- }.Build(),
- "c2": pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ },
+ "c2": &pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:coder:varint:v1",
- }.Build(),
- }.Build(),
- "c3": pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ },
+ },
+ "c3": &pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:coder:global_window:v1",
- }.Build(),
- }.Build(),
+ },
+ },
},
- }.Build()
+ }
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/logging.go b/sdks/go/pkg/beam/core/runtime/harness/logging.go
index f5d6d18b622..20fe7660828 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/logging.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/logging.go
@@ -38,16 +38,16 @@ type logger struct {
func (l *logger) Log(ctx context.Context, sev log.Severity, calldepth int, msg string) {
now := timestamppb.New(time.Now())
- entry := fnpb.LogEntry_builder{
+ entry := &fnpb.LogEntry{
Timestamp: now,
Severity: convertSeverity(sev),
Message: msg,
- }.Build()
+ }
if _, file, line, ok := runtime.Caller(calldepth + 1); ok {
- entry.SetLogLocation(fmt.Sprintf("%v:%v", file, line))
+ entry.LogLocation = fmt.Sprintf("%v:%v", file, line)
}
- entry.SetInstructionId(metrics.GetBundleID(ctx))
- entry.SetTransformId(metrics.GetTransformID(ctx))
+ entry.InstructionId = metrics.GetBundleID(ctx)
+ entry.TransformId = metrics.GetTransformID(ctx)
select {
case l.out <- entry:
@@ -174,7 +174,7 @@ func (w *remoteWriter) connect(ctx context.Context, makeClient func(ctx context.
return errBuffClosed
}
msgs = append(msgs, newMsg)
- flush = newMsg.GetSeverity() >= fnpb.LogEntry_Severity_CRITICAL
+ flush = newMsg.Severity >= fnpb.LogEntry_Severity_CRITICAL
}
// If there are still messages in the buffer, drain them out to batch them to a cap.
@@ -185,12 +185,12 @@ func (w *remoteWriter) connect(ctx context.Context, makeClient func(ctx context.
return errBuffClosed
}
msgs = append(msgs, newMsg)
- flush = newMsg.GetSeverity() >= fnpb.LogEntry_Severity_CRITICAL
+ flush = newMsg.Severity >= fnpb.LogEntry_Severity_CRITICAL
}
- list := fnpb.LogEntry_List_builder{
+ list := &fnpb.LogEntry_List{
LogEntries: msgs,
- }.Build()
+ }
if err := client.Send(list); err != nil {
if err == io.EOF {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/logging_test.go b/sdks/go/pkg/beam/core/runtime/harness/logging_test.go
index 0087bb6cbf0..b2383111a4d 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/logging_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/logging_test.go
@@ -103,7 +103,7 @@ func TestLogger_connect(t *testing.T) {
var count int
for _, batch := range cacher.logs {
- count += len(batch.GetLogEntries())
+ count += len(batch.LogEntries)
}
if got, want := count, 10; got != want {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/monitoring.go b/sdks/go/pkg/beam/core/runtime/harness/monitoring.go
index ab5c091c3db..08b87514732 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/monitoring.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/monitoring.go
@@ -69,11 +69,11 @@ func (c *shortIDCache) getShortID(l metrics.Labels, urn metricsx.Urn) string {
}
s = c.getNextShortID()
c.labels2ShortIds[k] = s
- c.shortIds2Infos[s] = pipepb.MonitoringInfo_builder{
+ c.shortIds2Infos[s] = &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(urn),
Type: metricsx.UrnToType(urn),
Labels: l.Map(),
- }.Build()
+ }
return s
}
@@ -120,12 +120,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(l, metricsx.UrnUserSumInt64)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnUserSumInt64),
Type: metricsx.UrnToType(metricsx.UrnUserSumInt64),
Labels: l.Map(),
Payload: payload,
- }.Build())
+ })
}
},
DistributionInt64: func(l metrics.Labels, count, sum, min, max int64) {
@@ -136,12 +136,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(l, metricsx.UrnUserDistInt64)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnUserDistInt64),
Type: metricsx.UrnToType(metricsx.UrnUserDistInt64),
Labels: l.Map(),
Payload: payload,
- }.Build())
+ })
}
},
GaugeInt64: func(l metrics.Labels, v int64, t time.Time) {
@@ -152,12 +152,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(l, metricsx.UrnUserLatestMsInt64)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnUserLatestMsInt64),
Type: metricsx.UrnToType(metricsx.UrnUserLatestMsInt64),
Labels: l.Map(),
Payload: payload,
- }.Build())
+ })
}
},
MsecsInt64: func(l string, states *[4]metrics.ExecutionState) {
@@ -171,12 +171,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(metrics.PTransformLabels(l), ul)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(ul),
Type: metricsx.UrnToType(ul),
Labels: label,
Payload: payload,
- }.Build())
+ })
}
}
},
@@ -199,14 +199,14 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnElementCount),
Type: metricsx.UrnToType(metricsx.UrnElementCount),
Labels: map[string]string{
"PCOLLECTION": pcol.ID,
},
Payload: payload,
- }.Build())
+ })
}
// Skip pcollections without size
if pcol.SizeCount != 0 {
@@ -218,14 +218,14 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnSampledByteSize),
Type: metricsx.UrnToType(metricsx.UrnSampledByteSize),
Labels: map[string]string{
"PCOLLECTION": pcol.ID,
},
Payload: payload,
- }.Build())
+ })
}
}
}
@@ -238,14 +238,14 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(metrics.PTransformLabels(snapshot.Source.ID), metricsx.UrnDataChannelReadIndex)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnDataChannelReadIndex),
Type: metricsx.UrnToType(metricsx.UrnDataChannelReadIndex),
Labels: map[string]string{
"PTRANSFORM": snapshot.Source.ID,
},
Payload: payload,
- }.Build())
+ })
}
return monitoringInfo, payloads, snapshot.Source.ConsumingReceivedData
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go b/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go
index 7535ad4d5c3..23f6fbf2461 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go
@@ -122,13 +122,15 @@ func TestSetCache_CacheableCase(t *testing.T) {
}
func makeRequest(transformID, sideInputID string, t token) *fnpb.ProcessBundleRequest_CacheToken {
- return fnpb.ProcessBundleRequest_CacheToken_builder{
+ return &fnpb.ProcessBundleRequest_CacheToken{
Token: []byte(t),
- SideInput: fnpb.ProcessBundleRequest_CacheToken_SideInput_builder{
- TransformId: transformID,
- SideInputId: sideInputID,
- }.Build(),
- }.Build()
+ Type: &fnpb.ProcessBundleRequest_CacheToken_SideInput_{
+ SideInput: &fnpb.ProcessBundleRequest_CacheToken_SideInput{
+ TransformId: transformID,
+ SideInputId: sideInputID,
+ },
+ },
+ }
}
func makeCacheToken(transformID, sideInputID string, tok token) cacheToken {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/statemgr.go b/sdks/go/pkg/beam/core/runtime/harness/statemgr.go
index e082d9806de..4c1bc0b55fe 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/statemgr.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/statemgr.go
@@ -228,14 +228,16 @@ type stateKeyWriter struct {
}
func newSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID string, instID instructionID, k, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- MultimapSideInput: fnpb.StateKey_MultimapSideInput_builder{
- TransformId: id.PtransformID,
- SideInputId: sideInputID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapSideInput_{
+ MultimapSideInput: &fnpb.StateKey_MultimapSideInput{
+ TransformId: id.PtransformID,
+ SideInputId: sideInputID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -244,13 +246,15 @@ func newSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID string,
}
func newIterableSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID string, instID instructionID, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- IterableSideInput: fnpb.StateKey_IterableSideInput_builder{
- TransformId: id.PtransformID,
- SideInputId: sideInputID,
- Window: w,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_IterableSideInput_{
+ IterableSideInput: &fnpb.StateKey_IterableSideInput{
+ TransformId: id.PtransformID,
+ SideInputId: sideInputID,
+ Window: w,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
@@ -260,11 +264,13 @@ func newIterableSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID
}
func newRunnerReader(ch *StateChannel, instID instructionID, k []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- Runner: fnpb.StateKey_Runner_builder{
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_Runner_{
+ Runner: &fnpb.StateKey_Runner{
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -273,14 +279,16 @@ func newRunnerReader(ch *StateChannel, instID instructionID, k []byte) *stateKey
}
func newBagUserStateReader(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- BagUserState: fnpb.StateKey_BagUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_BagUserState_{
+ BagUserState: &fnpb.StateKey_BagUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -289,14 +297,16 @@ func newBagUserStateReader(ch *StateChannel, id exec.StreamID, instID instructio
}
func newBagUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, wt writeTypeEnum) *stateKeyWriter {
- key := fnpb.StateKey_builder{
- BagUserState: fnpb.StateKey_BagUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_BagUserState_{
+ BagUserState: &fnpb.StateKey_BagUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyWriter{
instID: instID,
key: key,
@@ -306,15 +316,17 @@ func newBagUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructio
}
func newMultimapUserStateReader(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, mk []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- MultimapUserState: fnpb.StateKey_MultimapUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- MapKey: mk,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapUserState_{
+ MultimapUserState: &fnpb.StateKey_MultimapUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ MapKey: mk,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -323,15 +335,17 @@ func newMultimapUserStateReader(ch *StateChannel, id exec.StreamID, instID instr
}
func newMultimapUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, mk []byte, wt writeTypeEnum) *stateKeyWriter {
- key := fnpb.StateKey_builder{
- MultimapUserState: fnpb.StateKey_MultimapUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- MapKey: mk,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapUserState_{
+ MultimapUserState: &fnpb.StateKey_MultimapUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ MapKey: mk,
+ },
+ },
+ }
return &stateKeyWriter{
instID: instID,
key: key,
@@ -341,14 +355,16 @@ func newMultimapUserStateWriter(ch *StateChannel, id exec.StreamID, instID instr
}
func newMultimapKeysUserStateReader(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- MultimapKeysUserState: fnpb.StateKey_MultimapKeysUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapKeysUserState_{
+ MultimapKeysUserState: &fnpb.StateKey_MultimapKeysUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -357,14 +373,16 @@ func newMultimapKeysUserStateReader(ch *StateChannel, id exec.StreamID, instID i
}
func newMultimapKeysUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, wt writeTypeEnum) *stateKeyWriter {
- key := fnpb.StateKey_builder{
- MultimapKeysUserState: fnpb.StateKey_MultimapKeysUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapKeysUserState_{
+ MultimapKeysUserState: &fnpb.StateKey_MultimapKeysUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyWriter{
instID: instID,
key: key,
@@ -389,14 +407,16 @@ func (r *stateKeyReader) Read(buf []byte) (int, error) {
localChannel := r.ch
r.mu.Unlock()
- req := fnpb.StateRequest_builder{
+ req := &fnpb.StateRequest{
// Id: set by StateChannel
InstructionId: string(r.instID),
StateKey: r.key,
- Get: fnpb.StateGetRequest_builder{
- ContinuationToken: r.token,
- }.Build(),
- }.Build()
+ Request: &fnpb.StateRequest_Get{
+ Get: &fnpb.StateGetRequest{
+ ContinuationToken: r.token,
+ },
+ },
+ }
resp, err := localChannel.Send(req)
if err != nil {
r.Close()
@@ -457,21 +477,25 @@ func (r *stateKeyWriter) Write(buf []byte) (int, error) {
var req *fnpb.StateRequest
switch r.writeType {
case writeTypeAppend:
- req = fnpb.StateRequest_builder{
+ req = &fnpb.StateRequest{
// Id: set by StateChannel
InstructionId: string(r.instID),
StateKey: r.key,
- Append: fnpb.StateAppendRequest_builder{
- Data: toSend,
- }.Build(),
- }.Build()
+ Request: &fnpb.StateRequest_Append{
+ Append: &fnpb.StateAppendRequest{
+ Data: toSend,
+ },
+ },
+ }
case writeTypeClear:
- req = fnpb.StateRequest_builder{
+ req = &fnpb.StateRequest{
// ID: set by StateChannel
InstructionId: string(r.instID),
StateKey: r.key,
- Clear: &fnpb.StateClearRequest{},
- }.Build()
+ Request: &fnpb.StateRequest_Clear{
+ Clear: &fnpb.StateClearRequest{},
+ },
+ }
default:
return 0, errors.Errorf("Unknown write type %v", r.writeType)
}
@@ -607,8 +631,8 @@ func (c *StateChannel) read(ctx context.Context) {
}
c.mu.Lock()
- ch, ok := c.responses[msg.GetId()]
- delete(c.responses, msg.GetId())
+ ch, ok := c.responses[msg.Id]
+ delete(c.responses, msg.Id)
c.mu.Unlock()
if !ok {
// This can happen if Send returns an error that write handles, but
@@ -638,7 +662,7 @@ func (c *StateChannel) write(ctx context.Context) {
}
err = c.client.Send(req)
if err != nil {
- id = req.GetId()
+ id = req.Id
break // non-nil errors mean the stream is broken and can't be re-used.
}
}
@@ -664,14 +688,14 @@ func (c *StateChannel) write(ctx context.Context) {
c.terminateStreamOnError(err)
if ok {
- ch <- fnpb.StateResponse_builder{Id: id, Error: fmt.Sprintf("StateChannel[%v].write failed to send: %v", c.id, err)}.Build()
+ ch <- &fnpb.StateResponse{Id: id, Error: fmt.Sprintf("StateChannel[%v].write failed to send: %v", c.id, err)}
}
}
// Send sends a state request and returns the response.
func (c *StateChannel) Send(req *fnpb.StateRequest) (*fnpb.StateResponse, error) {
id := fmt.Sprintf("r%v", atomic.AddInt32(&c.nextRequestNo, 1))
- req.SetId(id)
+ req.Id = id
ch := make(chan *fnpb.StateResponse, 1)
c.mu.Lock()
@@ -692,8 +716,8 @@ func (c *StateChannel) Send(req *fnpb.StateRequest) (*fnpb.StateResponse, error)
defer c.mu.Unlock()
return nil, errors.Wrapf(c.closedErr, "StateChannel[%v].Send(%v): context canceled", c.id, id)
}
- if resp.GetError() != "" {
- return nil, errors.New(resp.GetError())
+ if resp.Error != "" {
+ return nil, errors.New(resp.Error)
}
return resp, nil
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go b/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go
index 81b1f2ca11e..53c4ca05ee1 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go
@@ -93,21 +93,25 @@ func TestStateChannel(t *testing.T) {
for i := 0; i < count; i++ {
go func() {
req := <-client.send
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(), // Ids need to match up to ensure routing can occur properly.
- Get: fnpb.StateGetResponse_builder{
- ContinuationToken: req.GetGet().GetContinuationToken(),
- }.Build(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id, // Ids need to match up to ensure routing can occur properly.
+ Response: &fnpb.StateResponse_Get{
+ Get: &fnpb.StateGetResponse{
+ ContinuationToken: req.GetGet().GetContinuationToken(),
+ },
+ },
+ }
}()
}
for i := 0; i < count; i++ {
token := []byte(fmt.Sprintf("%d", i))
- resp, err := c.Send(fnpb.StateRequest_builder{
- Get: fnpb.StateGetRequest_builder{
- ContinuationToken: token,
- }.Build(),
- }.Build())
+ resp, err := c.Send(&fnpb.StateRequest{
+ Request: &fnpb.StateRequest_Get{
+ Get: &fnpb.StateGetRequest{
+ ContinuationToken: token,
+ },
+ },
+ })
if err != nil {
t.Fatalf("unexpected error from Send: %v", err)
}
@@ -124,9 +128,9 @@ func TestStateChannel(t *testing.T) {
req := <-client.send // Send should succeed.
client.setRecvErr(io.EOF)
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
}()
_, err := c.Send(&fnpb.StateRequest{})
return err
@@ -140,9 +144,9 @@ func TestStateChannel(t *testing.T) {
req := <-client.send // Send should succeed.
client.setRecvErr(expectedError)
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
}()
_, err := c.Send(&fnpb.StateRequest{})
return err
@@ -156,13 +160,13 @@ func TestStateChannel(t *testing.T) {
req := <-client.send // Send should succeed.
c.mu.Lock()
- ch := c.responses[req.GetId()]
- delete(c.responses, req.GetId())
+ ch := c.responses[req.Id]
+ delete(c.responses, req.Id)
c.mu.Unlock()
- resp := fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ resp := &fnpb.StateResponse{
+ Id: req.Id,
+ }
client.recv <- resp
// unblock Send.
ch <- resp
@@ -179,9 +183,9 @@ func TestStateChannel(t *testing.T) {
// This can be plumbed through on either side, write or read,
// the important part is that we get it.
client.setRecvErr(expectedError)
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
}()
_, err := c.Send(&fnpb.StateRequest{})
return err
@@ -231,7 +235,7 @@ func TestStateChannel(t *testing.T) {
client.setRecvErr(nil)
// Drain the next send, and ensure the response is unblocked.
req := <-client.send
- client.recv <- fnpb.StateResponse_builder{Id: req.GetId()}.Build() // Ids need to match up to ensure routing can occur properly.
+ client.recv <- &fnpb.StateResponse{Id: req.Id} // Ids need to match up to ensure routing can occur properly.
}()
if _, err := c.Send(&fnpb.StateRequest{}); !contains(err, test.expectedErr) {
t.Errorf("Unexpected error from Send: got %v, want %v", err, test.expectedErr)
@@ -333,9 +337,9 @@ func TestStateKeyReader(t *testing.T) {
go func() {
if test.noGet {
req := <-ch.requests
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
return
}
for i, buflen := range test.buflens {
@@ -350,13 +354,15 @@ func TestStateKeyReader(t *testing.T) {
}
req := <-ch.requests
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Get: fnpb.StateGetResponse_builder{
- ContinuationToken: token,
- Data: buf,
- }.Build(),
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Get{
+ Get: &fnpb.StateGetResponse{
+ ContinuationToken: token,
+ Data: buf,
+ },
+ },
+ }
}
}()
@@ -457,30 +463,30 @@ func TestStateKeyWriter(t *testing.T) {
if !ok {
t.Errorf("Append write: got %v, want data of type StateRequest_Append", req.Request)
} else {
- data := sra.Append.GetData()
+ data := sra.Append.Data
if !bytes.Equal(data, test.data) {
t.Errorf("Expected request data: got %v, want %v", data, test.data)
}
}
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Append: &fnpb.StateAppendResponse{},
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Append{},
+ }
case writeTypeClear:
_, ok := req.Request.(*fnpb.StateRequest_Clear)
if !ok {
t.Errorf("Append write: got %v, want data of type StateRequest_Append", req.Request)
}
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Clear: &fnpb.StateClearResponse{},
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Clear{},
+ }
default:
// Still return response so that write doesn't hang.
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Append: &fnpb.StateAppendResponse{},
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Append{},
+ }
}
}()
diff --git a/sdks/go/pkg/beam/core/runtime/harness/worker_status.go b/sdks/go/pkg/beam/core/runtime/harness/worker_status.go
index 9b53f3cf065..0ec7e1f805c 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/worker_status.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/worker_status.go
@@ -129,7 +129,7 @@ func (w *workerStatusHandler) reader(ctx context.Context, stub fnpb.BeamFnWorker
goroutineDump(statusInfo)
buildInfo(statusInfo)
- response := fnpb.WorkerStatusResponse_builder{Id: req.GetId(), StatusInfo: statusInfo.String()}.Build()
+ response := &fnpb.WorkerStatusResponse{Id: req.GetId(), StatusInfo: statusInfo.String()}
if err := stub.Send(response); err != nil && err != io.EOF {
log.Errorf(ctx, "workerStatus.Writer: Failed to respond: %v", err)
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go b/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go
index dffe0e26fb2..9cc2e533079 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go
@@ -37,7 +37,7 @@ type BeamFnWorkerStatusServicer struct {
}
func (w *BeamFnWorkerStatusServicer) WorkerStatus(b fnpb.BeamFnWorkerStatus_WorkerStatusServer) error {
- b.Send(fnpb.WorkerStatusRequest_builder{Id: "1"}.Build())
+ b.Send(&fnpb.WorkerStatusRequest{Id: "1"})
resp, err := b.Recv()
if err != nil {
return fmt.Errorf("error receiving response b.recv: %v", err)
diff --git a/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go b/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
index 06d58290346..95b92f929c0 100644
--- a/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
+++ b/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
@@ -46,12 +46,12 @@ func TestFromMonitoringInfos_Counters(t *testing.T) {
"NAME": "customCounter",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnUserSumInt64),
Type: UrnToType(UrnUserSumInt64),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -111,30 +111,30 @@ func TestFromMonitoringInfos_Msec(t *testing.T) {
if err != nil {
t.Fatalf("Failed to encode Int64Counter: %v", err)
}
- mStartBundleInfo := pipepb.MonitoringInfo_builder{
+ mStartBundleInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(0)),
Type: UrnToType(ExecutionMsecUrn(0)),
Labels: labels,
Payload: startValue,
- }.Build()
- mProcessBundleInfo := pipepb.MonitoringInfo_builder{
+ }
+ mProcessBundleInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(1)),
Type: UrnToType(ExecutionMsecUrn(1)),
Labels: labels,
Payload: processValue,
- }.Build()
- mFinishBundleInfo := pipepb.MonitoringInfo_builder{
+ }
+ mFinishBundleInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(2)),
Type: UrnToType(ExecutionMsecUrn(2)),
Labels: labels,
Payload: finishValue,
- }.Build()
- mTotalTimeInfo := pipepb.MonitoringInfo_builder{
+ }
+ mTotalTimeInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(3)),
Type: UrnToType(ExecutionMsecUrn(3)),
Labels: labels,
Payload: totalValue,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mStartBundleInfo, mProcessBundleInfo, mFinishBundleInfo, mTotalTimeInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -174,12 +174,12 @@ func TestFromMonitoringInfos_PColCounters(t *testing.T) {
"NAME": "customCounter",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnElementCount),
Type: UrnToType(UrnElementCount),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -224,12 +224,12 @@ func TestFromMonitoringInfos_SampledByteSize(t *testing.T) {
"NAME": "customCounter",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnSampledByteSize),
Type: UrnToType(UrnSampledByteSize),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -274,12 +274,12 @@ func TestFromMonitoringInfos_Distributions(t *testing.T) {
"NAME": "customDist",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnUserDistInt64),
Type: UrnToType(UrnUserDistInt64),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -324,12 +324,12 @@ func TestFromMonitoringInfos_Gauges(t *testing.T) {
"NAME": "customGauge",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnUserLatestMsInt64),
Type: UrnToType(UrnUserLatestMsInt64),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go b/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go
index 443853b0769..7745cec942e 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go
@@ -21,10 +21,10 @@ import (
)
func shallowClonePipeline(p *pipepb.Pipeline) *pipepb.Pipeline {
- ret := pipepb.Pipeline_builder{
+ ret := &pipepb.Pipeline{
Components: shallowCloneComponents(p.GetComponents()),
Requirements: reflectx.ShallowClone(p.GetRequirements()).([]string),
- }.Build()
+ }
ret.RootTransformIds, _ = reflectx.ShallowClone(p.GetRootTransformIds()).([]string)
return ret
}
@@ -45,15 +45,15 @@ func ShallowClonePTransform(t *pipepb.PTransform) *pipepb.PTransform {
return nil
}
- ret := pipepb.PTransform_builder{
- UniqueName: t.GetUniqueName(),
- Spec: t.GetSpec(),
- DisplayData: t.GetDisplayData(),
- Annotations: t.GetAnnotations(),
- }.Build()
- ret.Subtransforms, _ = reflectx.ShallowClone(t.GetSubtransforms()).([]string)
- ret.Inputs, _ = reflectx.ShallowClone(t.GetInputs()).(map[string]string)
- ret.Outputs, _ = reflectx.ShallowClone(t.GetOutputs()).(map[string]string)
- ret.SetEnvironmentId(t.GetEnvironmentId())
+ ret := &pipepb.PTransform{
+ UniqueName: t.UniqueName,
+ Spec: t.Spec,
+ DisplayData: t.DisplayData,
+ Annotations: t.Annotations,
+ }
+ ret.Subtransforms, _ = reflectx.ShallowClone(t.Subtransforms).([]string)
+ ret.Inputs, _ = reflectx.ShallowClone(t.Inputs).(map[string]string)
+ ret.Outputs, _ = reflectx.ShallowClone(t.Outputs).(map[string]string)
+ ret.EnvironmentId = t.EnvironmentId
return ret
}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go b/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go
index cc674b96979..b58a3098379 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go
@@ -26,11 +26,11 @@ import (
func TestShallowClonePTransform(t *testing.T) {
tests := []*pipepb.PTransform{
{},
- pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- pipepb.PTransform_builder{Spec: pipepb.FunctionSpec_builder{Urn: "foo"}.Build()}.Build(),
- pipepb.PTransform_builder{Subtransforms: []string{"a", "b"}}.Build(),
- pipepb.PTransform_builder{Inputs: map[string]string{"a": "b"}}.Build(),
- pipepb.PTransform_builder{Outputs: map[string]string{"a": "b"}}.Build(),
+ {UniqueName: "a"},
+ {Spec: &pipepb.FunctionSpec{Urn: "foo"}},
+ {Subtransforms: []string{"a", "b"}},
+ {Inputs: map[string]string{"a": "b"}},
+ {Outputs: map[string]string{"a": "b"}},
}
for _, test := range tests {
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go b/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go
index 89e671f37ad..9e527f2fd32 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go
@@ -36,11 +36,11 @@ import (
// modified.
func Update(p *pipepb.Pipeline, values *pipepb.Components) (*pipepb.Pipeline, error) {
ret := shallowClonePipeline(p)
- reflectx.UpdateMap(ret.GetComponents().GetTransforms(), values.GetTransforms())
- reflectx.UpdateMap(ret.GetComponents().GetPcollections(), values.GetPcollections())
- reflectx.UpdateMap(ret.GetComponents().GetWindowingStrategies(), values.GetWindowingStrategies())
- reflectx.UpdateMap(ret.GetComponents().GetCoders(), values.GetCoders())
- reflectx.UpdateMap(ret.GetComponents().GetEnvironments(), values.GetEnvironments())
+ reflectx.UpdateMap(ret.Components.Transforms, values.Transforms)
+ reflectx.UpdateMap(ret.Components.Pcollections, values.Pcollections)
+ reflectx.UpdateMap(ret.Components.WindowingStrategies, values.WindowingStrategies)
+ reflectx.UpdateMap(ret.Components.Coders, values.Coders)
+ reflectx.UpdateMap(ret.Components.Environments, values.Environments)
return Normalize(ret)
}
@@ -60,12 +60,12 @@ func Normalize(p *pipepb.Pipeline) (*pipepb.Pipeline, error) {
ret := shallowClonePipeline(p)
if IdempotentNormalize {
- ret.GetComponents().SetTransforms(ensureUniqueNames(ret.GetComponents().GetTransforms()))
+ ret.Components.Transforms = ensureUniqueNames(ret.Components.Transforms)
} else {
- ret.GetComponents().SetTransforms(ensureUniqueNamesLegacy(ret.GetComponents().GetTransforms()))
+ ret.Components.Transforms = ensureUniqueNamesLegacy(ret.Components.Transforms)
}
- ret.GetComponents().SetTransforms(computeCompositeInputOutput(ret.GetComponents().GetTransforms()))
- ret.SetRootTransformIds(computeRoots(ret.GetComponents().GetTransforms()))
+ ret.Components.Transforms = computeCompositeInputOutput(ret.Components.Transforms)
+ ret.RootTransformIds = computeRoots(ret.Components.Transforms)
return ret, nil
}
@@ -85,7 +85,7 @@ func walkCoders(coders, accum map[string]*pipepb.Coder, id string) {
c := coders[id]
accum[id] = c
- for _, sub := range c.GetComponentCoderIds() {
+ for _, sub := range c.ComponentCoderIds {
walkCoders(coders, accum, sub)
}
}
@@ -106,7 +106,7 @@ func computeRoots(xforms map[string]*pipepb.PTransform) []string {
func makeParentMap(xforms map[string]*pipepb.PTransform) map[string]string {
parent := make(map[string]string)
for id, t := range xforms {
- for _, key := range t.GetSubtransforms() {
+ for _, key := range t.Subtransforms {
parent[key] = id
}
}
@@ -138,7 +138,7 @@ func computeCompositeInputOutput(xforms map[string]*pipepb.PTransform) map[strin
// maps of composite transforms. Update the transform map.
func walk(id string, ret map[string]*pipepb.PTransform, seen map[string]bool, primitiveXformsForInput map[string][]string) {
t := ret[id]
- if seen[id] || len(t.GetSubtransforms()) == 0 {
+ if seen[id] || len(t.Subtransforms) == 0 {
return
}
@@ -150,7 +150,7 @@ func walk(id string, ret map[string]*pipepb.PTransform, seen map[string]bool, pr
in := make(map[string]bool)
out := make(map[string]bool)
local := map[string]bool{id: true}
- for _, sid := range t.GetSubtransforms() {
+ for _, sid := range t.Subtransforms {
walk(sid, ret, seen, primitiveXformsForInput)
inout(ret[sid], in, out)
local[sid] = true
@@ -164,9 +164,9 @@ func walk(id string, ret map[string]*pipepb.PTransform, seen map[string]bool, pr
externalIns(local, primitiveXformsForInput, extIn, out)
upd := ShallowClonePTransform(t)
- upd.SetInputs(diff(in, out))
- upd.SetOutputs(diffAndMerge(out, in, extIn))
- upd.SetSubtransforms(TopologicalSort(ret, upd.GetSubtransforms()))
+ upd.Inputs = diff(in, out)
+ upd.Outputs = diffAndMerge(out, in, extIn)
+ upd.Subtransforms = TopologicalSort(ret, upd.Subtransforms)
ret[id] = upd
seen[id] = true
@@ -312,9 +312,9 @@ func ensureUniqueNames(xforms map[string]*pipepb.PTransform) map[string]*pipepb.
name := findFreeName(seen, base)
seen[name] = true
- if name != t.GetUniqueName() {
+ if name != t.UniqueName {
upd := ShallowClonePTransform(t)
- upd.SetUniqueName(name)
+ upd.UniqueName = name
ret[id] = upd
}
return name
@@ -373,12 +373,12 @@ func ensureUniqueNamesLegacy(xforms map[string]*pipepb.PTransform) map[string]*p
seen := make(map[string]bool)
for _, id := range ordering {
t := xforms[id]
- name := findFreeName(seen, t.GetUniqueName())
+ name := findFreeName(seen, t.UniqueName)
seen[name] = true
- if name != t.GetUniqueName() {
+ if name != t.UniqueName {
upd := ShallowClonePTransform(t)
- upd.SetUniqueName(name)
+ upd.UniqueName = name
ret[id] = upd
}
}
@@ -426,12 +426,12 @@ func ApplySdkImageOverrides(p *pipepb.Pipeline, patterns map[string]string) erro
for re, replacement := range regexes {
newImg := re.ReplaceAllLiteralString(oldImg, replacement)
if newImg != oldImg {
- payload.SetContainerImage(newImg)
+ payload.ContainerImage = newImg
pl, err := proto.Marshal(&payload)
if err != nil {
return err
}
- env.SetPayload(pl)
+ env.Payload = pl
break // Apply at most one override to each environment.
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go b/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go
index 02f4d660f5f..3024787e616 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go
@@ -33,130 +33,130 @@ func TestEnsureUniqueName(t *testing.T) {
{
name: "AlreadyUnique",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "b"},
+ "3": {UniqueName: "c"},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "b"},
+ "3": {UniqueName: "c"},
},
},
{
name: "NeedsUniqueLeaves",
in: map[string]*pipepb.PTransform{
- "2": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
+ "2": {UniqueName: "a"},
+ "1": {UniqueName: "a"},
+ "3": {UniqueName: "a"},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "a'1"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a'2"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "a'1"},
+ "3": {UniqueName: "a'2"},
},
},
{
name: "StripUniqueLeaves",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "a'1"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a'2"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "a'1"},
+ "3": {UniqueName: "a'2"},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "a'1"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a'2"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "a'1"},
+ "3": {UniqueName: "a'2"},
},
},
{
name: "NonTopologicalIdOrder",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s1"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2"}}.Build(), // root
+ "e1": {UniqueName: "a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a", Subtransforms: []string{"s1"}},
+ "s3": {UniqueName: "a", Subtransforms: []string{"s2"}}, // root
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a/a/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a/a/a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a/a", Subtransforms: []string{"s1"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2"}}.Build(), // root
+ "e1": {UniqueName: "a/a/a/a"},
+ "s1": {UniqueName: "a/a/a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a/a", Subtransforms: []string{"s1"}},
+ "s3": {UniqueName: "a", Subtransforms: []string{"s2"}}, // root
},
},
{
name: "UniqueComps",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a"},
+ "e2": {UniqueName: "a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a", Subtransforms: []string{"e2"}},
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a'1/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a'1", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a/a"},
+ "e2": {UniqueName: "a'1/a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a'1", Subtransforms: []string{"e2"}},
},
},
{
name: "StripComps",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a'1/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a'1", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a/a"},
+ "e2": {UniqueName: "a'1/a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a'1", Subtransforms: []string{"e2"}},
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a'1/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a'1", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a/a"},
+ "e2": {UniqueName: "a'1/a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a'1", Subtransforms: []string{"e2"}},
},
},
{
name: "large",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e3": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e4": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e5": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e6": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e7": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e8": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e9": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e10": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e11": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e12": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2", "s3"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s4", "s5"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s6", "s7"}}.Build(),
- "s4": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s5": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e2", "e3"}}.Build(),
- "s6": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e4", "e5", "e6"}}.Build(),
- "s7": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}}.Build(),
+ "e1": {UniqueName: "a"},
+ "e2": {UniqueName: "a"},
+ "e3": {UniqueName: "a"},
+ "e4": {UniqueName: "a"},
+ "e5": {UniqueName: "a"},
+ "e6": {UniqueName: "a"},
+ "e7": {UniqueName: "a"},
+ "e8": {UniqueName: "a"},
+ "e9": {UniqueName: "a"},
+ "e10": {UniqueName: "a"},
+ "e11": {UniqueName: "a"},
+ "e12": {UniqueName: "a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"s2", "s3"}},
+ "s2": {UniqueName: "a", Subtransforms: []string{"s4", "s5"}},
+ "s3": {UniqueName: "a", Subtransforms: []string{"s6", "s7"}},
+ "s4": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s5": {UniqueName: "a", Subtransforms: []string{"e2", "e3"}},
+ "s6": {UniqueName: "a", Subtransforms: []string{"e4", "e5", "e6"}},
+ "s7": {UniqueName: "a", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}},
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a/a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a/a/a'1/a"}.Build(),
- "e3": pipepb.PTransform_builder{UniqueName: "a/a/a'1/a'1"}.Build(),
- "e4": pipepb.PTransform_builder{UniqueName: "a/a'1/a/a"}.Build(),
- "e5": pipepb.PTransform_builder{UniqueName: "a/a'1/a/a'1"}.Build(),
- "e6": pipepb.PTransform_builder{UniqueName: "a/a'1/a/a'2"}.Build(),
- "e7": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a"}.Build(),
- "e8": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'1"}.Build(),
- "e9": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'2"}.Build(),
- "e10": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'3"}.Build(),
- "e11": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'4"}.Build(),
- "e12": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'5"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2", "s3"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a/a", Subtransforms: []string{"s4", "s5"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a/a'1", Subtransforms: []string{"s6", "s7"}}.Build(),
- "s4": pipepb.PTransform_builder{UniqueName: "a/a/a", Subtransforms: []string{"e1"}}.Build(),
- "s5": pipepb.PTransform_builder{UniqueName: "a/a/a'1", Subtransforms: []string{"e2", "e3"}}.Build(),
- "s6": pipepb.PTransform_builder{UniqueName: "a/a'1/a", Subtransforms: []string{"e4", "e5", "e6"}}.Build(),
- "s7": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}}.Build(),
+ "e1": {UniqueName: "a/a/a/a"},
+ "e2": {UniqueName: "a/a/a'1/a"},
+ "e3": {UniqueName: "a/a/a'1/a'1"},
+ "e4": {UniqueName: "a/a'1/a/a"},
+ "e5": {UniqueName: "a/a'1/a/a'1"},
+ "e6": {UniqueName: "a/a'1/a/a'2"},
+ "e7": {UniqueName: "a/a'1/a'1/a"},
+ "e8": {UniqueName: "a/a'1/a'1/a'1"},
+ "e9": {UniqueName: "a/a'1/a'1/a'2"},
+ "e10": {UniqueName: "a/a'1/a'1/a'3"},
+ "e11": {UniqueName: "a/a'1/a'1/a'4"},
+ "e12": {UniqueName: "a/a'1/a'1/a'5"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"s2", "s3"}},
+ "s2": {UniqueName: "a/a", Subtransforms: []string{"s4", "s5"}},
+ "s3": {UniqueName: "a/a'1", Subtransforms: []string{"s6", "s7"}},
+ "s4": {UniqueName: "a/a/a", Subtransforms: []string{"e1"}},
+ "s5": {UniqueName: "a/a/a'1", Subtransforms: []string{"e2", "e3"}},
+ "s6": {UniqueName: "a/a'1/a", Subtransforms: []string{"e4", "e5", "e6"}},
+ "s7": {UniqueName: "a/a'1/a'1", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}},
},
},
}
@@ -179,124 +179,124 @@ func TestComputeInputOutput(t *testing.T) {
{
name: "singleton composite",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Inputs: map[string]string{"i0": "p1"},
Outputs: map[string]string{"i0": "p2"},
- }.Build(),
+ },
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
Inputs: map[string]string{"p1": "p1"},
Outputs: map[string]string{"p2": "p2"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Inputs: map[string]string{"i0": "p1"},
Outputs: map[string]string{"i0": "p2"},
- }.Build(),
+ },
},
},
{
name: "closed composite",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2", "3"},
- }.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "2": {UniqueName: "b", Outputs: map[string]string{"i0": "p1"}},
+ "3": {UniqueName: "c", Inputs: map[string]string{"i0": "p1"}},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2", "3"},
- }.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "2": {UniqueName: "b", Outputs: map[string]string{"i0": "p1"}},
+ "3": {UniqueName: "c", Inputs: map[string]string{"i0": "p1"}},
},
},
{
name: "nested composites",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"3", "7", "8"},
- }.Build(),
- "3": pipepb.PTransform_builder{
+ },
+ "3": {
UniqueName: "c",
Subtransforms: []string{"4", "5", "6"},
- }.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}}.Build(),
- "6": pipepb.PTransform_builder{UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}}.Build(),
- "7": pipepb.PTransform_builder{UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}}.Build(),
- "8": pipepb.PTransform_builder{UniqueName: "h", Inputs: map[string]string{"i0": "p7"}}.Build(),
+ },
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}},
+ "6": {UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}},
+ "7": {UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}},
+ "8": {UniqueName: "h", Inputs: map[string]string{"i0": "p7"}},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
Inputs: map[string]string{"p1": "p1", "p5": "p5", "p8": "p8"},
Outputs: map[string]string{"p3": "p3"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"3", "7", "8"},
Inputs: map[string]string{"p1": "p1", "p5": "p5", "p8": "p8"},
Outputs: map[string]string{"p3": "p3"},
- }.Build(),
- "3": pipepb.PTransform_builder{
+ },
+ "3": {
UniqueName: "c",
Subtransforms: []string{"4", "6", "5"}, // topologically sorted.
Inputs: map[string]string{"p1": "p1", "p5": "p5"},
Outputs: map[string]string{"p4": "p4", "p6": "p6", "p3": "p3"},
- }.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}}.Build(),
- "6": pipepb.PTransform_builder{UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}}.Build(),
- "7": pipepb.PTransform_builder{UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}}.Build(),
- "8": pipepb.PTransform_builder{UniqueName: "h", Inputs: map[string]string{"i0": "p7"}}.Build(),
+ },
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}},
+ "6": {UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}},
+ "7": {UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}},
+ "8": {UniqueName: "h", Inputs: map[string]string{"i0": "p7"}},
},
}, {
name: "sibling composite",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"3", "4"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"5"},
- }.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "3": {UniqueName: "c", Outputs: map[string]string{"i0": "p1"}},
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p1"}},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"3", "4"},
Outputs: map[string]string{"p1": "p1"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"5"},
Inputs: map[string]string{"p1": "p1"},
- }.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "3": {UniqueName: "c", Outputs: map[string]string{"i0": "p1"}},
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p1"}},
},
},
}
@@ -317,15 +317,15 @@ func BenchmarkComputeInputOutput(b *testing.B) {
for i := 0; i < 3000; i++ {
compositeID := fmt.Sprintf("x%d", i)
primitiveID := fmt.Sprintf("y%d", i)
- in[compositeID] = pipepb.PTransform_builder{
+ in[compositeID] = &pipepb.PTransform{
UniqueName: compositeID,
Subtransforms: []string{primitiveID},
- }.Build()
- in[primitiveID] = pipepb.PTransform_builder{
+ }
+ in[primitiveID] = &pipepb.PTransform{
UniqueName: primitiveID,
Inputs: map[string]string{"i0": fmt.Sprintf("p%d", i)},
Outputs: map[string]string{"i0": fmt.Sprintf("p%d", i+1)},
- }.Build()
+ }
}
b.ResetTimer()
b.ReportAllocs()
@@ -395,11 +395,11 @@ func TestApplySdkImageOverrides(t *testing.T) {
wantEnvs[id] = env
}
- p := pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ p := &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: envs,
- }.Build(),
- }.Build()
+ },
+ }
if err := ApplySdkImageOverrides(p, test.patterns); err != nil {
t.Fatalf("ApplySdkImageOverrides failed: %v", err)
}
@@ -412,16 +412,16 @@ func TestApplySdkImageOverrides(t *testing.T) {
func buildEnvironment(t *testing.T, containerImg string) *pipepb.Environment {
t.Helper()
- env := pipepb.Environment_builder{
+ env := &pipepb.Environment{
Urn: "alpha",
- DisplayData: []*pipepb.DisplayData{pipepb.DisplayData_builder{Urn: "beta"}.Build()},
+ DisplayData: []*pipepb.DisplayData{{Urn: "beta"}},
Capabilities: []string{"delta", "gamma"},
- }.Build()
- pl := pipepb.DockerPayload_builder{ContainerImage: containerImg}.Build()
- plb, err := proto.Marshal(pl)
+ }
+ pl := pipepb.DockerPayload{ContainerImage: containerImg}
+ plb, err := proto.Marshal(&pl)
if err != nil {
t.Fatalf("Failed to marshal DockerPayload with container image %v: %v", containerImg, err)
}
- env.SetPayload(plb)
+ env.Payload = plb
return env
}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/util.go b/sdks/go/pkg/beam/core/runtime/pipelinex/util.go
index 8ba87ff1e65..4735e7b77d2 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/util.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/util.go
@@ -25,7 +25,7 @@ import (
// Bounded returns true iff all PCollections are bounded.
func Bounded(p *pipepb.Pipeline) bool {
for _, col := range p.GetComponents().GetPcollections() {
- if col.GetIsBounded() == pipepb.IsBounded_UNBOUNDED {
+ if col.IsBounded == pipepb.IsBounded_UNBOUNDED {
return false
}
}
@@ -41,7 +41,7 @@ func ContainerImages(p *pipepb.Pipeline) []string {
for _, t := range p.GetComponents().GetEnvironments() {
var payload pipepb.DockerPayload
proto.Unmarshal(t.GetPayload(), &payload)
- ret = append(ret, payload.GetContainerImage())
+ ret = append(ret, payload.ContainerImage)
}
return ret
}
@@ -75,7 +75,7 @@ func newVisiter(xforms map[string]*pipepb.PTransform, ids []string) *visiter {
next: make(map[string][]string),
}
for _, id := range ids {
- for _, in := range xforms[id].GetInputs() {
+ for _, in := range xforms[id].Inputs {
ret.next[in] = append(ret.next[in], id)
}
}
@@ -91,8 +91,8 @@ func (v *visiter) visit(xforms map[string]*pipepb.PTransform, id string) {
}
v.seen[id] = true
// Deterministically iterate through the output keys.
- outputKeys := make([]string, 0, len(xforms[id].GetOutputs()))
- for _, k := range xforms[id].GetOutputs() {
+ outputKeys := make([]string, 0, len(xforms[id].Outputs))
+ for _, k := range xforms[id].Outputs {
outputKeys = append(outputKeys, k)
}
sort.Strings(outputKeys)
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go b/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go
index 795e0fb7679..bdadd669629 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go
@@ -145,23 +145,23 @@ func ptSink(input string, sides ...string) *pipepb.PTransform {
for i, s := range sides {
ins[fmt.Sprintf("i%d", i+1)] = s
}
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Inputs: ins,
- }.Build()
+ }
}
// ptComp generates a composite PCollection. Unlike the other helpers, it takes in
// the *subtransform ids* instead of the input/output pcollections.
func ptComp(subs ...string) *pipepb.PTransform {
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Subtransforms: subs,
- }.Build()
+ }
}
func ptImpulse(output string) *pipepb.PTransform {
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Outputs: map[string]string{"o0": output},
- }.Build()
+ }
}
func ptNoSide(input string, outputs ...string) *pipepb.PTransform {
@@ -169,10 +169,10 @@ func ptNoSide(input string, outputs ...string) *pipepb.PTransform {
for i, o := range outputs {
outs[fmt.Sprintf("o%d", i)] = o
}
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Inputs: map[string]string{"i0": input},
Outputs: outs,
- }.Build()
+ }
}
// validateSortForTest ensures that in the sorted ids, outputs are declared before
@@ -186,7 +186,7 @@ func validateSortForTest(t *testing.T, xforms map[string]*pipepb.PTransform, sor
for _, id := range sorted {
pt := xforms[id]
- for _, in := range pt.GetInputs() {
+ for _, in := range pt.Inputs {
if !seenPCollections[in] {
t.Errorf("out of order pcollection %v required by %v", in, id)
missingPCollections[in] = true
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/expand.go b/sdks/go/pkg/beam/core/runtime/xlangx/expand.go
index 4aae6062bbc..94dda75e805 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/expand.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/expand.go
@@ -62,7 +62,7 @@ func Expand(edge *graph.MultiEdge, ext *graph.ExternalTransform) error {
// should be removed from proto before submitting expansion request.
extTransformID := p.GetRootTransformIds()[0]
extTransform := transforms[extTransformID]
- for extTransform.GetUniqueName() != "External" {
+ for extTransform.UniqueName != "External" {
delete(transforms, extTransformID)
p, err = pipelinex.Normalize(p) // Update root transform IDs.
if err != nil {
@@ -136,12 +136,12 @@ func expand(
}
return h(ctx, &HandlerParams{
Config: config,
- Req: jobpb.ExpansionRequest_builder{
+ Req: &jobpb.ExpansionRequest{
Components: comps,
Transform: transform,
Namespace: ext.Namespace,
OutputCoderRequests: outputCoderID,
- }.Build(),
+ },
edge: edge,
ext: ext,
})
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go b/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go
index 20d2acc7eb5..723d2e25cce 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go
@@ -27,27 +27,27 @@ func addCoderID(c *pipepb.Components, idMap map[string]string, cid string, newID
return idMap[cid]
}
- coder, exists := c.GetCoders()[cid]
+ coder, exists := c.Coders[cid]
if !exists {
- panic(errors.Errorf("attempted to add namespace to missing coder id: %v not in %v", cid, c.GetCoders()))
+ panic(errors.Errorf("attempted to add namespace to missing coder id: %v not in %v", cid, c.Coders))
}
// Updating ComponentCoderIDs of Coder
if coder.GetComponentCoderIds() != nil {
var updatedComponentCoderIDs []string
- updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.GetComponentCoderIds()...)
+ updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.ComponentCoderIds...)
- for i, ccid := range coder.GetComponentCoderIds() {
+ for i, ccid := range coder.ComponentCoderIds {
updatedComponentCoderIDs[i] = addCoderID(c, idMap, ccid, newID)
}
- coder.SetComponentCoderIds(updatedComponentCoderIDs)
+ coder.ComponentCoderIds = updatedComponentCoderIDs
}
idMap[cid] = newID(cid)
// Updating Coders map
- c.GetCoders()[idMap[cid]] = coder
- delete(c.GetCoders(), cid)
+ c.Coders[idMap[cid]] = coder
+ delete(c.Coders, cid)
return idMap[cid]
}
@@ -57,21 +57,21 @@ func addWindowingStrategyID(c *pipepb.Components, idMap map[string]string, wid s
return idMap[wid]
}
- windowingStrategy, exists := c.GetWindowingStrategies()[wid]
+ windowingStrategy, exists := c.WindowingStrategies[wid]
if !exists {
- panic(errors.Errorf("attempted to add namespace to missing windowing strategy id: %v not in %v", wid, c.GetWindowingStrategies()))
+ panic(errors.Errorf("attempted to add namespace to missing windowing strategy id: %v not in %v", wid, c.WindowingStrategies))
}
// Updating WindowCoderID of WindowingStrategy
- if windowingStrategy.GetWindowCoderId() != "" {
- windowingStrategy.SetWindowCoderId(addCoderID(c, idMap, windowingStrategy.GetWindowCoderId(), newID))
+ if windowingStrategy.WindowCoderId != "" {
+ windowingStrategy.WindowCoderId = addCoderID(c, idMap, windowingStrategy.WindowCoderId, newID)
}
idMap[wid] = newID(wid)
// Updating WindowingStrategies map
- c.GetWindowingStrategies()[idMap[wid]] = windowingStrategy
- delete(c.GetWindowingStrategies(), wid)
+ c.WindowingStrategies[idMap[wid]] = windowingStrategy
+ delete(c.WindowingStrategies, wid)
return idMap[wid]
}
@@ -88,14 +88,14 @@ func addNamespace(t *pipepb.PTransform, c *pipepb.Components, namespace string)
// environment. If multiple Go SDK environments become possible, then
// namespacing of non-default environments should happen here.
- for _, pcolsMap := range []map[string]string{t.GetInputs(), t.GetOutputs()} {
+ for _, pcolsMap := range []map[string]string{t.Inputs, t.Outputs} {
for _, pid := range pcolsMap {
- if pcol, exists := c.GetPcollections()[pid]; exists {
+ if pcol, exists := c.Pcollections[pid]; exists {
// Update Coder ID of PCollection
- pcol.SetCoderId(addCoderID(c, idMap, pcol.GetCoderId(), newID))
+ pcol.CoderId = addCoderID(c, idMap, pcol.CoderId, newID)
// Update WindowingStrategyID of PCollection
- pcol.SetWindowingStrategyId(addWindowingStrategyID(c, idMap, pcol.GetWindowingStrategyId(), newID))
+ pcol.WindowingStrategyId = addWindowingStrategyID(c, idMap, pcol.WindowingStrategyId, newID)
}
}
}
@@ -106,32 +106,32 @@ func addNamespace(t *pipepb.PTransform, c *pipepb.Components, namespace string)
continue
}
var updatedComponentCoderIDs []string
- updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.GetComponentCoderIds()...)
+ updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.ComponentCoderIds...)
for i, ccid := range coder.GetComponentCoderIds() {
if _, exists := idMap[ccid]; exists {
updatedComponentCoderIDs[i] = idMap[ccid]
}
}
- coder.SetComponentCoderIds(updatedComponentCoderIDs)
+ coder.ComponentCoderIds = updatedComponentCoderIDs
}
- sourceName := t.GetUniqueName()
- for _, t := range c.GetTransforms() {
- if t.GetUniqueName() != sourceName {
- if id, exists := idMap[t.GetEnvironmentId()]; exists {
- t.SetEnvironmentId(id)
+ sourceName := t.UniqueName
+ for _, t := range c.Transforms {
+ if t.UniqueName != sourceName {
+ if id, exists := idMap[t.EnvironmentId]; exists {
+ t.EnvironmentId = id
}
- for _, pcolsMap := range []map[string]string{t.GetInputs(), t.GetOutputs()} {
+ for _, pcolsMap := range []map[string]string{t.Inputs, t.Outputs} {
for _, pid := range pcolsMap {
- if pcol, exists := c.GetPcollections()[pid]; exists {
+ if pcol, exists := c.Pcollections[pid]; exists {
// Update Coder ID of PCollection
- if id, exists := idMap[pcol.GetCoderId()]; exists {
- pcol.SetCoderId(id)
+ if id, exists := idMap[pcol.CoderId]; exists {
+ pcol.CoderId = id
}
// Update WindowingStrategyID of PCollection
- if id, exists := idMap[pcol.GetWindowingStrategyId()]; exists {
- pcol.SetWindowingStrategyId(id)
+ if id, exists := idMap[pcol.WindowingStrategyId]; exists {
+ pcol.WindowingStrategyId = id
}
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go b/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go
index eb98d0d7e7d..e2ee43780bc 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go
@@ -41,515 +41,515 @@ func TestAddNamespace(t *testing.T) {
}{
{
name: "[Correctness] Single Input Multi Output",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p1": {CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Correctness] Single Input Single Output",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t1",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p1": {CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p3": {CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Correctness] Multi Input Single Output",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t3",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p3": {CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p4": {CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"},
+ "p5": {CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0@daASxQwenJ": {WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Correctness] Component Coders",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build(), ComponentCoderIds: []string{"c2"}}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}, ComponentCoderIds: []string{"c2"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"},
+ "p1": {CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0@daASxQwenJ": {WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build(), ComponentCoderIds: []string{"c2@daASxQwenJ"}}.Build(),
- "c1@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c0"}, ComponentCoderIds: []string{"c2@daASxQwenJ"}},
+ "c1@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Consistency] Missing EnvironmentID",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
},
Environments: map[string]*pipepb.Environment{
// Missing "e0"
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
err: "trying to add an Environment whose key is absent should panic",
},
{
name: "[Consistency] Missing WindowingStrategyID",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
// Missing w0
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
err: "trying to add an WindowingStrategy whose key is absent should panic",
},
{
name: "[Consistency] Missing CoderID",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
// Missing c0
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
err: "trying to add an WindowingStrategy whose key is absent should panic",
@@ -559,7 +559,7 @@ func TestAddNamespace(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if strings.Contains(tt.name, "Correctness") {
- transform := tt.init.GetTransforms()[tt.transformID]
+ transform := tt.init.Transforms[tt.transformID]
addNamespace(transform, tt.init, tt.namespace)
@@ -570,7 +570,7 @@ func TestAddNamespace(t *testing.T) {
if strings.Contains(tt.name, "Consistency") {
defer expectPanic(t, tt.err)
- transform := tt.init.GetTransforms()[tt.transformID]
+ transform := tt.init.Transforms[tt.transformID]
addNamespace(transform, tt.init, tt.namespace)
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/payload.go b/sdks/go/pkg/beam/core/runtime/xlangx/payload.go
index fcab9cd769a..8c01ffbd34f 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/payload.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/payload.go
@@ -52,10 +52,10 @@ func CreateExternalConfigurationPayload(pl any) (*pipepb.ExternalConfigurationPa
}
// Put schema and row into payload proto, and marshal it.
- ecp := pipepb.ExternalConfigurationPayload_builder{
+ ecp := &pipepb.ExternalConfigurationPayload{
Schema: scm,
Payload: buf.Bytes(),
- }.Build()
+ }
return ecp, nil
}
@@ -102,7 +102,7 @@ func DecodeStructPayload(plBytes []byte) (any, error) {
err = errors.WithContextf(err, "creating Row decoder for type %v", rt)
return nil, errors.WithContext(err, "decoding external payload")
}
- buf := bytes.NewBuffer(ecp.GetPayload())
+ buf := bytes.NewBuffer(ecp.Payload)
val, err := dec(buf)
if err != nil {
err = errors.WithContext(err, "decoding Row to payload")
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/registry.go b/sdks/go/pkg/beam/core/runtime/xlangx/registry.go
index d796dee5a15..1957cc61bb9 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/registry.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/registry.go
@@ -78,7 +78,7 @@ type HandlerParams struct {
// CoderMarshaller returns a coder marshaller initialized with the request's namespace.
func (p *HandlerParams) CoderMarshaller() *graphx.CoderMarshaller {
cm := graphx.NewCoderMarshaller()
- cm.Namespace = p.Req.GetNamespace()
+ cm.Namespace = p.Req.Namespace
return cm
}
@@ -134,7 +134,7 @@ func makePCol(node *graph.Node, index int, local, namespace string) PCol {
func (p *HandlerParams) Outputs() []PCol {
out := make([]PCol, 0, len(p.ext.OutputsMap))
for local, i := range p.ext.OutputsMap {
- out = append(out, makePCol(p.edge.Output[i].To, i, local, p.Req.GetNamespace()))
+ out = append(out, makePCol(p.edge.Output[i].To, i, local, p.Req.Namespace))
}
return out
}
@@ -144,7 +144,7 @@ func (p *HandlerParams) Outputs() []PCol {
func (p *HandlerParams) Inputs() []PCol {
out := make([]PCol, 0, len(p.ext.InputsMap))
for local, i := range p.ext.InputsMap {
- out = append(out, makePCol(p.edge.Input[i].From, i, local, p.Req.GetNamespace()))
+ out = append(out, makePCol(p.edge.Input[i].From, i, local, p.Req.Namespace))
}
return out
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go b/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go
index a62025039ac..aabe6eface4 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go
@@ -87,7 +87,7 @@ func ResolveArtifactsWithConfig(ctx context.Context, edges []*graph.MultiEdge, c
return nil, errors.WithContextf(err,
"resolving remote artifacts for edge %v", e.Name())
}
- envs := components.GetEnvironments()
+ envs := components.Environments
for eid, env := range envs {
if strings.HasPrefix(eid, "go") {
continue
@@ -111,21 +111,21 @@ func ResolveArtifactsWithConfig(ctx context.Context, edges []*graph.MultiEdge, c
fullSdkPath = cfg.JoinFn(cfg.SdkPath, name)
}
resolvedDeps = append(resolvedDeps,
- pipepb.ArtifactInformation_builder{
+ &pipepb.ArtifactInformation{
TypeUrn: "beam:artifact:type:file:v1",
TypePayload: protox.MustEncode(
- pipepb.ArtifactFilePayload_builder{
+ &pipepb.ArtifactFilePayload{
Path: fullSdkPath,
Sha256: sha256,
- }.Build(),
+ },
),
- RoleUrn: a.GetRoleUrn(),
- RolePayload: a.GetRolePayload(),
- }.Build(),
+ RoleUrn: a.RoleUrn,
+ RolePayload: a.RolePayload,
+ },
)
paths[fullTmpPath] = fullSdkPath
}
- env.SetDependencies(resolvedDeps)
+ env.Dependencies = resolvedDeps
}
}
}
@@ -142,24 +142,24 @@ func UpdateArtifactTypeFromFileToURL(edges []*graph.MultiEdge) {
panic(errors.WithContextf(err,
"updating URL artifacts type for edge %v", e.Name()))
}
- envs := components.GetEnvironments()
+ envs := components.Environments
for _, env := range envs {
deps := env.GetDependencies()
var resolvedDeps []*pipepb.ArtifactInformation
for _, a := range deps {
path, sha256 := artifact.MustExtractFilePayload(a)
if strings.Contains(path, "://") {
- a.SetTypeUrn("beam:artifact:type:url:v1")
- a.SetTypePayload(protox.MustEncode(
- pipepb.ArtifactUrlPayload_builder{
+ a.TypeUrn = "beam:artifact:type:url:v1"
+ a.TypePayload = protox.MustEncode(
+ &pipepb.ArtifactUrlPayload{
Url: path,
Sha256: sha256,
- }.Build(),
- ))
+ },
+ )
}
resolvedDeps = append(resolvedDeps, a)
}
- env.SetDependencies(resolvedDeps)
+ env.Dependencies = resolvedDeps
}
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go b/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go
index 1693ad14b22..eec13c451a1 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go
@@ -25,30 +25,30 @@ import (
func createExternalEdge(typeUrn string, typePayload []byte) *graph.MultiEdge {
env := map[string]*pipepb.Environment{
- "env_java": pipepb.Environment_builder{
+ "env_java": {
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: typeUrn,
TypePayload: typePayload,
- }.Build(),
+ },
},
- }.Build(),
+ },
}
return &graph.MultiEdge{
Op: graph.External,
External: &graph.ExternalTransform{
Expanded: &graph.ExpandedTransform{
- Components: pipepb.Components_builder{
+ Components: &pipepb.Components{
Environments: env,
- }.Build(),
+ },
},
},
}
}
func TestUpdateFileArtifactWithUrlPath(t *testing.T) {
- payload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: "gs://dummy"}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: "gs://dummy"})
e := createExternalEdge("beam:artifact:type:file:v1", payload)
UpdateArtifactTypeFromFileToURL([]*graph.MultiEdge{e})
expected := createExternalEdge("beam:artifact:type:url:v1", payload)
@@ -63,8 +63,8 @@ func TestUpdateFileArtifactWithUrlPath(t *testing.T) {
}
func TestUpdateFileArtifactWithLocalPath(t *testing.T) {
- payload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: "/tmp/artifact/dummy"}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: "/tmp/artifact/dummy"})
e := createExternalEdge("beam:artifact:type:file:v1", payload)
UpdateArtifactTypeFromFileToURL([]*graph.MultiEdge{e})
expected := createExternalEdge("beam:artifact:type:file:v1", payload)
diff --git a/sdks/go/pkg/beam/io/pubsubio/pubsubio.go b/sdks/go/pkg/beam/io/pubsubio/pubsubio.go
index 8eef70b7f04..bcf20e89c89 100644
--- a/sdks/go/pkg/beam/io/pubsubio/pubsubio.go
+++ b/sdks/go/pkg/beam/io/pubsubio/pubsubio.go
@@ -64,16 +64,16 @@ type ReadOptions struct {
func Read(s beam.Scope, project, topic string, opts *ReadOptions) beam.PCollection {
s = s.Scope("pubsubio.Read")
- payload := pipepb.PubSubReadPayload_builder{
+ payload := &pipepb.PubSubReadPayload{
Topic: pubsubx.MakeQualifiedTopicName(project, topic),
- }.Build()
+ }
if opts != nil {
- payload.SetIdAttribute(opts.IDAttribute)
- payload.SetTimestampAttribute(opts.TimestampAttribute)
+ payload.IdAttribute = opts.IDAttribute
+ payload.TimestampAttribute = opts.TimestampAttribute
if opts.Subscription != "" {
- payload.SetSubscription(pubsubx.MakeQualifiedSubscriptionName(project, opts.Subscription))
+ payload.Subscription = pubsubx.MakeQualifiedSubscriptionName(project, opts.Subscription)
}
- payload.SetWithAttributes(opts.WithAttributes)
+ payload.WithAttributes = opts.WithAttributes
}
out := beam.External(s, readURN, protox.MustEncode(payload), nil, []beam.FullType{typex.New(reflectx.ByteSlice)}, false)
@@ -118,9 +118,9 @@ var pubSubMessageT = reflect.TypeOf((*pb.PubsubMessage)(nil))
func Write(s beam.Scope, project, topic string, col beam.PCollection) {
s = s.Scope("pubsubio.Write")
- payload := pipepb.PubSubWritePayload_builder{
+ payload := &pipepb.PubSubWritePayload{
Topic: pubsubx.MakeQualifiedTopicName(project, topic),
- }.Build()
+ }
out := col
if col.Type().Type() == reflectx.ByteSlice {
diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
index 3eb826fae93..55ec131df06 100644
--- a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
+++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
@@ -26,15 +26,13 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
-// protoc-gen-go v1.36.0
+// protoc-gen-go v1.27.1
// protoc v5.27.3
// source: org/apache/beam/model/fn_execution/v1/beam_fn_api.proto
// TODO: Consider consolidating common components in another package
// and language namespaces for re-use with Runner Api.
-//go:build !protoopaque
-
package fnexecution_v1
import (
@@ -46,6 +44,7 @@ import (
structpb "google.golang.org/protobuf/types/known/structpb"
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
reflect "reflect"
+ sync "sync"
)
const (
@@ -131,6 +130,11 @@ func (x FnApiTransforms_Runner) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
+// Deprecated: Use FnApiTransforms_Runner.Descriptor instead.
+func (FnApiTransforms_Runner) EnumDescriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{0, 0}
+}
+
type LogEntry_Severity_Enum int32
const (
@@ -199,19 +203,26 @@ func (x LogEntry_Severity_Enum) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
+// Deprecated: Use LogEntry_Severity_Enum.Descriptor instead.
+func (LogEntry_Severity_Enum) EnumDescriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{36, 1, 0}
+}
+
// Describes transforms necessary to execute Beam over the FnAPI but are
// implementation details rather than part of the core model.
type FnApiTransforms struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- unknownFields protoimpl.UnknownFields
+ state protoimpl.MessageState
sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
}
func (x *FnApiTransforms) Reset() {
*x = FnApiTransforms{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *FnApiTransforms) String() string {
@@ -222,7 +233,7 @@ func (*FnApiTransforms) ProtoMessage() {}
func (x *FnApiTransforms) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -232,16 +243,9 @@ func (x *FnApiTransforms) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-type FnApiTransforms_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
-}
-
-func (b0 FnApiTransforms_builder) Build() *FnApiTransforms {
- m0 := &FnApiTransforms{}
- b, x := &b0, m0
- _, _ = b, x
- return m0
+// Deprecated: Use FnApiTransforms.ProtoReflect.Descriptor instead.
+func (*FnApiTransforms) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{0}
}
// A descriptor for connecting to a remote port using the Beam Fn Data API.
@@ -249,22 +253,25 @@ func (b0 FnApiTransforms_builder) Build() *FnApiTransforms {
// runner and the SDK).
// Stable
type RemoteGrpcPort struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) An API descriptor which describes where to
// connect to including any authentication that is required.
ApiServiceDescriptor *pipeline_v1.ApiServiceDescriptor `protobuf:"bytes,1,opt,name=api_service_descriptor,json=apiServiceDescriptor,proto3" json:"api_service_descriptor,omitempty"`
// (Required) The ID of the Coder that will be used to encode and decode data
// sent over this port.
- CoderId string `protobuf:"bytes,2,opt,name=coder_id,json=coderId,proto3" json:"coder_id,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ CoderId string `protobuf:"bytes,2,opt,name=coder_id,json=coderId,proto3" json:"coder_id,omitempty"`
}
func (x *RemoteGrpcPort) Reset() {
*x = RemoteGrpcPort{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *RemoteGrpcPort) String() string {
@@ -275,7 +282,7 @@ func (*RemoteGrpcPort) ProtoMessage() {}
func (x *RemoteGrpcPort) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -285,6 +292,11 @@ func (x *RemoteGrpcPort) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use RemoteGrpcPort.ProtoReflect.Descriptor instead.
+func (*RemoteGrpcPort) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{1}
+}
+
func (x *RemoteGrpcPort) GetApiServiceDescriptor() *pipeline_v1.ApiServiceDescriptor {
if x != nil {
return x.ApiServiceDescriptor
@@ -299,58 +311,22 @@ func (x *RemoteGrpcPort) GetCoderId() string {
return ""
}
-func (x *RemoteGrpcPort) SetApiServiceDescriptor(v *pipeline_v1.ApiServiceDescriptor) {
- x.ApiServiceDescriptor = v
-}
-
-func (x *RemoteGrpcPort) SetCoderId(v string) {
- x.CoderId = v
-}
-
-func (x *RemoteGrpcPort) HasApiServiceDescriptor() bool {
- if x == nil {
- return false
- }
- return x.ApiServiceDescriptor != nil
-}
-
-func (x *RemoteGrpcPort) ClearApiServiceDescriptor() {
- x.ApiServiceDescriptor = nil
-}
-
-type RemoteGrpcPort_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) An API descriptor which describes where to
- // connect to including any authentication that is required.
- ApiServiceDescriptor *pipeline_v1.ApiServiceDescriptor
- // (Required) The ID of the Coder that will be used to encode and decode data
- // sent over this port.
- CoderId string
-}
-
-func (b0 RemoteGrpcPort_builder) Build() *RemoteGrpcPort {
- m0 := &RemoteGrpcPort{}
- b, x := &b0, m0
- _, _ = b, x
- x.ApiServiceDescriptor = b.ApiServiceDescriptor
- x.CoderId = b.CoderId
- return m0
-}
-
// Requests the ProcessBundleDescriptor with the given id.
type GetProcessBundleDescriptorRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- ProcessBundleDescriptorId string `protobuf:"bytes,1,opt,name=process_bundle_descriptor_id,json=processBundleDescriptorId,proto3" json:"process_bundle_descriptor_id,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ ProcessBundleDescriptorId string `protobuf:"bytes,1,opt,name=process_bundle_descriptor_id,json=processBundleDescriptorId,proto3" json:"process_bundle_descriptor_id,omitempty"`
}
func (x *GetProcessBundleDescriptorRequest) Reset() {
*x = GetProcessBundleDescriptorRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *GetProcessBundleDescriptorRequest) String() string {
@@ -361,7 +337,7 @@ func (*GetProcessBundleDescriptorRequest) ProtoMessage() {}
func (x *GetProcessBundleDescriptorRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -371,6 +347,11 @@ func (x *GetProcessBundleDescriptorRequest) ProtoReflect() protoreflect.Message
return mi.MessageOf(x)
}
+// Deprecated: Use GetProcessBundleDescriptorRequest.ProtoReflect.Descriptor instead.
+func (*GetProcessBundleDescriptorRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{2}
+}
+
func (x *GetProcessBundleDescriptorRequest) GetProcessBundleDescriptorId() string {
if x != nil {
return x.ProcessBundleDescriptorId
@@ -378,36 +359,21 @@ func (x *GetProcessBundleDescriptorRequest) GetProcessBundleDescriptorId() strin
return ""
}
-func (x *GetProcessBundleDescriptorRequest) SetProcessBundleDescriptorId(v string) {
- x.ProcessBundleDescriptorId = v
-}
-
-type GetProcessBundleDescriptorRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- ProcessBundleDescriptorId string
-}
-
-func (b0 GetProcessBundleDescriptorRequest_builder) Build() *GetProcessBundleDescriptorRequest {
- m0 := &GetProcessBundleDescriptorRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.ProcessBundleDescriptorId = b.ProcessBundleDescriptorId
- return m0
-}
-
// A request sent by a runner which the SDK is asked to fulfill.
// For any unsupported request type, an error should be returned with a
// matching instruction id.
// Stable
type InstructionRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) A unique identifier provided by the runner which represents
// this requests execution. The InstructionResponse MUST have the matching id.
InstructionId string `protobuf:"bytes,1,opt,name=instruction_id,json=instructionId,proto3" json:"instruction_id,omitempty"`
// (Required) A request that the SDK Harness needs to interpret.
//
- // Types that are valid to be assigned to Request:
+ // Types that are assignable to Request:
//
// *InstructionRequest_ProcessBundle
// *InstructionRequest_ProcessBundleProgress
@@ -417,16 +383,16 @@ type InstructionRequest struct {
// *InstructionRequest_HarnessMonitoringInfos
// *InstructionRequest_SampleData
// *InstructionRequest_Register
- Request isInstructionRequest_Request `protobuf_oneof:"request"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ Request isInstructionRequest_Request `protobuf_oneof:"request"`
}
func (x *InstructionRequest) Reset() {
*x = InstructionRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *InstructionRequest) String() string {
@@ -437,7 +403,7 @@ func (*InstructionRequest) ProtoMessage() {}
func (x *InstructionRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -447,6 +413,11 @@ func (x *InstructionRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use InstructionRequest.ProtoReflect.Descriptor instead.
+func (*InstructionRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{3}
+}
+
func (x *InstructionRequest) GetInstructionId() string {
if x != nil {
return x.InstructionId
@@ -454,375 +425,69 @@ func (x *InstructionRequest) GetInstructionId() string {
return ""
}
-func (x *InstructionRequest) GetRequest() isInstructionRequest_Request {
- if x != nil {
- return x.Request
+func (m *InstructionRequest) GetRequest() isInstructionRequest_Request {
+ if m != nil {
+ return m.Request
}
return nil
}
func (x *InstructionRequest) GetProcessBundle() *ProcessBundleRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_ProcessBundle); ok {
- return x.ProcessBundle
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_ProcessBundle); ok {
+ return x.ProcessBundle
}
return nil
}
func (x *InstructionRequest) GetProcessBundleProgress() *ProcessBundleProgressRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_ProcessBundleProgress); ok {
- return x.ProcessBundleProgress
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_ProcessBundleProgress); ok {
+ return x.ProcessBundleProgress
}
return nil
}
func (x *InstructionRequest) GetProcessBundleSplit() *ProcessBundleSplitRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_ProcessBundleSplit); ok {
- return x.ProcessBundleSplit
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_ProcessBundleSplit); ok {
+ return x.ProcessBundleSplit
}
return nil
}
func (x *InstructionRequest) GetFinalizeBundle() *FinalizeBundleRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_FinalizeBundle); ok {
- return x.FinalizeBundle
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_FinalizeBundle); ok {
+ return x.FinalizeBundle
}
return nil
}
func (x *InstructionRequest) GetMonitoringInfos() *MonitoringInfosMetadataRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_MonitoringInfos); ok {
- return x.MonitoringInfos
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_MonitoringInfos); ok {
+ return x.MonitoringInfos
}
return nil
}
func (x *InstructionRequest) GetHarnessMonitoringInfos() *HarnessMonitoringInfosRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_HarnessMonitoringInfos); ok {
- return x.HarnessMonitoringInfos
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_HarnessMonitoringInfos); ok {
+ return x.HarnessMonitoringInfos
}
return nil
}
func (x *InstructionRequest) GetSampleData() *SampleDataRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_SampleData); ok {
- return x.SampleData
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_SampleData); ok {
+ return x.SampleData
}
return nil
}
func (x *InstructionRequest) GetRegister() *RegisterRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_Register); ok {
- return x.Register
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_Register); ok {
+ return x.Register
}
return nil
}
-func (x *InstructionRequest) SetInstructionId(v string) {
- x.InstructionId = v
-}
-
-func (x *InstructionRequest) SetProcessBundle(v *ProcessBundleRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_ProcessBundle{v}
-}
-
-func (x *InstructionRequest) SetProcessBundleProgress(v *ProcessBundleProgressRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_ProcessBundleProgress{v}
-}
-
-func (x *InstructionRequest) SetProcessBundleSplit(v *ProcessBundleSplitRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_ProcessBundleSplit{v}
-}
-
-func (x *InstructionRequest) SetFinalizeBundle(v *FinalizeBundleRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_FinalizeBundle{v}
-}
-
-func (x *InstructionRequest) SetMonitoringInfos(v *MonitoringInfosMetadataRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_MonitoringInfos{v}
-}
-
-func (x *InstructionRequest) SetHarnessMonitoringInfos(v *HarnessMonitoringInfosRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_HarnessMonitoringInfos{v}
-}
-
-func (x *InstructionRequest) SetSampleData(v *SampleDataRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_SampleData{v}
-}
-
-func (x *InstructionRequest) SetRegister(v *RegisterRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_Register{v}
-}
-
-func (x *InstructionRequest) HasRequest() bool {
- if x == nil {
- return false
- }
- return x.Request != nil
-}
-
-func (x *InstructionRequest) HasProcessBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_ProcessBundle)
- return ok
-}
-
-func (x *InstructionRequest) HasProcessBundleProgress() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_ProcessBundleProgress)
- return ok
-}
-
-func (x *InstructionRequest) HasProcessBundleSplit() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_ProcessBundleSplit)
- return ok
-}
-
-func (x *InstructionRequest) HasFinalizeBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_FinalizeBundle)
- return ok
-}
-
-func (x *InstructionRequest) HasMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_MonitoringInfos)
- return ok
-}
-
-func (x *InstructionRequest) HasHarnessMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_HarnessMonitoringInfos)
- return ok
-}
-
-func (x *InstructionRequest) HasSampleData() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_SampleData)
- return ok
-}
-
-func (x *InstructionRequest) HasRegister() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_Register)
- return ok
-}
-
-func (x *InstructionRequest) ClearRequest() {
- x.Request = nil
-}
-
-func (x *InstructionRequest) ClearProcessBundle() {
- if _, ok := x.Request.(*InstructionRequest_ProcessBundle); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearProcessBundleProgress() {
- if _, ok := x.Request.(*InstructionRequest_ProcessBundleProgress); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearProcessBundleSplit() {
- if _, ok := x.Request.(*InstructionRequest_ProcessBundleSplit); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearFinalizeBundle() {
- if _, ok := x.Request.(*InstructionRequest_FinalizeBundle); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearMonitoringInfos() {
- if _, ok := x.Request.(*InstructionRequest_MonitoringInfos); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearHarnessMonitoringInfos() {
- if _, ok := x.Request.(*InstructionRequest_HarnessMonitoringInfos); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearSampleData() {
- if _, ok := x.Request.(*InstructionRequest_SampleData); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearRegister() {
- if _, ok := x.Request.(*InstructionRequest_Register); ok {
- x.Request = nil
- }
-}
-
-const InstructionRequest_Request_not_set_case case_InstructionRequest_Request = 0
-const InstructionRequest_ProcessBundle_case case_InstructionRequest_Request = 1001
-const InstructionRequest_ProcessBundleProgress_case case_InstructionRequest_Request = 1002
-const InstructionRequest_ProcessBundleSplit_case case_InstructionRequest_Request = 1003
-const InstructionRequest_FinalizeBundle_case case_InstructionRequest_Request = 1004
-const InstructionRequest_MonitoringInfos_case case_InstructionRequest_Request = 1005
-const InstructionRequest_HarnessMonitoringInfos_case case_InstructionRequest_Request = 1006
-const InstructionRequest_SampleData_case case_InstructionRequest_Request = 1007
-const InstructionRequest_Register_case case_InstructionRequest_Request = 1000
-
-func (x *InstructionRequest) WhichRequest() case_InstructionRequest_Request {
- if x == nil {
- return InstructionRequest_Request_not_set_case
- }
- switch x.Request.(type) {
- case *InstructionRequest_ProcessBundle:
- return InstructionRequest_ProcessBundle_case
- case *InstructionRequest_ProcessBundleProgress:
- return InstructionRequest_ProcessBundleProgress_case
- case *InstructionRequest_ProcessBundleSplit:
- return InstructionRequest_ProcessBundleSplit_case
- case *InstructionRequest_FinalizeBundle:
- return InstructionRequest_FinalizeBundle_case
- case *InstructionRequest_MonitoringInfos:
- return InstructionRequest_MonitoringInfos_case
- case *InstructionRequest_HarnessMonitoringInfos:
- return InstructionRequest_HarnessMonitoringInfos_case
- case *InstructionRequest_SampleData:
- return InstructionRequest_SampleData_case
- case *InstructionRequest_Register:
- return InstructionRequest_Register_case
- default:
- return InstructionRequest_Request_not_set_case
- }
-}
-
-type InstructionRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) A unique identifier provided by the runner which represents
- // this requests execution. The InstructionResponse MUST have the matching id.
- InstructionId string
- // (Required) A request that the SDK Harness needs to interpret.
-
- // Fields of oneof Request:
- ProcessBundle *ProcessBundleRequest
- ProcessBundleProgress *ProcessBundleProgressRequest
- ProcessBundleSplit *ProcessBundleSplitRequest
- FinalizeBundle *FinalizeBundleRequest
- MonitoringInfos *MonitoringInfosMetadataRequest
- HarnessMonitoringInfos *HarnessMonitoringInfosRequest
- SampleData *SampleDataRequest
- // DEPRECATED
- Register *RegisterRequest
- // -- end of Request
-}
-
-func (b0 InstructionRequest_builder) Build() *InstructionRequest {
- m0 := &InstructionRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.InstructionId = b.InstructionId
- if b.ProcessBundle != nil {
- x.Request = &InstructionRequest_ProcessBundle{b.ProcessBundle}
- }
- if b.ProcessBundleProgress != nil {
- x.Request = &InstructionRequest_ProcessBundleProgress{b.ProcessBundleProgress}
- }
- if b.ProcessBundleSplit != nil {
- x.Request = &InstructionRequest_ProcessBundleSplit{b.ProcessBundleSplit}
- }
- if b.FinalizeBundle != nil {
- x.Request = &InstructionRequest_FinalizeBundle{b.FinalizeBundle}
- }
- if b.MonitoringInfos != nil {
- x.Request = &InstructionRequest_MonitoringInfos{b.MonitoringInfos}
- }
- if b.HarnessMonitoringInfos != nil {
- x.Request = &InstructionRequest_HarnessMonitoringInfos{b.HarnessMonitoringInfos}
- }
- if b.SampleData != nil {
- x.Request = &InstructionRequest_SampleData{b.SampleData}
- }
- if b.Register != nil {
- x.Request = &InstructionRequest_Register{b.Register}
- }
- return m0
-}
-
-type case_InstructionRequest_Request protoreflect.FieldNumber
-
-func (x case_InstructionRequest_Request) String() string {
- md := file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3].Descriptor()
- if x == 0 {
- return "not set"
- }
- return protoimpl.X.MessageFieldStringOf(md, protoreflect.FieldNumber(x))
-}
-
type isInstructionRequest_Request interface {
isInstructionRequest_Request()
}
@@ -879,7 +544,10 @@ func (*InstructionRequest_Register) isInstructionRequest_Request() {}
// The response for an associated request the SDK had been asked to fulfill.
// Stable
type InstructionResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) A reference provided by the runner which represents a requests
// execution. The InstructionResponse MUST have the matching id when
// responding to the runner.
@@ -891,7 +559,7 @@ type InstructionResponse struct {
// If the instruction did not fail, it is required to return an equivalent
// response type depending on the request this matches.
//
- // Types that are valid to be assigned to Response:
+ // Types that are assignable to Response:
//
// *InstructionResponse_ProcessBundle
// *InstructionResponse_ProcessBundleProgress
@@ -901,16 +569,16 @@ type InstructionResponse struct {
// *InstructionResponse_HarnessMonitoringInfos
// *InstructionResponse_SampleData
// *InstructionResponse_Register
- Response isInstructionResponse_Response `protobuf_oneof:"response"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ Response isInstructionResponse_Response `protobuf_oneof:"response"`
}
func (x *InstructionResponse) Reset() {
*x = InstructionResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *InstructionResponse) String() string {
@@ -921,7 +589,7 @@ func (*InstructionResponse) ProtoMessage() {}
func (x *InstructionResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -931,6 +599,11 @@ func (x *InstructionResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use InstructionResponse.ProtoReflect.Descriptor instead.
+func (*InstructionResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{4}
+}
+
func (x *InstructionResponse) GetInstructionId() string {
if x != nil {
return x.InstructionId
@@ -945,412 +618,95 @@ func (x *InstructionResponse) GetError() string {
return ""
}
-func (x *InstructionResponse) GetResponse() isInstructionResponse_Response {
- if x != nil {
- return x.Response
+func (m *InstructionResponse) GetResponse() isInstructionResponse_Response {
+ if m != nil {
+ return m.Response
}
return nil
}
func (x *InstructionResponse) GetProcessBundle() *ProcessBundleResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_ProcessBundle); ok {
- return x.ProcessBundle
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_ProcessBundle); ok {
+ return x.ProcessBundle
}
return nil
}
func (x *InstructionResponse) GetProcessBundleProgress() *ProcessBundleProgressResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_ProcessBundleProgress); ok {
- return x.ProcessBundleProgress
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_ProcessBundleProgress); ok {
+ return x.ProcessBundleProgress
}
return nil
}
func (x *InstructionResponse) GetProcessBundleSplit() *ProcessBundleSplitResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_ProcessBundleSplit); ok {
- return x.ProcessBundleSplit
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_ProcessBundleSplit); ok {
+ return x.ProcessBundleSplit
}
return nil
}
func (x *InstructionResponse) GetFinalizeBundle() *FinalizeBundleResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_FinalizeBundle); ok {
- return x.FinalizeBundle
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_FinalizeBundle); ok {
+ return x.FinalizeBundle
}
return nil
}
func (x *InstructionResponse) GetMonitoringInfos() *MonitoringInfosMetadataResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_MonitoringInfos); ok {
- return x.MonitoringInfos
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_MonitoringInfos); ok {
+ return x.MonitoringInfos
}
return nil
}
func (x *InstructionResponse) GetHarnessMonitoringInfos() *HarnessMonitoringInfosResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_HarnessMonitoringInfos); ok {
- return x.HarnessMonitoringInfos
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_HarnessMonitoringInfos); ok {
+ return x.HarnessMonitoringInfos
}
return nil
}
func (x *InstructionResponse) GetSampleData() *SampleDataResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_SampleData); ok {
- return x.SampleData
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_SampleData); ok {
+ return x.SampleData
}
return nil
}
func (x *InstructionResponse) GetRegister() *RegisterResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_Register); ok {
- return x.Register
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_Register); ok {
+ return x.Register
}
return nil
}
-func (x *InstructionResponse) SetInstructionId(v string) {
- x.InstructionId = v
+type isInstructionResponse_Response interface {
+ isInstructionResponse_Response()
}
-func (x *InstructionResponse) SetError(v string) {
- x.Error = v
+type InstructionResponse_ProcessBundle struct {
+ ProcessBundle *ProcessBundleResponse `protobuf:"bytes,1001,opt,name=process_bundle,json=processBundle,proto3,oneof"`
}
-func (x *InstructionResponse) SetProcessBundle(v *ProcessBundleResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_ProcessBundle{v}
+type InstructionResponse_ProcessBundleProgress struct {
+ ProcessBundleProgress *ProcessBundleProgressResponse `protobuf:"bytes,1002,opt,name=process_bundle_progress,json=processBundleProgress,proto3,oneof"`
}
-func (x *InstructionResponse) SetProcessBundleProgress(v *ProcessBundleProgressResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_ProcessBundleProgress{v}
+type InstructionResponse_ProcessBundleSplit struct {
+ ProcessBundleSplit *ProcessBundleSplitResponse `protobuf:"bytes,1003,opt,name=process_bundle_split,json=processBundleSplit,proto3,oneof"`
}
-func (x *InstructionResponse) SetProcessBundleSplit(v *ProcessBundleSplitResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_ProcessBundleSplit{v}
+type InstructionResponse_FinalizeBundle struct {
+ FinalizeBundle *FinalizeBundleResponse `protobuf:"bytes,1004,opt,name=finalize_bundle,json=finalizeBundle,proto3,oneof"`
}
-func (x *InstructionResponse) SetFinalizeBundle(v *FinalizeBundleResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_FinalizeBundle{v}
+type InstructionResponse_MonitoringInfos struct {
+ MonitoringInfos *MonitoringInfosMetadataResponse `protobuf:"bytes,1005,opt,name=monitoring_infos,json=monitoringInfos,proto3,oneof"`
}
-func (x *InstructionResponse) SetMonitoringInfos(v *MonitoringInfosMetadataResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_MonitoringInfos{v}
-}
-
-func (x *InstructionResponse) SetHarnessMonitoringInfos(v *HarnessMonitoringInfosResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_HarnessMonitoringInfos{v}
-}
-
-func (x *InstructionResponse) SetSampleData(v *SampleDataResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_SampleData{v}
-}
-
-func (x *InstructionResponse) SetRegister(v *RegisterResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_Register{v}
-}
-
-func (x *InstructionResponse) HasResponse() bool {
- if x == nil {
- return false
- }
- return x.Response != nil
-}
-
-func (x *InstructionResponse) HasProcessBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_ProcessBundle)
- return ok
-}
-
-func (x *InstructionResponse) HasProcessBundleProgress() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_ProcessBundleProgress)
- return ok
-}
-
-func (x *InstructionResponse) HasProcessBundleSplit() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_ProcessBundleSplit)
- return ok
-}
-
-func (x *InstructionResponse) HasFinalizeBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_FinalizeBundle)
- return ok
-}
-
-func (x *InstructionResponse) HasMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_MonitoringInfos)
- return ok
-}
-
-func (x *InstructionResponse) HasHarnessMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_HarnessMonitoringInfos)
- return ok
-}
-
-func (x *InstructionResponse) HasSampleData() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_SampleData)
- return ok
-}
-
-func (x *InstructionResponse) HasRegister() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_Register)
- return ok
-}
-
-func (x *InstructionResponse) ClearResponse() {
- x.Response = nil
-}
-
-func (x *InstructionResponse) ClearProcessBundle() {
- if _, ok := x.Response.(*InstructionResponse_ProcessBundle); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearProcessBundleProgress() {
- if _, ok := x.Response.(*InstructionResponse_ProcessBundleProgress); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearProcessBundleSplit() {
- if _, ok := x.Response.(*InstructionResponse_ProcessBundleSplit); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearFinalizeBundle() {
- if _, ok := x.Response.(*InstructionResponse_FinalizeBundle); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearMonitoringInfos() {
- if _, ok := x.Response.(*InstructionResponse_MonitoringInfos); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearHarnessMonitoringInfos() {
- if _, ok := x.Response.(*InstructionResponse_HarnessMonitoringInfos); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearSampleData() {
- if _, ok := x.Response.(*InstructionResponse_SampleData); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearRegister() {
- if _, ok := x.Response.(*InstructionResponse_Register); ok {
- x.Response = nil
- }
-}
-
-const InstructionResponse_Response_not_set_case case_InstructionResponse_Response = 0
-const InstructionResponse_ProcessBundle_case case_InstructionResponse_Response = 1001
-const InstructionResponse_ProcessBundleProgress_case case_InstructionResponse_Response = 1002
-const InstructionResponse_ProcessBundleSplit_case case_InstructionResponse_Response = 1003
-const InstructionResponse_FinalizeBundle_case case_InstructionResponse_Response = 1004
-const InstructionResponse_MonitoringInfos_case case_InstructionResponse_Response = 1005
-const InstructionResponse_HarnessMonitoringInfos_case case_InstructionResponse_Response = 1006
-const InstructionResponse_SampleData_case case_InstructionResponse_Response = 1007
-const InstructionResponse_Register_case case_InstructionResponse_Response = 1000
-
-func (x *InstructionResponse) WhichResponse() case_InstructionResponse_Response {
- if x == nil {
- return InstructionResponse_Response_not_set_case
- }
- switch x.Response.(type) {
- case *InstructionResponse_ProcessBundle:
- return InstructionResponse_ProcessBundle_case
- case *InstructionResponse_ProcessBundleProgress:
- return InstructionResponse_ProcessBundleProgress_case
- case *InstructionResponse_ProcessBundleSplit:
- return InstructionResponse_ProcessBundleSplit_case
- case *InstructionResponse_FinalizeBundle:
- return InstructionResponse_FinalizeBundle_case
- case *InstructionResponse_MonitoringInfos:
- return InstructionResponse_MonitoringInfos_case
- case *InstructionResponse_HarnessMonitoringInfos:
- return InstructionResponse_HarnessMonitoringInfos_case
- case *InstructionResponse_SampleData:
- return InstructionResponse_SampleData_case
- case *InstructionResponse_Register:
- return InstructionResponse_Register_case
- default:
- return InstructionResponse_Response_not_set_case
- }
-}
-
-type InstructionResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) A reference provided by the runner which represents a requests
- // execution. The InstructionResponse MUST have the matching id when
- // responding to the runner.
- InstructionId string
- // If this is specified, then this instruction has failed.
- // A human readable string representing the reason as to why processing has
- // failed.
- Error string
- // If the instruction did not fail, it is required to return an equivalent
- // response type depending on the request this matches.
-
- // Fields of oneof Response:
- ProcessBundle *ProcessBundleResponse
- ProcessBundleProgress *ProcessBundleProgressResponse
- ProcessBundleSplit *ProcessBundleSplitResponse
- FinalizeBundle *FinalizeBundleResponse
- MonitoringInfos *MonitoringInfosMetadataResponse
- HarnessMonitoringInfos *HarnessMonitoringInfosResponse
- SampleData *SampleDataResponse
- // DEPRECATED
- Register *RegisterResponse
- // -- end of Response
-}
-
-func (b0 InstructionResponse_builder) Build() *InstructionResponse {
- m0 := &InstructionResponse{}
- b, x := &b0, m0
- _, _ = b, x
- x.InstructionId = b.InstructionId
- x.Error = b.Error
- if b.ProcessBundle != nil {
- x.Response = &InstructionResponse_ProcessBundle{b.ProcessBundle}
- }
- if b.ProcessBundleProgress != nil {
- x.Response = &InstructionResponse_ProcessBundleProgress{b.ProcessBundleProgress}
- }
- if b.ProcessBundleSplit != nil {
- x.Response = &InstructionResponse_ProcessBundleSplit{b.ProcessBundleSplit}
- }
- if b.FinalizeBundle != nil {
- x.Response = &InstructionResponse_FinalizeBundle{b.FinalizeBundle}
- }
- if b.MonitoringInfos != nil {
- x.Response = &InstructionResponse_MonitoringInfos{b.MonitoringInfos}
- }
- if b.HarnessMonitoringInfos != nil {
- x.Response = &InstructionResponse_HarnessMonitoringInfos{b.HarnessMonitoringInfos}
- }
- if b.SampleData != nil {
- x.Response = &InstructionResponse_SampleData{b.SampleData}
- }
- if b.Register != nil {
- x.Response = &InstructionResponse_Register{b.Register}
- }
- return m0
-}
-
-type case_InstructionResponse_Response protoreflect.FieldNumber
-
-func (x case_InstructionResponse_Response) String() string {
- md := file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4].Descriptor()
- if x == 0 {
- return "not set"
- }
- return protoimpl.X.MessageFieldStringOf(md, protoreflect.FieldNumber(x))
-}
-
-type isInstructionResponse_Response interface {
- isInstructionResponse_Response()
-}
-
-type InstructionResponse_ProcessBundle struct {
- ProcessBundle *ProcessBundleResponse `protobuf:"bytes,1001,opt,name=process_bundle,json=processBundle,proto3,oneof"`
-}
-
-type InstructionResponse_ProcessBundleProgress struct {
- ProcessBundleProgress *ProcessBundleProgressResponse `protobuf:"bytes,1002,opt,name=process_bundle_progress,json=processBundleProgress,proto3,oneof"`
-}
-
-type InstructionResponse_ProcessBundleSplit struct {
- ProcessBundleSplit *ProcessBundleSplitResponse `protobuf:"bytes,1003,opt,name=process_bundle_split,json=processBundleSplit,proto3,oneof"`
-}
-
-type InstructionResponse_FinalizeBundle struct {
- FinalizeBundle *FinalizeBundleResponse `protobuf:"bytes,1004,opt,name=finalize_bundle,json=finalizeBundle,proto3,oneof"`
-}
-
-type InstructionResponse_MonitoringInfos struct {
- MonitoringInfos *MonitoringInfosMetadataResponse `protobuf:"bytes,1005,opt,name=monitoring_infos,json=monitoringInfos,proto3,oneof"`
-}
-
-type InstructionResponse_HarnessMonitoringInfos struct {
- HarnessMonitoringInfos *HarnessMonitoringInfosResponse `protobuf:"bytes,1006,opt,name=harness_monitoring_infos,json=harnessMonitoringInfos,proto3,oneof"`
+type InstructionResponse_HarnessMonitoringInfos struct {
+ HarnessMonitoringInfos *HarnessMonitoringInfosResponse `protobuf:"bytes,1006,opt,name=harness_monitoring_infos,json=harnessMonitoringInfos,proto3,oneof"`
}
type InstructionResponse_SampleData struct {
@@ -1383,18 +739,21 @@ func (*InstructionResponse_Register) isInstructionResponse_Response() {}
// "beam:protocol:data_sampling:v1" capability. Samples are taken only from the
// specified PCollection ids. An empty list will return everything.
type SampleDataRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Optional) The PCollection ids to filter for.
PcollectionIds []string `protobuf:"bytes,1,rep,name=pcollection_ids,json=pcollectionIds,proto3" json:"pcollection_ids,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
}
func (x *SampleDataRequest) Reset() {
*x = SampleDataRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *SampleDataRequest) String() string {
@@ -1405,7 +764,7 @@ func (*SampleDataRequest) ProtoMessage() {}
func (x *SampleDataRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1415,6 +774,11 @@ func (x *SampleDataRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use SampleDataRequest.ProtoReflect.Descriptor instead.
+func (*SampleDataRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{5}
+}
+
func (x *SampleDataRequest) GetPcollectionIds() []string {
if x != nil {
return x.PcollectionIds
@@ -1422,29 +786,13 @@ func (x *SampleDataRequest) GetPcollectionIds() []string {
return nil
}
-func (x *SampleDataRequest) SetPcollectionIds(v []string) {
- x.PcollectionIds = v
-}
-
-type SampleDataRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Optional) The PCollection ids to filter for.
- PcollectionIds []string
-}
-
-func (b0 SampleDataRequest_builder) Build() *SampleDataRequest {
- m0 := &SampleDataRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.PcollectionIds = b.PcollectionIds
- return m0
-}
-
// An element sampled when the SDK is processing a bundle. This is a proto
// message to allow for additional per-element metadata.
type SampledElement struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) Sampled raw bytes for an element. This is a
// single encoded element in the nested context.
Element []byte `protobuf:"bytes,1,opt,name=element,proto3" json:"element,omitempty"`
@@ -1452,16 +800,16 @@ type SampledElement struct {
SampleTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=sample_timestamp,json=sampleTimestamp,proto3" json:"sample_timestamp,omitempty"`
// (Optional) This will be set if this element was sampled because of a user
// exception.
- Exception *SampledElement_Exception `protobuf:"bytes,3,opt,name=exception,proto3" json:"exception,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ Exception *SampledElement_Exception `protobuf:"bytes,3,opt,name=exception,proto3" json:"exception,omitempty"`
}
func (x *SampledElement) Reset() {
*x = SampledElement{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *SampledElement) String() string {
@@ -1472,7 +820,7 @@ func (*SampledElement) ProtoMessage() {}
func (x *SampledElement) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1482,6 +830,11 @@ func (x *SampledElement) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use SampledElement.ProtoReflect.Descriptor instead.
+func (*SampledElement) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{6}
+}
+
func (x *SampledElement) GetElement() []byte {
if x != nil {
return x.Element
@@ -1503,81 +856,24 @@ func (x *SampledElement) GetException() *SampledElement_Exception {
return nil
}
-func (x *SampledElement) SetElement(v []byte) {
- if v == nil {
- v = []byte{}
- }
- x.Element = v
-}
-
-func (x *SampledElement) SetSampleTimestamp(v *timestamppb.Timestamp) {
- x.SampleTimestamp = v
-}
-
-func (x *SampledElement) SetException(v *SampledElement_Exception) {
- x.Exception = v
-}
-
-func (x *SampledElement) HasSampleTimestamp() bool {
- if x == nil {
- return false
- }
- return x.SampleTimestamp != nil
-}
-
-func (x *SampledElement) HasException() bool {
- if x == nil {
- return false
- }
- return x.Exception != nil
-}
-
-func (x *SampledElement) ClearSampleTimestamp() {
- x.SampleTimestamp = nil
-}
-
-func (x *SampledElement) ClearException() {
- x.Exception = nil
-}
-
-type SampledElement_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) Sampled raw bytes for an element. This is a
- // single encoded element in the nested context.
- Element []byte
- // (Required) Timestamp of when the sample was taken.
- SampleTimestamp *timestamppb.Timestamp
- // (Optional) This will be set if this element was sampled because of a user
- // exception.
- Exception *SampledElement_Exception
-}
-
-func (b0 SampledElement_builder) Build() *SampledElement {
- m0 := &SampledElement{}
- b, x := &b0, m0
- _, _ = b, x
- x.Element = b.Element
- x.SampleTimestamp = b.SampleTimestamp
- x.Exception = b.Exception
- return m0
-}
-
// If supported, the `SampleDataResponse` will contain samples from PCollections
// based upon the filters specified in the request.
type SampleDataResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// Map from PCollection id to sampled elements.
- ElementSamples map[string]*SampleDataResponse_ElementList `protobuf:"bytes,1,rep,name=element_samples,json=elementSamples,proto3" json:"element_samples,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ ElementSamples map[string]*SampleDataResponse_ElementList `protobuf:"bytes,1,rep,name=element_samples,json=elementSamples,proto3" json:"element_samples,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (x *SampleDataResponse) Reset() {
*x = SampleDataResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *SampleDataResponse) String() string {
@@ -1588,7 +884,7 @@ func (*SampleDataResponse) ProtoMessage() {}
func (x *SampleDataResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1598,6 +894,11 @@ func (x *SampleDataResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use SampleDataResponse.ProtoReflect.Descriptor instead.
+func (*SampleDataResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{7}
+}
+
func (x *SampleDataResponse) GetElementSamples() map[string]*SampleDataResponse_ElementList {
if x != nil {
return x.ElementSamples
@@ -1605,25 +906,6 @@ func (x *SampleDataResponse) GetElementSamples() map[string]*SampleDataResponse_
return nil
}
-func (x *SampleDataResponse) SetElementSamples(v map[string]*SampleDataResponse_ElementList) {
- x.ElementSamples = v
-}
-
-type SampleDataResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // Map from PCollection id to sampled elements.
- ElementSamples map[string]*SampleDataResponse_ElementList
-}
-
-func (b0 SampleDataResponse_builder) Build() *SampleDataResponse {
- m0 := &SampleDataResponse{}
- b, x := &b0, m0
- _, _ = b, x
- x.ElementSamples = b.ElementSamples
- return m0
-}
-
// A request to provide full MonitoringInfo associated with the entire SDK
// harness process, not specific to a bundle.
//
@@ -1637,16 +919,18 @@ func (b0 SampleDataResponse_builder) Build() *SampleDataResponse {
// as the MonitoringInfo could be reconstructed fully by overwriting its
// payload field with the bytes specified here.
type HarnessMonitoringInfosRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- unknownFields protoimpl.UnknownFields
+ state protoimpl.MessageState
sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
}
func (x *HarnessMonitoringInfosRequest) Reset() {
*x = HarnessMonitoringInfosRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *HarnessMonitoringInfosRequest) String() string {
@@ -1657,7 +941,7 @@ func (*HarnessMonitoringInfosRequest) ProtoMessage() {}
func (x *HarnessMonitoringInfosRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1667,20 +951,16 @@ func (x *HarnessMonitoringInfosRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-type HarnessMonitoringInfosRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
-}
-
-func (b0 HarnessMonitoringInfosRequest_builder) Build() *HarnessMonitoringInfosRequest {
- m0 := &HarnessMonitoringInfosRequest{}
- b, x := &b0, m0
- _, _ = b, x
- return m0
+// Deprecated: Use HarnessMonitoringInfosRequest.ProtoReflect.Descriptor instead.
+func (*HarnessMonitoringInfosRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{8}
}
type HarnessMonitoringInfosResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// An identifier to MonitoringInfo.payload mapping containing
// Metrics associated with the SDK harness, not a specific bundle.
//
@@ -1693,16 +973,16 @@ type HarnessMonitoringInfosResponse struct {
// for the lifetime of the associated control connection as long
// as the MonitoringInfo could be reconstructed fully by overwriting its
// payload field with the bytes specified here.
- MonitoringData map[string][]byte `protobuf:"bytes,1,rep,name=monitoring_data,json=monitoringData,proto3" json:"monitoring_data,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ MonitoringData map[string][]byte `protobuf:"bytes,1,rep,name=monitoring_data,json=monitoringData,proto3" json:"monitoring_data,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (x *HarnessMonitoringInfosResponse) Reset() {
*x = HarnessMonitoringInfosResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *HarnessMonitoringInfosResponse) String() string {
@@ -1713,7 +993,7 @@ func (*HarnessMonitoringInfosResponse) ProtoMessage() {}
func (x *HarnessMonitoringInfosResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1723,6 +1003,11 @@ func (x *HarnessMonitoringInfosResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use HarnessMonitoringInfosResponse.ProtoReflect.Descriptor instead.
+func (*HarnessMonitoringInfosResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{9}
+}
+
func (x *HarnessMonitoringInfosResponse) GetMonitoringData() map[string][]byte {
if x != nil {
return x.MonitoringData
@@ -1730,52 +1015,25 @@ func (x *HarnessMonitoringInfosResponse) GetMonitoringData() map[string][]byte {
return nil
}
-func (x *HarnessMonitoringInfosResponse) SetMonitoringData(v map[string][]byte) {
- x.MonitoringData = v
-}
-
-type HarnessMonitoringInfosResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // An identifier to MonitoringInfo.payload mapping containing
- // Metrics associated with the SDK harness, not a specific bundle.
- //
- // An SDK can report metrics using an identifier that only contains the
- // associated payload. A runner who wants to receive the full metrics
- // information can request all the monitoring metadata via a
- // MonitoringInfosMetadataRequest providing a list of ids as necessary.
- //
- // The SDK is allowed to reuse the identifiers
- // for the lifetime of the associated control connection as long
- // as the MonitoringInfo could be reconstructed fully by overwriting its
- // payload field with the bytes specified here.
- MonitoringData map[string][]byte
-}
-
-func (b0 HarnessMonitoringInfosResponse_builder) Build() *HarnessMonitoringInfosResponse {
- m0 := &HarnessMonitoringInfosResponse{}
- b, x := &b0, m0
- _, _ = b, x
- x.MonitoringData = b.MonitoringData
- return m0
-}
-
// A list of objects which can be referred to by the runner in
// future requests.
// Stable
type RegisterRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Optional) The set of descriptors used to process bundles.
ProcessBundleDescriptor []*ProcessBundleDescriptor `protobuf:"bytes,1,rep,name=process_bundle_descriptor,json=processBundleDescriptor,proto3" json:"process_bundle_descriptor,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
}
func (x *RegisterRequest) Reset() {
*x = RegisterRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *RegisterRequest) String() string {
@@ -1786,7 +1044,7 @@ func (*RegisterRequest) ProtoMessage() {}
func (x *RegisterRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1796,6 +1054,11 @@ func (x *RegisterRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use RegisterRequest.ProtoReflect.Descriptor instead.
+func (*RegisterRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{10}
+}
+
func (x *RegisterRequest) GetProcessBundleDescriptor() []*ProcessBundleDescriptor {
if x != nil {
return x.ProcessBundleDescriptor
@@ -1803,37 +1066,20 @@ func (x *RegisterRequest) GetProcessBundleDescriptor() []*ProcessBundleDescripto
return nil
}
-func (x *RegisterRequest) SetProcessBundleDescriptor(v []*ProcessBundleDescriptor) {
- x.ProcessBundleDescriptor = v
-}
-
-type RegisterRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Optional) The set of descriptors used to process bundles.
- ProcessBundleDescriptor []*ProcessBundleDescriptor
-}
-
-func (b0 RegisterRequest_builder) Build() *RegisterRequest {
- m0 := &RegisterRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.ProcessBundleDescriptor = b.ProcessBundleDescriptor
- return m0
-}
-
// Stable
type RegisterResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- unknownFields protoimpl.UnknownFields
+ state protoimpl.MessageState
sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
}
func (x *RegisterResponse) Reset() {
*x = RegisterResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *RegisterResponse) String() string {
@@ -1844,7 +1090,7 @@ func (*RegisterResponse) ProtoMessage() {}
func (x *RegisterResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1854,34 +1100,30 @@ func (x *RegisterResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-type RegisterResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
-}
-
-func (b0 RegisterResponse_builder) Build() *RegisterResponse {
- m0 := &RegisterResponse{}
- b, x := &b0, m0
- _, _ = b, x
- return m0
+// Deprecated: Use RegisterResponse.ProtoReflect.Descriptor instead.
+func (*RegisterResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{11}
}
// Definitions that should be used to construct the bundle processing graph.
type ProcessBundleDescriptor struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) A pipeline level unique id which can be used as a reference to
// refer to this.
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
// (Required) A map from pipeline-scoped id to PTransform.
- Transforms map[string]*pipeline_v1.PTransform `protobuf:"bytes,2,rep,name=transforms,proto3" json:"transforms,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Transforms map[string]*pipeline_v1.PTransform `protobuf:"bytes,2,rep,name=transforms,proto3" json:"transforms,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to PCollection.
- Pcollections map[string]*pipeline_v1.PCollection `protobuf:"bytes,3,rep,name=pcollections,proto3" json:"pcollections,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Pcollections map[string]*pipeline_v1.PCollection `protobuf:"bytes,3,rep,name=pcollections,proto3" json:"pcollections,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to WindowingStrategy.
- WindowingStrategies map[string]*pipeline_v1.WindowingStrategy `protobuf:"bytes,4,rep,name=windowing_strategies,json=windowingStrategies,proto3" json:"windowing_strategies,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ WindowingStrategies map[string]*pipeline_v1.WindowingStrategy `protobuf:"bytes,4,rep,name=windowing_strategies,json=windowingStrategies,proto3" json:"windowing_strategies,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to Coder.
- Coders map[string]*pipeline_v1.Coder `protobuf:"bytes,5,rep,name=coders,proto3" json:"coders,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Coders map[string]*pipeline_v1.Coder `protobuf:"bytes,5,rep,name=coders,proto3" json:"coders,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to Environment.
- Environments map[string]*pipeline_v1.Environment `protobuf:"bytes,6,rep,name=environments,proto3" json:"environments,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Environments map[string]*pipeline_v1.Environment `protobuf:"bytes,6,rep,name=environments,proto3" json:"environments,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// A descriptor describing the end point to use for State API
// calls. Required if the Runner intends to send remote references over the
// data plane or if any of the transforms rely on user state or side inputs.
@@ -1889,15 +1131,15 @@ type ProcessBundleDescriptor struct {
// A descriptor describing the end point to use for Data API for user timers.
// Required if the ProcessBundleDescriptor contains any transforms that have user timers.
TimerApiServiceDescriptor *pipeline_v1.ApiServiceDescriptor `protobuf:"bytes,8,opt,name=timer_api_service_descriptor,json=timerApiServiceDescriptor,proto3" json:"timer_api_service_descriptor,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
}
func (x *ProcessBundleDescriptor) Reset() {
*x = ProcessBundleDescriptor{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *ProcessBundleDescriptor) String() string {
@@ -1908,7 +1150,7 @@ func (*ProcessBundleDescriptor) ProtoMessage() {}
func (x *ProcessBundleDescriptor) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1918,6 +1160,11 @@ func (x *ProcessBundleDescriptor) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use ProcessBundleDescriptor.ProtoReflect.Descriptor instead.
+func (*ProcessBundleDescriptor) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{12}
+}
+
func (x *ProcessBundleDescriptor) GetId() string {
if x != nil {
return x.Id
@@ -1974,106 +1221,15 @@ func (x *ProcessBundleDescriptor) GetTimerApiServiceDescriptor() *pipeline_v1.Ap
return nil
}
-func (x *ProcessBundleDescriptor) SetId(v string) {
- x.Id = v
-}
-
-func (x *ProcessBundleDescriptor) SetTransforms(v map[string]*pipeline_v1.PTransform) {
- x.Transforms = v
-}
-
... 71248 lines suppressed ...