You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2025/01/16 23:15:32 UTC
(beam) 01/01: Revert "Migrate Beam Go to use the new Opaque Protocol Buffers (#33434)"
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch revert-33434-hybridProtos
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 2bd5d7b3bd4c5872312874a6095297437d3cead9
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Thu Jan 16 15:15:20 2025 -0800
Revert "Migrate Beam Go to use the new Opaque Protocol Buffers (#33434)"
This reverts commit 4797d75fb9560b532c8b8cc65a17d70823293949.
---
sdks/go/cmd/beamctl/cmd/artifact.go | 2 +-
sdks/go/container/boot.go | 2 +-
sdks/go/container/boot_test.go | 38 +-
sdks/go/container/pool/workerpool.go | 22 +-
sdks/go/container/pool/workerpool_test.go | 60 +-
sdks/go/container/tools/buffered_logging_test.go | 24 +-
sdks/go/container/tools/logging.go | 8 +-
sdks/go/container/tools/logging_test.go | 8 +-
sdks/go/container/tools/provision_test.go | 4 +-
sdks/go/examples/snippets/06schemas_test.go | 62 +-
sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go | 18 +-
sdks/go/pkg/beam/artifact/gcsproxy/staging.go | 28 +-
sdks/go/pkg/beam/artifact/materialize.go | 124 +-
sdks/go/pkg/beam/artifact/materialize_test.go | 94 +-
sdks/go/pkg/beam/artifact/server_test.go | 14 +-
sdks/go/pkg/beam/artifact/stage.go | 37 +-
sdks/go/pkg/beam/artifact/stage_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/translate.go | 20 +-
.../pkg/beam/core/runtime/exec/translate_test.go | 77 +-
sdks/go/pkg/beam/core/runtime/graphx/coder.go | 24 +-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 164 +-
.../beam/core/runtime/graphx/schema/schema_test.go | 850 +-
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 534 +-
.../pkg/beam/core/runtime/graphx/translate_test.go | 102 +-
sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 24 +-
sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go | 66 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 32 +-
.../pkg/beam/core/runtime/harness/datamgr_test.go | 100 +-
sdks/go/pkg/beam/core/runtime/harness/harness.go | 175 +-
.../pkg/beam/core/runtime/harness/harness_test.go | 56 +-
sdks/go/pkg/beam/core/runtime/harness/logging.go | 18 +-
.../pkg/beam/core/runtime/harness/logging_test.go | 2 +-
.../go/pkg/beam/core/runtime/harness/monitoring.go | 32 +-
.../runtime/harness/statecache/statecache_test.go | 14 +-
sdks/go/pkg/beam/core/runtime/harness/statemgr.go | 204 +-
.../pkg/beam/core/runtime/harness/statemgr_test.go | 104 +-
.../pkg/beam/core/runtime/harness/worker_status.go | 2 +-
.../core/runtime/harness/worker_status_test.go | 2 +-
.../beam/core/runtime/metricsx/metricsx_test.go | 36 +-
sdks/go/pkg/beam/core/runtime/pipelinex/clone.go | 24 +-
.../pkg/beam/core/runtime/pipelinex/clone_test.go | 10 +-
sdks/go/pkg/beam/core/runtime/pipelinex/replace.go | 46 +-
.../beam/core/runtime/pipelinex/replace_test.go | 292 +-
sdks/go/pkg/beam/core/runtime/pipelinex/util.go | 10 +-
.../pkg/beam/core/runtime/pipelinex/util_test.go | 18 +-
sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/namespace.go | 60 +-
.../pkg/beam/core/runtime/xlangx/namespace_test.go | 460 +-
sdks/go/pkg/beam/core/runtime/xlangx/payload.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/registry.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/resolve.go | 30 +-
.../pkg/beam/core/runtime/xlangx/resolve_test.go | 20 +-
sdks/go/pkg/beam/io/pubsubio/pubsubio.go | 16 +-
.../beam/model/fnexecution_v1/beam_fn_api.pb.go | 5960 +++------
.../model/fnexecution_v1/beam_fn_api_grpc.pb.go | 497 +-
.../fnexecution_v1/beam_fn_api_protoopaque.pb.go | 8793 -------------
.../model/fnexecution_v1/beam_provision_api.pb.go | 305 +-
.../fnexecution_v1/beam_provision_api_grpc.pb.go | 47 +-
.../beam_provision_api_protoopaque.pb.go | 575 -
.../model/jobmanagement_v1/beam_artifact_api.pb.go | 1509 +--
.../jobmanagement_v1/beam_artifact_api_grpc.pb.go | 366 +-
.../beam_artifact_api_protoopaque.pb.go | 2109 ---
.../jobmanagement_v1/beam_expansion_api.pb.go | 451 +-
.../jobmanagement_v1/beam_expansion_api_grpc.pb.go | 49 +-
.../beam_expansion_api_protoopaque.pb.go | 780 --
.../beam/model/jobmanagement_v1/beam_job_api.pb.go | 1670 ++-
.../model/jobmanagement_v1/beam_job_api_grpc.pb.go | 187 +-
.../beam_job_api_protoopaque.pb.go | 2611 ----
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 8416 +++++-------
.../model/pipeline_v1/beam_runner_api_grpc.pb.go | 81 +-
.../pipeline_v1/beam_runner_api_protoopaque.pb.go | 12779 -------------------
sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go | 163 +-
.../model/pipeline_v1/endpoints_protoopaque.pb.go | 280 -
.../model/pipeline_v1/external_transforms.pb.go | 780 +-
.../external_transforms_protoopaque.pb.go | 974 --
sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go | 1385 +-
.../model/pipeline_v1/metrics_protoopaque.pb.go | 1754 ---
sdks/go/pkg/beam/model/pipeline_v1/schema.pb.go | 2231 +---
.../model/pipeline_v1/schema_protoopaque.pb.go | 2998 -----
.../model/pipeline_v1/standard_window_fns.pb.go | 334 +-
.../standard_window_fns_protoopaque.pb.go | 685 -
sdks/go/pkg/beam/provision/provision_test.go | 4 +-
.../beam/runners/dataflow/dataflowlib/execute.go | 4 +-
.../beam/runners/dataflow/dataflowlib/job_test.go | 16 +-
.../runners/dataflow/dataflowlib/metrics_test.go | 12 +-
sdks/go/pkg/beam/runners/prism/internal/coders.go | 24 +-
.../pkg/beam/runners/prism/internal/coders_test.go | 194 +-
.../beam/runners/prism/internal/environments.go | 14 +-
sdks/go/pkg/beam/runners/prism/internal/execute.go | 23 +-
.../beam/runners/prism/internal/handlecombine.go | 30 +-
.../runners/prism/internal/handlecombine_test.go | 136 +-
.../pkg/beam/runners/prism/internal/handlepardo.go | 42 +-
.../beam/runners/prism/internal/handlerunner.go | 10 +-
.../runners/prism/internal/jobservices/artifact.go | 34 +-
.../prism/internal/jobservices/management.go | 108 +-
.../prism/internal/jobservices/management_test.go | 130 +-
.../runners/prism/internal/jobservices/metrics.go | 30 +-
.../prism/internal/jobservices/metrics_test.go | 14 +-
.../prism/internal/jobservices/server_test.go | 38 +-
.../pkg/beam/runners/prism/internal/preprocess.go | 2 +-
.../beam/runners/prism/internal/preprocess_test.go | 284 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 92 +-
sdks/go/pkg/beam/runners/prism/internal/web/web.go | 12 +-
.../beam/runners/prism/internal/worker/bundle.go | 88 +-
.../beam/runners/prism/internal/worker/worker.go | 122 +-
.../runners/prism/internal/worker/worker_test.go | 108 +-
.../beam/runners/universal/extworker/extworker.go | 20 +-
.../runners/universal/extworker/extworker_test.go | 60 +-
.../beam/runners/universal/runnerlib/execute.go | 8 +-
.../go/pkg/beam/runners/universal/runnerlib/job.go | 12 +-
.../pkg/beam/runners/universal/runnerlib/stage.go | 44 +-
.../beam/runners/universal/runnerlib/stage_test.go | 38 +-
sdks/go/pkg/beam/testing/teststream/teststream.go | 19 +-
.../pkg/beam/testing/teststream/teststream_test.go | 4 +-
.../pkg/beam/transforms/xlang/schema/external.go | 4 +-
sdks/go/pkg/beam/x/hooks/perf/perf.go | 6 +-
sdks/go/scripts/genproto.sh | 22 +-
.../go/test/regression/coders/fromyaml/fromyaml.go | 8 +-
sdks/java/container/boot.go | 4 +-
sdks/java/container/boot_test.go | 6 +-
sdks/java/expansion-service/container/boot.go | 11 +-
.../transform-service/controller-container/boot.go | 7 +-
sdks/python/container/boot.go | 6 +-
sdks/typescript/container/boot.go | 6 +-
124 files changed, 12021 insertions(+), 52793 deletions(-)
diff --git a/sdks/go/cmd/beamctl/cmd/artifact.go b/sdks/go/cmd/beamctl/cmd/artifact.go
index b1ca18e9643..1b5b7c9df8f 100644
--- a/sdks/go/cmd/beamctl/cmd/artifact.go
+++ b/sdks/go/cmd/beamctl/cmd/artifact.go
@@ -95,7 +95,7 @@ func listFn(cmd *cobra.Command, args []string) error {
}
for _, a := range md.GetManifest().GetArtifact() {
- cmd.Println(a.GetName())
+ cmd.Println(a.Name)
}
return nil
}
diff --git a/sdks/go/container/boot.go b/sdks/go/container/boot.go
index f10b2d59abd..3f8562f6ca9 100644
--- a/sdks/go/container/boot.go
+++ b/sdks/go/container/boot.go
@@ -184,7 +184,7 @@ func main() {
enableGoogleCloudProfiler := strings.Contains(options, enableGoogleCloudProfilerOption)
if enableGoogleCloudProfiler {
- err := configureGoogleCloudProfilerEnvVars(ctx, logger, info.GetMetadata())
+ err := configureGoogleCloudProfilerEnvVars(ctx, logger, info.Metadata)
if err != nil {
logger.Printf(ctx, "could not configure Google Cloud Profiler variables, got %v", err)
}
diff --git a/sdks/go/container/boot_test.go b/sdks/go/container/boot_test.go
index 97c0e4172dd..49c78047249 100644
--- a/sdks/go/container/boot_test.go
+++ b/sdks/go/container/boot_test.go
@@ -29,11 +29,11 @@ import (
)
func TestEnsureEndpointsSet_AllSet(t *testing.T) {
- provisionInfo := fnpb.ProvisionInfo_builder{
- LoggingEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testLoggingEndpointUrl"}.Build(),
- ArtifactEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testArtifactEndpointUrl"}.Build(),
- ControlEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testControlEndpointUrl"}.Build(),
- }.Build()
+ provisionInfo := &fnpb.ProvisionInfo{
+ LoggingEndpoint: &pipepb.ApiServiceDescriptor{Url: "testLoggingEndpointUrl"},
+ ArtifactEndpoint: &pipepb.ApiServiceDescriptor{Url: "testArtifactEndpointUrl"},
+ ControlEndpoint: &pipepb.ApiServiceDescriptor{Url: "testControlEndpointUrl"},
+ }
*loggingEndpoint = ""
*artifactEndpoint = ""
*controlEndpoint = ""
@@ -53,11 +53,11 @@ func TestEnsureEndpointsSet_AllSet(t *testing.T) {
}
func TestEnsureEndpointsSet_OneMissing(t *testing.T) {
- provisionInfo := fnpb.ProvisionInfo_builder{
- LoggingEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testLoggingEndpointUrl"}.Build(),
- ArtifactEndpoint: pipepb.ApiServiceDescriptor_builder{Url: "testArtifactEndpointUrl"}.Build(),
- ControlEndpoint: pipepb.ApiServiceDescriptor_builder{Url: ""}.Build(),
- }.Build()
+ provisionInfo := &fnpb.ProvisionInfo{
+ LoggingEndpoint: &pipepb.ApiServiceDescriptor{Url: "testLoggingEndpointUrl"},
+ ArtifactEndpoint: &pipepb.ApiServiceDescriptor{Url: "testArtifactEndpointUrl"},
+ ControlEndpoint: &pipepb.ApiServiceDescriptor{Url: ""},
+ }
*loggingEndpoint = ""
*artifactEndpoint = ""
*controlEndpoint = ""
@@ -85,7 +85,7 @@ func TestGetGoWorkerArtifactName_NoArtifacts(t *testing.T) {
func TestGetGoWorkerArtifactName_OneArtifact(t *testing.T) {
artifact := constructArtifactInformation(t, artifact.URNGoWorkerBinaryRole, "test/path", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact}
+ artifacts := []*pipepb.ArtifactInformation{&artifact}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -99,7 +99,7 @@ func TestGetGoWorkerArtifactName_OneArtifact(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsFirstIsWorker(t *testing.T) {
artifact1 := constructArtifactInformation(t, artifact.URNGoWorkerBinaryRole, "test/path", "sha")
artifact2 := constructArtifactInformation(t, "other role", "test/path2", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -113,7 +113,7 @@ func TestGetGoWorkerArtifactName_MultipleArtifactsFirstIsWorker(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsSecondIsWorker(t *testing.T) {
artifact1 := constructArtifactInformation(t, "other role", "test/path", "sha")
artifact2 := constructArtifactInformation(t, artifact.URNGoWorkerBinaryRole, "test/path2", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -127,7 +127,7 @@ func TestGetGoWorkerArtifactName_MultipleArtifactsSecondIsWorker(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsLegacyWay(t *testing.T) {
artifact1 := constructArtifactInformation(t, "other role", "test/path", "sha")
artifact2 := constructArtifactInformation(t, "other role", "worker", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
val, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err != nil {
@@ -141,7 +141,7 @@ func TestGetGoWorkerArtifactName_MultipleArtifactsLegacyWay(t *testing.T) {
func TestGetGoWorkerArtifactName_MultipleArtifactsNoneMatch(t *testing.T) {
artifact1 := constructArtifactInformation(t, "other role", "test/path", "sha")
artifact2 := constructArtifactInformation(t, "other role", "test/path2", "sha")
- artifacts := []*pipepb.ArtifactInformation{artifact1, artifact2}
+ artifacts := []*pipepb.ArtifactInformation{&artifact1, &artifact2}
_, err := getGoWorkerArtifactName(context.Background(), &tools.Logger{}, artifacts)
if err == nil {
@@ -193,16 +193,16 @@ func TestCopyExe(t *testing.T) {
}
}
-func constructArtifactInformation(t *testing.T, roleUrn string, path string, sha string) *pipepb.ArtifactInformation {
+func constructArtifactInformation(t *testing.T, roleUrn string, path string, sha string) pipepb.ArtifactInformation {
t.Helper()
- typePayload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{Path: path, Sha256: sha}.Build())
+ typePayload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{Path: path, Sha256: sha})
- return pipepb.ArtifactInformation_builder{
+ return pipepb.ArtifactInformation{
RoleUrn: roleUrn,
TypeUrn: artifact.URNFileArtifact,
TypePayload: typePayload,
- }.Build()
+ }
}
func TestConfigureGoogleCloudProfilerEnvVars(t *testing.T) {
diff --git a/sdks/go/container/pool/workerpool.go b/sdks/go/container/pool/workerpool.go
index 4c3a308fd0e..5f3642eac03 100644
--- a/sdks/go/container/pool/workerpool.go
+++ b/sdks/go/container/pool/workerpool.go
@@ -80,24 +80,24 @@ func (s *Process) StartWorker(_ context.Context, req *fnpb.StartWorkerRequest) (
s.mu.Lock()
defer s.mu.Unlock()
if s.workers == nil {
- return fnpb.StartWorkerResponse_builder{
+ return &fnpb.StartWorkerResponse{
Error: "worker pool shutting down",
- }.Build(), nil
+ }, nil
}
if _, ok := s.workers[req.GetWorkerId()]; ok {
- return fnpb.StartWorkerResponse_builder{
+ return &fnpb.StartWorkerResponse{
Error: fmt.Sprintf("worker with ID %q already exists", req.GetWorkerId()),
- }.Build(), nil
+ }, nil
}
if req.GetLoggingEndpoint() == nil {
- return fnpb.StartWorkerResponse_builder{Error: fmt.Sprintf("Missing logging endpoint for worker %v", req.GetWorkerId())}.Build(), nil
+ return &fnpb.StartWorkerResponse{Error: fmt.Sprintf("Missing logging endpoint for worker %v", req.GetWorkerId())}, nil
}
if req.GetControlEndpoint() == nil {
- return fnpb.StartWorkerResponse_builder{Error: fmt.Sprintf("Missing control endpoint for worker %v", req.GetWorkerId())}.Build(), nil
+ return &fnpb.StartWorkerResponse{Error: fmt.Sprintf("Missing control endpoint for worker %v", req.GetWorkerId())}, nil
}
- if req.GetLoggingEndpoint().HasAuthentication() || req.GetControlEndpoint().HasAuthentication() {
- return fnpb.StartWorkerResponse_builder{Error: "[BEAM-10610] Secure endpoints not supported."}.Build(), nil
+ if req.GetLoggingEndpoint().Authentication != nil || req.GetControlEndpoint().Authentication != nil {
+ return &fnpb.StartWorkerResponse{Error: "[BEAM-10610] Secure endpoints not supported."}, nil
}
ctx := grpcx.WriteWorkerID(s.root, req.GetWorkerId())
@@ -118,7 +118,7 @@ func (s *Process) StartWorker(_ context.Context, req *fnpb.StartWorkerRequest) (
cmd.Env = nil // Use the current environment.
if err := cmd.Start(); err != nil {
- return fnpb.StartWorkerResponse_builder{Error: fmt.Sprintf("Unable to start boot for worker %v: %v", req.GetWorkerId(), err)}.Build(), nil
+ return &fnpb.StartWorkerResponse{Error: fmt.Sprintf("Unable to start boot for worker %v: %v", req.GetWorkerId(), err)}, nil
}
return &fnpb.StartWorkerResponse{}, nil
}
@@ -137,9 +137,9 @@ func (s *Process) StopWorker(_ context.Context, req *fnpb.StopWorkerRequest) (*f
delete(s.workers, req.GetWorkerId())
return &fnpb.StopWorkerResponse{}, nil
}
- return fnpb.StopWorkerResponse_builder{
+ return &fnpb.StopWorkerResponse{
Error: fmt.Sprintf("no worker with id %q running", req.GetWorkerId()),
- }.Build(), nil
+ }, nil
}
diff --git a/sdks/go/container/pool/workerpool_test.go b/sdks/go/container/pool/workerpool_test.go
index ed001ec0149..add2e25681f 100644
--- a/sdks/go/container/pool/workerpool_test.go
+++ b/sdks/go/container/pool/workerpool_test.go
@@ -31,15 +31,15 @@ func TestProcess(t *testing.T) {
t.Skip("Binary `true` doesn't exist, skipping tests.")
}
- endpoint := pipepb.ApiServiceDescriptor_builder{
+ endpoint := &pipepb.ApiServiceDescriptor{
Url: "localhost:0",
- }.Build()
- secureEndpoint := pipepb.ApiServiceDescriptor_builder{
+ }
+ secureEndpoint := &pipepb.ApiServiceDescriptor{
Url: "localhost:0",
- Authentication: pipepb.AuthenticationSpec_builder{
+ Authentication: &pipepb.AuthenticationSpec{
Urn: "beam:authentication:oauth2_client_credentials_grant:v1",
- }.Build(),
- }.Build()
+ },
+ }
ctx, cancelFn := context.WithCancel(context.Background())
t.Cleanup(cancelFn)
@@ -54,60 +54,60 @@ func TestProcess(t *testing.T) {
errExpected bool
}{
{
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "Worker1",
ControlEndpoint: endpoint,
LoggingEndpoint: endpoint,
- }.Build(),
+ },
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "Worker2",
ControlEndpoint: endpoint,
LoggingEndpoint: endpoint,
- }.Build(),
+ },
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "Worker1",
ControlEndpoint: endpoint,
LoggingEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true, // Repeated start
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "missingControl",
LoggingEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "missingLogging",
ControlEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "secureLogging",
LoggingEndpoint: secureEndpoint,
ControlEndpoint: endpoint,
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StartWorkerRequest_builder{
+ req: &fnpb.StartWorkerRequest{
WorkerId: "secureControl",
LoggingEndpoint: endpoint,
ControlEndpoint: secureEndpoint,
- }.Build(),
+ },
errExpected: true,
},
}
for _, test := range startTests {
resp, err := server.StartWorker(ctx, test.req)
if test.errExpected {
- if err != nil || resp.GetError() == "" {
+ if err != nil || resp.Error == "" {
t.Errorf("Expected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
} else {
- if err != nil || resp.GetError() != "" {
+ if err != nil || resp.Error != "" {
t.Errorf("Unexpected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
}
@@ -117,29 +117,29 @@ func TestProcess(t *testing.T) {
errExpected bool
}{
{
- req: fnpb.StopWorkerRequest_builder{
+ req: &fnpb.StopWorkerRequest{
WorkerId: "Worker1",
- }.Build(),
+ },
}, {
- req: fnpb.StopWorkerRequest_builder{
+ req: &fnpb.StopWorkerRequest{
WorkerId: "Worker1",
- }.Build(),
+ },
errExpected: true,
}, {
- req: fnpb.StopWorkerRequest_builder{
+ req: &fnpb.StopWorkerRequest{
WorkerId: "NonExistent",
- }.Build(),
+ },
errExpected: true,
},
}
for _, test := range stopTests {
resp, err := server.StopWorker(ctx, test.req)
if test.errExpected {
- if err != nil || resp.GetError() == "" {
+ if err != nil || resp.Error == "" {
t.Errorf("Expected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
} else {
- if err != nil || resp.GetError() != "" {
+ if err != nil || resp.Error != "" {
t.Errorf("Unexpected error starting %v: err: %v, resp: %v", test.req.GetWorkerId(), err, resp)
}
}
diff --git a/sdks/go/container/tools/buffered_logging_test.go b/sdks/go/container/tools/buffered_logging_test.go
index 9f11cf319c4..9f542d2d5ab 100644
--- a/sdks/go/container/tools/buffered_logging_test.go
+++ b/sdks/go/container/tools/buffered_logging_test.go
@@ -63,11 +63,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "test message"; got != want {
+ if got, want := received.Message, "test message"; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
})
@@ -96,11 +96,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()
for i, message := range received {
- if got, want := message.GetMessage(), messages[i]; got != want {
+ if got, want := message.Message, messages[i]; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := message.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := message.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
}
@@ -125,11 +125,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "test error"; got != want {
+ if got, want := received.Message, "test error"; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_ERROR; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_ERROR; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
})
@@ -158,11 +158,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()
for i, message := range received {
- if got, want := message.GetMessage(), messages[i]; got != want {
+ if got, want := message.Message, messages[i]; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := message.GetSeverity(), fnpb.LogEntry_Severity_ERROR; got != want {
+ if got, want := message.Severity, fnpb.LogEntry_Severity_ERROR; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
}
@@ -177,11 +177,11 @@ func TestBufferedLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "foo bar"; got != want {
+ if got, want := received.Message, "foo bar"; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got severity %v, want %v", got, want)
}
})
@@ -229,11 +229,11 @@ func TestBufferedLogger(t *testing.T) {
messages = append(messages, lastMessage)
for i, message := range received {
- if got, want := message.GetMessage(), messages[i]; got != want {
+ if got, want := message.Message, messages[i]; got != want {
t.Errorf("got message %q, want %q", got, want)
}
- if got, want := message.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := message.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("got severity %v, want %v", got, want)
}
}
diff --git a/sdks/go/container/tools/logging.go b/sdks/go/container/tools/logging.go
index ed0f7b82c9f..ced13b744d4 100644
--- a/sdks/go/container/tools/logging.go
+++ b/sdks/go/container/tools/logging.go
@@ -86,15 +86,15 @@ func (l *Logger) Log(ctx context.Context, sev fnpb.LogEntry_Severity_Enum, messa
}
}
- err := l.client.Send(fnpb.LogEntry_List_builder{
+ err := l.client.Send(&fnpb.LogEntry_List{
LogEntries: []*fnpb.LogEntry{
- fnpb.LogEntry_builder{
+ {
Severity: sev,
Timestamp: timestamppb.Now(),
Message: message,
- }.Build(),
+ },
},
- }.Build())
+ })
if err != nil {
exitErr = err
return
diff --git a/sdks/go/container/tools/logging_test.go b/sdks/go/container/tools/logging_test.go
index b837b40fa24..8730a0fe9c1 100644
--- a/sdks/go/container/tools/logging_test.go
+++ b/sdks/go/container/tools/logging_test.go
@@ -50,11 +50,11 @@ func TestLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "foo bar"; got != want {
+ if got, want := received.Message, "foo bar"; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_DEBUG; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got severity %v, want %v", got, want)
}
})
@@ -66,11 +66,11 @@ func TestLogger(t *testing.T) {
received := catcher.msgs[0].GetLogEntries()[0]
- if got, want := received.GetMessage(), "failed to install dependency bar"; got != want {
+ if got, want := received.Message, "failed to install dependency bar"; got != want {
t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want)
}
- if got, want := received.GetSeverity(), fnpb.LogEntry_Severity_ERROR; got != want {
+ if got, want := received.Severity, fnpb.LogEntry_Severity_ERROR; got != want {
t.Errorf("l.Errorf(\"failed to install dependency %%v\", \"bar\"): got severity %v, want %v", got, want)
}
})
diff --git a/sdks/go/container/tools/provision_test.go b/sdks/go/container/tools/provision_test.go
index 1820092c0cf..dbb78040632 100644
--- a/sdks/go/container/tools/provision_test.go
+++ b/sdks/go/container/tools/provision_test.go
@@ -66,7 +66,7 @@ type ProvisionServiceServicer struct {
}
func (p ProvisionServiceServicer) GetProvisionInfo(ctx context.Context, req *fnpb.GetProvisionInfoRequest) (*fnpb.GetProvisionInfoResponse, error) {
- return fnpb.GetProvisionInfoResponse_builder{Info: fnpb.ProvisionInfo_builder{RetrievalToken: "token"}.Build()}.Build(), nil
+ return &fnpb.GetProvisionInfoResponse{Info: &fnpb.ProvisionInfo{RetrievalToken: "token"}}, nil
}
func setup(addr *string, wg *sync.WaitGroup) {
@@ -103,7 +103,7 @@ func TestProvisionInfo(t *testing.T) {
if err != nil {
t.Errorf("error in response: %v", err)
}
- want := fnpb.ProvisionInfo_builder{RetrievalToken: "token"}.Build()
+ want := &fnpb.ProvisionInfo{RetrievalToken: "token"}
if got.GetRetrievalToken() != want.GetRetrievalToken() {
t.Errorf("provision.Info() = %v, want %v", got, want)
}
diff --git a/sdks/go/examples/snippets/06schemas_test.go b/sdks/go/examples/snippets/06schemas_test.go
index df8799cc141..325a96274c2 100644
--- a/sdks/go/examples/snippets/06schemas_test.go
+++ b/sdks/go/examples/snippets/06schemas_test.go
@@ -31,49 +31,55 @@ import (
)
func atomicSchemaField(name string, typ pipepb.AtomicType) *pipepb.Field {
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
- Type: pipepb.FieldType_builder{
- AtomicType: typ.Enum(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: typ,
+ },
+ },
+ }
}
func rowSchemaField(name string, typ *pipepb.Schema) *pipepb.Field {
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
- Type: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: typ,
- }.Build(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: typ,
+ },
+ },
+ },
+ }
}
func listSchemaField(name string, typ *pipepb.Field) *pipepb.Field {
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
- Type: pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: typ.GetType(),
- }.Build(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: typ.GetType(),
+ },
+ },
+ },
+ }
}
func nillable(f *pipepb.Field) *pipepb.Field {
- f.GetType().SetNullable(true)
+ f.Type.Nullable = true
return f
}
func TestSchemaTypes(t *testing.T) {
- transactionSchema := pipepb.Schema_builder{
+ transactionSchema := &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("bank", pipepb.AtomicType_STRING),
atomicSchemaField("purchaseAmount", pipepb.AtomicType_DOUBLE),
},
- }.Build()
- shippingAddressSchema := pipepb.Schema_builder{
+ }
+ shippingAddressSchema := &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("streetAddress", pipepb.AtomicType_STRING),
atomicSchemaField("city", pipepb.AtomicType_STRING),
@@ -81,7 +87,7 @@ func TestSchemaTypes(t *testing.T) {
atomicSchemaField("country", pipepb.AtomicType_STRING),
atomicSchemaField("postCode", pipepb.AtomicType_STRING),
},
- }.Build()
+ }
tests := []struct {
rt reflect.Type
@@ -95,7 +101,7 @@ func TestSchemaTypes(t *testing.T) {
st: shippingAddressSchema,
}, {
rt: reflect.TypeOf(Purchase{}),
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("userId", pipepb.AtomicType_STRING),
atomicSchemaField("itemId", pipepb.AtomicType_INT64),
@@ -104,15 +110,15 @@ func TestSchemaTypes(t *testing.T) {
listSchemaField("transactions",
rowSchemaField("n/a", transactionSchema)),
},
- }.Build(),
+ },
}, {
rt: tnType,
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
atomicSchemaField("seconds", pipepb.AtomicType_INT64),
atomicSchemaField("nanos", pipepb.AtomicType_INT32),
},
- }.Build(),
+ },
preReg: func(reg *schema.Registry) {
reg.RegisterLogicalType(schema.ToLogicalType(tnType.Name(), tnType, tnStorageType))
},
diff --git a/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go b/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
index 602af701b22..ceb8a319be9 100644
--- a/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
+++ b/sdks/go/pkg/beam/artifact/gcsproxy/retrieval.go
@@ -75,7 +75,7 @@ func NewRetrievalServer(md *jobpb.ProxyManifest) (*RetrievalServer, error) {
// GetManifest returns the manifest for all artifacts.
func (s *RetrievalServer) GetManifest(ctx context.Context, req *jobpb.GetManifestRequest) (*jobpb.GetManifestResponse, error) {
- return jobpb.GetManifestResponse_builder{Manifest: s.md}.Build(), nil
+ return &jobpb.GetManifestResponse{Manifest: s.md}, nil
}
// GetArtifact returns a given artifact.
@@ -105,7 +105,7 @@ func (s *RetrievalServer) GetArtifact(req *jobpb.LegacyGetArtifactRequest, strea
for {
n, err := r.Read(data)
if n > 0 {
- if err := stream.Send(jobpb.ArtifactChunk_builder{Data: data[:n]}.Build()); err != nil {
+ if err := stream.Send(&jobpb.ArtifactChunk{Data: data[:n]}); err != nil {
return errors.Wrap(err, "chunk send failed")
}
}
@@ -122,20 +122,20 @@ func (s *RetrievalServer) GetArtifact(req *jobpb.LegacyGetArtifactRequest, strea
func validate(md *jobpb.ProxyManifest) error {
keys := make(map[string]bool)
for _, a := range md.GetManifest().GetArtifact() {
- if _, seen := keys[a.GetName()]; seen {
- return errors.Errorf("multiple artifact with name %v", a.GetName())
+ if _, seen := keys[a.Name]; seen {
+ return errors.Errorf("multiple artifact with name %v", a.Name)
}
- keys[a.GetName()] = true
+ keys[a.Name] = true
}
for _, l := range md.GetLocation() {
- fresh, seen := keys[l.GetName()]
+ fresh, seen := keys[l.Name]
if !seen {
- return errors.Errorf("no artifact named %v for location %v", l.GetName(), l.GetUri())
+ return errors.Errorf("no artifact named %v for location %v", l.Name, l.Uri)
}
if !fresh {
- return errors.Errorf("multiple locations for %v:%v", l.GetName(), l.GetUri())
+ return errors.Errorf("multiple locations for %v:%v", l.Name, l.Uri)
}
- keys[l.GetName()] = false
+ keys[l.Name] = false
}
for key, fresh := range keys {
diff --git a/sdks/go/pkg/beam/artifact/gcsproxy/staging.go b/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
index 4031d94c4df..9113e780f33 100644
--- a/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
+++ b/sdks/go/pkg/beam/artifact/gcsproxy/staging.go
@@ -75,7 +75,7 @@ func (s *StagingServer) CommitManifest(ctx context.Context, req *jobpb.CommitMan
}
s.mu.Unlock()
- data, err := proto.Marshal(jobpb.ProxyManifest_builder{Manifest: manifest, Location: loc}.Build())
+ data, err := proto.Marshal(&jobpb.ProxyManifest{Manifest: manifest, Location: loc})
if err != nil {
return nil, errors.Wrap(err, "failed to marshal proxy manifest")
}
@@ -93,7 +93,7 @@ func (s *StagingServer) CommitManifest(ctx context.Context, req *jobpb.CommitMan
// now, but would be needed for a staging server that serves multiple
// jobs. Such a server would also use the ID sent with each request.
- return jobpb.CommitManifestResponse_builder{RetrievalToken: gcsx.MakeObject(s.bucket, s.manifest)}.Build(), nil
+ return &jobpb.CommitManifestResponse{RetrievalToken: gcsx.MakeObject(s.bucket, s.manifest)}, nil
}
// matchLocations ensures that all artifacts have been staged and have valid
@@ -101,18 +101,18 @@ func (s *StagingServer) CommitManifest(ctx context.Context, req *jobpb.CommitMan
func matchLocations(artifacts []*jobpb.ArtifactMetadata, blobs map[string]staged) ([]*jobpb.ProxyManifest_Location, error) {
var loc []*jobpb.ProxyManifest_Location
for _, a := range artifacts {
- info, ok := blobs[a.GetName()]
+ info, ok := blobs[a.Name]
if !ok {
- return nil, errors.Errorf("artifact %v not staged", a.GetName())
+ return nil, errors.Errorf("artifact %v not staged", a.Name)
}
- if a.GetSha256() == "" {
- a.SetSha256(info.hash)
+ if a.Sha256 == "" {
+ a.Sha256 = info.hash
}
- if info.hash != a.GetSha256() {
- return nil, errors.Errorf("staged artifact for %v has invalid SHA256: %v, want %v", a.GetName(), info.hash, a.GetSha256())
+ if info.hash != a.Sha256 {
+ return nil, errors.Errorf("staged artifact for %v has invalid SHA256: %v, want %v", a.Name, info.hash, a.Sha256)
}
- loc = append(loc, jobpb.ProxyManifest_Location_builder{Name: a.GetName(), Uri: info.object}.Build())
+ loc = append(loc, &jobpb.ProxyManifest_Location{Name: a.Name, Uri: info.object})
}
return loc, nil
}
@@ -129,7 +129,7 @@ func (s *StagingServer) PutArtifact(ps jobpb.LegacyArtifactStagingService_PutArt
if md == nil {
return errors.Errorf("expected header as first message: %v", header)
}
- object := path.Join(s.root, md.GetName())
+ object := path.Join(s.root, md.Name)
// Stream content to GCS. We don't have to worry about partial
// or abandoned writes, because object writes are atomic.
@@ -142,15 +142,15 @@ func (s *StagingServer) PutArtifact(ps jobpb.LegacyArtifactStagingService_PutArt
r := &reader{sha256W: sha256.New(), stream: ps}
if err := gcsx.WriteObject(ctx, cl, s.bucket, object, r); err != nil {
- return errors.Wrapf(err, "failed to stage artifact %v", md.GetName())
+ return errors.Wrapf(err, "failed to stage artifact %v", md.Name)
}
hash := r.SHA256()
- if md.GetSha256() != "" && md.GetSha256() != hash {
- return errors.Errorf("invalid SHA256 for artifact %v: %v want %v", md.GetName(), hash, md.GetSha256())
+ if md.Sha256 != "" && md.Sha256 != hash {
+ return errors.Errorf("invalid SHA256 for artifact %v: %v want %v", md.Name, hash, md.Sha256)
}
s.mu.Lock()
- s.blobs[md.GetName()] = staged{object: gcsx.MakeObject(s.bucket, object), hash: hash}
+ s.blobs[md.Name] = staged{object: gcsx.MakeObject(s.bucket, object), hash: hash}
s.mu.Unlock()
return ps.SendAndClose(&jobpb.PutArtifactResponse{})
diff --git a/sdks/go/pkg/beam/artifact/materialize.go b/sdks/go/pkg/beam/artifact/materialize.go
index e955941061d..624e30efcd2 100644
--- a/sdks/go/pkg/beam/artifact/materialize.go
+++ b/sdks/go/pkg/beam/artifact/materialize.go
@@ -78,59 +78,59 @@ func newMaterialize(ctx context.Context, endpoint string, dependencies []*pipepb
}
func newMaterializeWithClient(ctx context.Context, client jobpb.ArtifactRetrievalServiceClient, dependencies []*pipepb.ArtifactInformation, dest string) ([]*pipepb.ArtifactInformation, error) {
- resolution, err := client.ResolveArtifacts(ctx, jobpb.ResolveArtifactsRequest_builder{Artifacts: dependencies}.Build())
+ resolution, err := client.ResolveArtifacts(ctx, &jobpb.ResolveArtifactsRequest{Artifacts: dependencies})
if err != nil {
return nil, err
}
var artifacts []*pipepb.ArtifactInformation
var list []retrievable
- for _, dep := range resolution.GetReplacements() {
+ for _, dep := range resolution.Replacements {
path, err := extractStagingToPath(dep)
if err != nil {
return nil, err
}
- filePayload := pipepb.ArtifactFilePayload_builder{
+ filePayload := pipepb.ArtifactFilePayload{
Path: path,
- }.Build()
- if dep.GetTypeUrn() == URNFileArtifact {
- typePayload := &pipepb.ArtifactFilePayload{}
- if err := proto.Unmarshal(dep.GetTypePayload(), typePayload); err != nil {
+ }
+ if dep.TypeUrn == URNFileArtifact {
+ typePayload := pipepb.ArtifactFilePayload{}
+ if err := proto.Unmarshal(dep.TypePayload, &typePayload); err != nil {
return nil, errors.Wrap(err, "failed to parse artifact file payload")
}
- filePayload.SetSha256(typePayload.GetSha256())
- } else if dep.GetTypeUrn() == URNUrlArtifact {
- typePayload := &pipepb.ArtifactUrlPayload{}
- if err := proto.Unmarshal(dep.GetTypePayload(), typePayload); err != nil {
+ filePayload.Sha256 = typePayload.Sha256
+ } else if dep.TypeUrn == URNUrlArtifact {
+ typePayload := pipepb.ArtifactUrlPayload{}
+ if err := proto.Unmarshal(dep.TypePayload, &typePayload); err != nil {
return nil, errors.Wrap(err, "failed to parse artifact url payload")
}
- filePayload.SetSha256(typePayload.GetSha256())
+ filePayload.Sha256 = typePayload.Sha256
}
- newTypePayload, err := proto.Marshal(filePayload)
+ newTypePayload, err := proto.Marshal(&filePayload)
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact type payload")
}
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNFileArtifact,
TypePayload: newTypePayload,
- RoleUrn: dep.GetRoleUrn(),
- RolePayload: dep.GetRolePayload(),
- }.Build())
+ RoleUrn: dep.RoleUrn,
+ RolePayload: dep.RolePayload,
+ })
- rolePayload, err := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
+ rolePayload, err := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
StagedName: path,
- }.Build())
+ })
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact role payload")
}
list = append(list, &artifact{
client: client,
- dep: pipepb.ArtifactInformation_builder{
- TypeUrn: dep.GetTypeUrn(),
- TypePayload: dep.GetTypePayload(),
+ dep: &pipepb.ArtifactInformation{
+ TypeUrn: dep.TypeUrn,
+ TypePayload: dep.TypePayload,
RoleUrn: URNStagingTo,
RolePayload: rolePayload,
- }.Build(),
+ },
})
}
@@ -147,39 +147,39 @@ func generateId() string {
func extractStagingToPath(artifact *pipepb.ArtifactInformation) (string, error) {
var stagedName string
- if artifact.GetRoleUrn() == URNStagingTo {
- role := &pipepb.ArtifactStagingToRolePayload{}
- if err := proto.Unmarshal(artifact.GetRolePayload(), role); err != nil {
+ if artifact.RoleUrn == URNStagingTo {
+ role := pipepb.ArtifactStagingToRolePayload{}
+ if err := proto.Unmarshal(artifact.RolePayload, &role); err != nil {
return "", err
}
- stagedName = role.GetStagedName()
- } else if artifact.GetTypeUrn() == URNFileArtifact {
- ty := &pipepb.ArtifactFilePayload{}
- if err := proto.Unmarshal(artifact.GetTypePayload(), ty); err != nil {
+ stagedName = role.StagedName
+ } else if artifact.TypeUrn == URNFileArtifact {
+ ty := pipepb.ArtifactFilePayload{}
+ if err := proto.Unmarshal(artifact.TypePayload, &ty); err != nil {
return "", err
}
- stagedName = generateId() + "-" + filepath.Base(ty.GetPath())
- } else if artifact.GetTypeUrn() == URNUrlArtifact {
- ty := &pipepb.ArtifactUrlPayload{}
- if err := proto.Unmarshal(artifact.GetTypePayload(), ty); err != nil {
+ stagedName = generateId() + "-" + filepath.Base(ty.Path)
+ } else if artifact.TypeUrn == URNUrlArtifact {
+ ty := pipepb.ArtifactUrlPayload{}
+ if err := proto.Unmarshal(artifact.TypePayload, &ty); err != nil {
return "", err
}
- stagedName = generateId() + "-" + path.Base(ty.GetUrl())
+ stagedName = generateId() + "-" + path.Base(ty.Url)
} else {
- return "", errors.Errorf("failed to extract staging path for artifact type %v role %v", artifact.GetTypeUrn(), artifact.GetRoleUrn())
+ return "", errors.Errorf("failed to extract staging path for artifact type %v role %v", artifact.TypeUrn, artifact.RoleUrn)
}
return stagedName, nil
}
func MustExtractFilePayload(artifact *pipepb.ArtifactInformation) (string, string) {
- if artifact.GetTypeUrn() != URNFileArtifact {
- log.Fatalf("Unsupported artifact type %v", artifact.GetTypeUrn())
+ if artifact.TypeUrn != URNFileArtifact {
+ log.Fatalf("Unsupported artifact type %v", artifact.TypeUrn)
}
- ty := &pipepb.ArtifactFilePayload{}
- if err := proto.Unmarshal(artifact.GetTypePayload(), ty); err != nil {
+ ty := pipepb.ArtifactFilePayload{}
+ if err := proto.Unmarshal(artifact.TypePayload, &ty); err != nil {
log.Fatalf("failed to parse artifact file payload: %v", err)
}
- return ty.GetPath(), ty.GetSha256()
+ return ty.Path, ty.Sha256
}
type artifact struct {
@@ -208,7 +208,7 @@ func (a artifact) retrieve(ctx context.Context, dest string) error {
return err
}
- stream, err := a.client.GetArtifact(ctx, jobpb.GetArtifactRequest_builder{Artifact: a.dep}.Build())
+ stream, err := a.client.GetArtifact(ctx, &jobpb.GetArtifactRequest{Artifact: a.dep})
if err != nil {
return err
}
@@ -244,10 +244,10 @@ func writeChunks(stream jobpb.ArtifactRetrievalService_GetArtifactClient, w io.W
if err != nil {
return "", err
}
- if _, err := sha256W.Write(chunk.GetData()); err != nil {
+ if _, err := sha256W.Write(chunk.Data); err != nil {
panic(err) // cannot fail
}
- if _, err := w.Write(chunk.GetData()); err != nil {
+ if _, err := w.Write(chunk.Data); err != nil {
return "", errors.Wrapf(err, "chunk write failed")
}
}
@@ -263,7 +263,7 @@ func legacyMaterialize(ctx context.Context, endpoint string, rt string, dest str
client := jobpb.NewLegacyArtifactRetrievalServiceClient(cc)
- m, err := client.GetManifest(ctx, jobpb.GetManifestRequest_builder{RetrievalToken: rt}.Build())
+ m, err := client.GetManifest(ctx, &jobpb.GetManifestRequest{RetrievalToken: rt})
if err != nil {
return nil, errors.Wrap(err, "failed to get manifest")
}
@@ -272,25 +272,25 @@ func legacyMaterialize(ctx context.Context, endpoint string, rt string, dest str
var artifacts []*pipepb.ArtifactInformation
var list []retrievable
for _, md := range mds {
- typePayload, err := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: md.GetName(),
- Sha256: md.GetSha256(),
- }.Build())
+ typePayload, err := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: md.Name,
+ Sha256: md.Sha256,
+ })
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact type payload")
}
- rolePayload, err := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: md.GetName(),
- }.Build())
+ rolePayload, err := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: md.Name,
+ })
if err != nil {
return nil, errors.Wrap(err, "failed to create artifact role payload")
}
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNFileArtifact,
TypePayload: typePayload,
RoleUrn: URNStagingTo,
RolePayload: rolePayload,
- }.Build())
+ })
list = append(list, &legacyArtifact{
client: client,
rt: rt,
@@ -382,7 +382,7 @@ func (a legacyArtifact) retrieve(ctx context.Context, dest string) error {
// previous retrieval attempt and may leave a corrupt/partial local file on
// failure.
func Retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceClient, a *jobpb.ArtifactMetadata, rt string, dest string) error {
- filename := filepath.Join(dest, filepath.FromSlash(a.GetName()))
+ filename := filepath.Join(dest, filepath.FromSlash(a.Name))
_, err := os.Stat(filename)
if err != nil && !os.IsNotExist(err) {
@@ -392,7 +392,7 @@ func Retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceCl
// File already exists. Validate or delete.
hash, err := computeSHA256(filename)
- if err == nil && a.GetSha256() == hash {
+ if err == nil && a.Sha256 == hash {
// NOTE(herohde) 10/5/2017: We ignore permissions here, because
// they may differ from the requested permissions due to umask
// settings on unix systems (which we in turn want to respect).
@@ -417,12 +417,12 @@ func Retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceCl
// It expects the file to not exist, but does not clean up on failure and
// may leave a corrupt file.
func retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceClient, a *jobpb.ArtifactMetadata, rt string, filename string) error {
- stream, err := client.GetArtifact(ctx, jobpb.LegacyGetArtifactRequest_builder{Name: a.GetName(), RetrievalToken: rt}.Build())
+ stream, err := client.GetArtifact(ctx, &jobpb.LegacyGetArtifactRequest{Name: a.Name, RetrievalToken: rt})
if err != nil {
return err
}
- fd, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(a.GetPermissions()))
+ fd, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(a.Permissions))
if err != nil {
return err
}
@@ -442,8 +442,8 @@ func retrieve(ctx context.Context, client jobpb.LegacyArtifactRetrievalServiceCl
}
// Artifact Sha256 hash is an optional field in metadata so we should only validate when its present.
- if a.GetSha256() != "" && sha256Hash != a.GetSha256() {
- return errors.Errorf("bad SHA256 for %v: %v, want %v", filename, sha256Hash, a.GetSha256())
+ if a.Sha256 != "" && sha256Hash != a.Sha256 {
+ return errors.Errorf("bad SHA256 for %v: %v, want %v", filename, sha256Hash, a.Sha256)
}
return nil
}
@@ -459,10 +459,10 @@ func retrieveChunks(stream jobpb.LegacyArtifactRetrievalService_GetArtifactClien
return "", err
}
- if _, err := sha256W.Write(chunk.GetData()); err != nil {
+ if _, err := sha256W.Write(chunk.Data); err != nil {
panic(err) // cannot fail
}
- if _, err := w.Write(chunk.GetData()); err != nil {
+ if _, err := w.Write(chunk.Data); err != nil {
return "", errors.Wrapf(err, "chunk write failed")
}
}
diff --git a/sdks/go/pkg/beam/artifact/materialize_test.go b/sdks/go/pkg/beam/artifact/materialize_test.go
index 2eb3fb6c7a7..31890ed045c 100644
--- a/sdks/go/pkg/beam/artifact/materialize_test.go
+++ b/sdks/go/pkg/beam/artifact/materialize_test.go
@@ -49,12 +49,12 @@ func TestRetrieve(t *testing.T) {
client := jobpb.NewLegacyArtifactRetrievalServiceClient(cc)
for _, a := range artifacts {
- filename := makeFilename(dst, a.GetName())
+ filename := makeFilename(dst, a.Name)
if err := Retrieve(ctx, client, a, rt, dst); err != nil {
- t.Errorf("failed to retrieve %v: %v", a.GetName(), err)
+ t.Errorf("failed to retrieve %v: %v", a.Name, err)
continue
}
- verifySHA256(t, filename, a.GetSha256())
+ verifySHA256(t, filename, a.Sha256)
}
}
@@ -78,7 +78,7 @@ func TestMultiRetrieve(t *testing.T) {
}
for _, a := range artifacts {
- verifySHA256(t, makeFilename(dst, a.GetName()), a.GetSha256())
+ verifySHA256(t, makeFilename(dst, a.Name), a.Sha256)
}
}
@@ -112,18 +112,20 @@ func stage(ctx context.Context, scl jobpb.LegacyArtifactStagingServiceClient, t
sha256W.Write(data)
hash := hex.EncodeToString(sha256W.Sum(nil))
md := makeArtifact(key, hash)
- pmd := jobpb.PutArtifactMetadata_builder{
+ pmd := &jobpb.PutArtifactMetadata{
Metadata: md,
StagingSessionToken: st,
- }.Build()
+ }
stream, err := scl.PutArtifact(ctx)
if err != nil {
t.Fatalf("put failed: %v", err)
}
- header := jobpb.PutArtifactRequest_builder{
- Metadata: proto.ValueOrDefault(pmd),
- }.Build()
+ header := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Metadata{
+ Metadata: pmd,
+ },
+ }
if err := stream.Send(header); err != nil {
t.Fatalf("send header failed: %v", err)
}
@@ -134,11 +136,13 @@ func stage(ctx context.Context, scl jobpb.LegacyArtifactStagingServiceClient, t
end = size
}
- chunk := jobpb.PutArtifactRequest_builder{
- Data: jobpb.ArtifactChunk_builder{
- Data: data[i:end],
- }.Build(),
- }.Build()
+ chunk := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Data{
+ Data: &jobpb.ArtifactChunk{
+ Data: data[i:end],
+ },
+ },
+ }
if err := stream.Send(chunk); err != nil {
t.Fatalf("send chunk[%v:%v] failed: %v", i, end, err)
}
@@ -189,8 +193,8 @@ func TestNewRetrieveWithFileGeneratedStageName(t *testing.T) {
generated := make(map[string]string)
for _, md := range mds {
name, _ := MustExtractFilePayload(md)
- payload, _ := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: name}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: name})
generated[name] = string(payload)
}
@@ -216,8 +220,8 @@ func TestNewRetrieveWithUrlGeneratedStageName(t *testing.T) {
generated := make(map[string]string)
for _, md := range mds {
name, _ := MustExtractFilePayload(md)
- payload, _ := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: name}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: name})
generated[name] = string(payload)
}
@@ -294,14 +298,14 @@ type fakeRetrievalService struct {
func (fake *fakeRetrievalService) resolvedArtifactsWithStagingTo() []*pipepb.ArtifactInformation {
var artifacts []*pipepb.ArtifactInformation
for name, contents := range fake.artifacts {
- payload, _ := proto.Marshal(pipepb.ArtifactStagingToRolePayload_builder{
- StagedName: name}.Build())
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ payload, _ := proto.Marshal(&pipepb.ArtifactStagingToRolePayload{
+ StagedName: name})
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: "resolved",
TypePayload: []byte(contents),
RoleUrn: URNStagingTo,
RolePayload: payload,
- }.Build())
+ })
}
return artifacts
}
@@ -309,12 +313,12 @@ func (fake *fakeRetrievalService) resolvedArtifactsWithStagingTo() []*pipepb.Art
func (fake *fakeRetrievalService) fileArtifactsWithoutStagingTo() []*pipepb.ArtifactInformation {
var artifacts []*pipepb.ArtifactInformation
for name := range fake.artifacts {
- payload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: filepath.Join("/tmp", name)}.Build())
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ payload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: filepath.Join("/tmp", name)})
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNFileArtifact,
TypePayload: payload,
- }.Build())
+ })
}
return artifacts
}
@@ -322,44 +326,44 @@ func (fake *fakeRetrievalService) fileArtifactsWithoutStagingTo() []*pipepb.Arti
func (fake *fakeRetrievalService) urlArtifactsWithoutStagingTo() []*pipepb.ArtifactInformation {
var artifacts []*pipepb.ArtifactInformation
for name := range fake.artifacts {
- payload, _ := proto.Marshal(pipepb.ArtifactUrlPayload_builder{
- Url: path.Join("gs://tmp", name)}.Build())
- artifacts = append(artifacts, pipepb.ArtifactInformation_builder{
+ payload, _ := proto.Marshal(&pipepb.ArtifactUrlPayload{
+ Url: path.Join("gs://tmp", name)})
+ artifacts = append(artifacts, &pipepb.ArtifactInformation{
TypeUrn: URNUrlArtifact,
TypePayload: payload,
- }.Build())
+ })
}
return artifacts
}
func (fake *fakeRetrievalService) unresolvedArtifacts() []*pipepb.ArtifactInformation {
return []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ &pipepb.ArtifactInformation{
TypeUrn: "unresolved",
- }.Build(),
+ },
}
}
func (fake *fakeRetrievalService) ResolveArtifacts(ctx context.Context, request *jobpb.ResolveArtifactsRequest, opts ...grpc.CallOption) (*jobpb.ResolveArtifactsResponse, error) {
- response := &jobpb.ResolveArtifactsResponse{}
- for _, dep := range request.GetArtifacts() {
- if dep.GetTypeUrn() == "unresolved" {
- response.SetReplacements(append(response.GetReplacements(), fake.resolvedArtifactsWithStagingTo()...))
+ response := jobpb.ResolveArtifactsResponse{}
+ for _, dep := range request.Artifacts {
+ if dep.TypeUrn == "unresolved" {
+ response.Replacements = append(response.Replacements, fake.resolvedArtifactsWithStagingTo()...)
} else {
- response.SetReplacements(append(response.GetReplacements(), dep))
+ response.Replacements = append(response.Replacements, dep)
}
}
- return response, nil
+ return &response, nil
}
func (fake *fakeRetrievalService) GetArtifact(ctx context.Context, request *jobpb.GetArtifactRequest, opts ...grpc.CallOption) (jobpb.ArtifactRetrievalService_GetArtifactClient, error) {
- switch request.GetArtifact().GetTypeUrn() {
+ switch request.Artifact.TypeUrn {
case "resolved":
- return &fakeGetArtifactResponseStream{data: request.GetArtifact().GetTypePayload()}, nil
+ return &fakeGetArtifactResponseStream{data: request.Artifact.TypePayload}, nil
case URNFileArtifact, URNUrlArtifact:
- return &fakeGetArtifactResponseStream{data: request.GetArtifact().GetRolePayload()}, nil
+ return &fakeGetArtifactResponseStream{data: request.Artifact.RolePayload}, nil
default:
- return nil, errors.Errorf("Unsupported artifact %v", request.GetArtifact())
+ return nil, errors.Errorf("Unsupported artifact %v", request.Artifact)
}
}
@@ -371,7 +375,7 @@ type fakeGetArtifactResponseStream struct {
func (fake *fakeGetArtifactResponseStream) Recv() (*jobpb.GetArtifactResponse, error) {
if fake.index < len(fake.data) {
fake.index++
- return jobpb.GetArtifactResponse_builder{Data: fake.data[fake.index-1 : fake.index]}.Build(), nil
+ return &jobpb.GetArtifactResponse{Data: fake.data[fake.index-1 : fake.index]}, nil
}
return nil, io.EOF
}
@@ -447,11 +451,11 @@ func makeTempFile(t *testing.T, filename string, size int) string {
}
func makeArtifact(key, hash string) *jobpb.ArtifactMetadata {
- return jobpb.ArtifactMetadata_builder{
+ return &jobpb.ArtifactMetadata{
Name: key,
Sha256: hash,
Permissions: 0644,
- }.Build()
+ }
}
func makeFilename(dir, key string) string {
diff --git a/sdks/go/pkg/beam/artifact/server_test.go b/sdks/go/pkg/beam/artifact/server_test.go
index f0ce3a59f05..3fdc8fa5cac 100644
--- a/sdks/go/pkg/beam/artifact/server_test.go
+++ b/sdks/go/pkg/beam/artifact/server_test.go
@@ -85,7 +85,7 @@ func (s *server) PutArtifact(ps jobpb.LegacyArtifactStagingService_PutArtifactSe
if header.GetMetadata() == nil {
return errors.Errorf("expected header as first message: %v", header)
}
- key := header.GetMetadata().GetMetadata().GetName()
+ key := header.GetMetadata().GetMetadata().Name
if header.GetMetadata().GetStagingSessionToken() == "" {
return errors.New("missing staging session token")
}
@@ -138,19 +138,19 @@ func (s *server) CommitManifest(ctx context.Context, req *jobpb.CommitManifestRe
artifacts := req.GetManifest().GetArtifact()
for _, md := range artifacts {
- if _, ok := m.m[md.GetName()]; !ok {
- return nil, errors.Errorf("artifact %v not staged", md.GetName())
+ if _, ok := m.m[md.Name]; !ok {
+ return nil, errors.Errorf("artifact %v not staged", md.Name)
}
}
// Update commit. Only one manifest can exist for each staging id.
for _, md := range artifacts {
- m.m[md.GetName()].md = md
+ m.m[md.Name].md = md
}
m.md = req.GetManifest()
- return jobpb.CommitManifestResponse_builder{RetrievalToken: token}.Build(), nil
+ return &jobpb.CommitManifestResponse{RetrievalToken: token}, nil
}
func (s *server) GetManifest(ctx context.Context, req *jobpb.GetManifestRequest) (*jobpb.GetManifestResponse, error) {
@@ -166,7 +166,7 @@ func (s *server) GetManifest(ctx context.Context, req *jobpb.GetManifestRequest)
m.mu.Lock()
defer m.mu.Unlock()
- return jobpb.GetManifestResponse_builder{Manifest: m.md}.Build(), nil
+ return &jobpb.GetManifestResponse{Manifest: m.md}, nil
}
func (s *server) GetArtifact(req *jobpb.LegacyGetArtifactRequest, stream jobpb.LegacyArtifactRetrievalService_GetArtifactServer) error {
@@ -195,7 +195,7 @@ func (s *server) GetArtifact(req *jobpb.LegacyGetArtifactRequest, stream jobpb.L
// Send chunks exactly as we received them.
for _, chunk := range chunks {
- if err := stream.Send(jobpb.ArtifactChunk_builder{Data: chunk}.Build()); err != nil {
+ if err := stream.Send(&jobpb.ArtifactChunk{Data: chunk}); err != nil {
return err
}
}
diff --git a/sdks/go/pkg/beam/artifact/stage.go b/sdks/go/pkg/beam/artifact/stage.go
index d248380c836..de017b4c3c0 100644
--- a/sdks/go/pkg/beam/artifact/stage.go
+++ b/sdks/go/pkg/beam/artifact/stage.go
@@ -31,18 +31,17 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
jobpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1"
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/errorx"
- "google.golang.org/protobuf/proto"
)
// Commit commits a manifest with the given staged artifacts. It returns the
// staging token, if successful.
func Commit(ctx context.Context, client jobpb.LegacyArtifactStagingServiceClient, artifacts []*jobpb.ArtifactMetadata, st string) (string, error) {
- req := jobpb.CommitManifestRequest_builder{
- Manifest: jobpb.Manifest_builder{
+ req := &jobpb.CommitManifestRequest{
+ Manifest: &jobpb.Manifest{
Artifact: artifacts,
- }.Build(),
+ },
StagingSessionToken: st,
- }.Build()
+ }
resp, err := client.CommitManifest(ctx, req)
if err != nil {
return "", err
@@ -128,15 +127,15 @@ func Stage(ctx context.Context, client jobpb.LegacyArtifactStagingServiceClient,
if err != nil {
return nil, err
}
- md := jobpb.ArtifactMetadata_builder{
+ md := &jobpb.ArtifactMetadata{
Name: key,
Permissions: uint32(stat.Mode()),
Sha256: hash,
- }.Build()
- pmd := jobpb.PutArtifactMetadata_builder{
+ }
+ pmd := &jobpb.PutArtifactMetadata{
Metadata: md,
StagingSessionToken: st,
- }.Build()
+ }
fd, err := os.Open(filename)
if err != nil {
@@ -149,9 +148,11 @@ func Stage(ctx context.Context, client jobpb.LegacyArtifactStagingServiceClient,
return nil, err
}
- header := jobpb.PutArtifactRequest_builder{
- Metadata: proto.ValueOrDefault(pmd),
- }.Build()
+ header := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Metadata{
+ Metadata: pmd,
+ },
+ }
if err := stream.Send(header); err != nil {
stream.CloseAndRecv() // ignore error
return nil, errors.Wrapf(err, "failed to send header for %v", filename)
@@ -180,11 +181,13 @@ func stageChunks(stream jobpb.LegacyArtifactStagingService_PutArtifactClient, r
panic(err) // cannot fail
}
- chunk := jobpb.PutArtifactRequest_builder{
- Data: jobpb.ArtifactChunk_builder{
- Data: data[:n],
- }.Build(),
- }.Build()
+ chunk := &jobpb.PutArtifactRequest{
+ Content: &jobpb.PutArtifactRequest_Data{
+ Data: &jobpb.ArtifactChunk{
+ Data: data[:n],
+ },
+ },
+ }
err := stream.Send(chunk)
if err == io.EOF {
return "", err
diff --git a/sdks/go/pkg/beam/artifact/stage_test.go b/sdks/go/pkg/beam/artifact/stage_test.go
index af2af73ad75..2355f0a0176 100644
--- a/sdks/go/pkg/beam/artifact/stage_test.go
+++ b/sdks/go/pkg/beam/artifact/stage_test.go
@@ -84,7 +84,7 @@ func validate(ctx context.Context, cc *grpc.ClientConn, t *testing.T, keys, sha2
rcl := jobpb.NewLegacyArtifactRetrievalServiceClient(cc)
for i, key := range keys {
- stream, err := rcl.GetArtifact(ctx, jobpb.LegacyGetArtifactRequest_builder{Name: key, RetrievalToken: rt}.Build())
+ stream, err := rcl.GetArtifact(ctx, &jobpb.LegacyGetArtifactRequest{Name: key, RetrievalToken: rt})
if err != nil {
t.Fatalf("failed to get artifact for %v: %v", key, err)
}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate.go b/sdks/go/pkg/beam/core/runtime/exec/translate.go
index 37b379784ce..b74ede228fd 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/translate.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/translate.go
@@ -349,7 +349,7 @@ func (b *builder) makeCoderForPCollection(id string) (*coder.Coder, *coder.Windo
if !ok {
return nil, nil, errors.Errorf("pcollection %v not found", id)
}
- c, err := b.coders.Coder(col.GetCoderId())
+ c, err := b.coders.Coder(col.CoderId)
if err != nil {
return nil, nil, err
}
@@ -539,11 +539,11 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
var cID string
var kcID string
if rmw := spec.GetReadModifyWriteSpec(); rmw != nil {
- cID = rmw.GetCoderId()
+ cID = rmw.CoderId
} else if bs := spec.GetBagSpec(); bs != nil {
- cID = bs.GetElementCoderId()
+ cID = bs.ElementCoderId
} else if cs := spec.GetCombiningSpec(); cs != nil {
- cID = cs.GetAccumulatorCoderId()
+ cID = cs.AccumulatorCoderId
cmbData := string(cs.GetCombineFn().GetPayload())
var cmbTp v1pb.TransformPayload
if err := protox.DecodeBase64(cmbData, &cmbTp); err != nil {
@@ -559,10 +559,10 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
}
stateIDToCombineFn[key] = cfn
} else if ms := spec.GetMapSpec(); ms != nil {
- cID = ms.GetValueCoderId()
- kcID = ms.GetKeyCoderId()
+ cID = ms.ValueCoderId
+ kcID = ms.KeyCoderId
} else if ss := spec.GetSetSpec(); ss != nil {
- kcID = ss.GetElementCoderId()
+ kcID = ss.ElementCoderId
} else {
return nil, errors.Errorf("Unrecognized state type %v", spec)
}
@@ -801,10 +801,10 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
u = &MapWindows{UID: b.idgen.New(), Fn: mapper, Out: out[0], FnUrn: fn.GetUrn()}
case graphx.URNFlatten:
- u = &Flatten{UID: b.idgen.New(), N: len(transform.GetInputs()), Out: out[0]}
+ u = &Flatten{UID: b.idgen.New(), N: len(transform.Inputs), Out: out[0]}
// Use the same flatten instance for all the inputs links to this transform.
- for i := 0; i < len(transform.GetInputs()); i++ {
+ for i := 0; i < len(transform.Inputs); i++ {
b.links[linkID{id.to, i}] = u
}
@@ -918,5 +918,5 @@ func unmarshalPort(data []byte) (Port, string, error) {
}
return Port{
URL: port.GetApiServiceDescriptor().GetUrl(),
- }, port.GetCoderId(), nil
+ }, port.CoderId, nil
}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate_test.go b/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
index 4203bc5c396..a9917ec456f 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
@@ -17,13 +17,12 @@ package exec
import (
"fmt"
+ fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1"
"reflect"
"strings"
"testing"
"time"
- fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1"
-
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
@@ -72,12 +71,12 @@ func TestUnmarshalKeyedValues(t *testing.T) {
func TestUnmarshalReshuffleCoders(t *testing.T) {
payloads := map[string][]byte{}
encode := func(id, urn string, comps ...string) {
- payloads[id] = protox.MustEncode(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ payloads[id] = protox.MustEncode(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urn,
- }.Build(),
+ },
ComponentCoderIds: comps,
- }.Build())
+ })
}
encode("a", "beam:coder:bytes:v1")
encode("b", "beam:coder:string_utf8:v1")
@@ -268,37 +267,37 @@ func TestUnmarshalWindowMapper(t *testing.T) {
func makeWindowFn(w *window.Fn) (*pipepb.FunctionSpec, error) {
switch w.Kind {
case window.GlobalWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNGlobalWindowsWindowFn,
- }.Build(), nil
+ }, nil
case window.FixedWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNFixedWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.FixedWindowsPayload_builder{
+ &pipepb.FixedWindowsPayload{
Size: durationpb.New(w.Size),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.SlidingWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNSlidingWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.SlidingWindowsPayload_builder{
+ &pipepb.SlidingWindowsPayload{
Size: durationpb.New(w.Size),
Period: durationpb.New(w.Period),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.Sessions:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: graphx.URNSessionsWindowFn,
Payload: protox.MustEncode(
- pipepb.SessionWindowsPayload_builder{
+ &pipepb.SessionWindowsPayload{
GapSize: durationpb.New(w.Gap),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
default:
return nil, errors.Errorf("unexpected windowing strategy: %v", w)
}
@@ -311,11 +310,11 @@ func makeWindowMappingFn(w *window.Fn) (*pipepb.FunctionSpec, error) {
}
switch w.Kind {
case window.GlobalWindows:
- wFn.SetUrn(graphx.URNWindowMappingGlobal)
+ wFn.Urn = graphx.URNWindowMappingGlobal
case window.FixedWindows:
- wFn.SetUrn(graphx.URNWindowMappingFixed)
+ wFn.Urn = graphx.URNWindowMappingFixed
case window.SlidingWindows:
- wFn.SetUrn(graphx.URNWindowMappingSliding)
+ wFn.Urn = graphx.URNWindowMappingSliding
default:
return nil, fmt.Errorf("unknown window fn type %v", w.Kind)
}
@@ -394,7 +393,7 @@ func TestUnmarshalPort(t *testing.T) {
{
inputData: []byte{},
outputPort: Port{URL: port.GetApiServiceDescriptor().GetUrl()},
- outputStr: (&fnpb.RemoteGrpcPort{}).GetCoderId(),
+ outputStr: fnpb.RemoteGrpcPort{}.CoderId,
outputError: nil,
},
}
@@ -414,12 +413,12 @@ func TestUnmarshalPort(t *testing.T) {
}
func TestUnmarshalPlan(t *testing.T) {
- transform := pipepb.PTransform_builder{
- Spec: pipepb.FunctionSpec_builder{
+ transform := pipepb.PTransform{
+ Spec: &pipepb.FunctionSpec{
Urn: urnDataSource,
- }.Build(),
+ },
Outputs: map[string]string{},
- }.Build()
+ }
tests := []struct {
name string
inputDesc *fnpb.ProcessBundleDescriptor
@@ -428,32 +427,32 @@ func TestUnmarshalPlan(t *testing.T) {
}{
{
name: "test_no_root_units",
- inputDesc: fnpb.ProcessBundleDescriptor_builder{
+ inputDesc: &fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{},
- }.Build(),
+ },
outputPlan: nil,
outputError: errors.New("no root units"),
},
{
name: "test_zero_transform",
- inputDesc: fnpb.ProcessBundleDescriptor_builder{
+ inputDesc: &fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{
"": {},
},
- }.Build(),
+ },
outputPlan: nil,
outputError: errors.New("no root units"),
},
{
name: "test_transform_outputs_length_not_one",
- inputDesc: fnpb.ProcessBundleDescriptor_builder{
+ inputDesc: &fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{
- "": transform,
+ "": &transform,
},
- }.Build(),
+ },
outputPlan: nil,
outputError: errors.Errorf("expected one output from DataSource, got %v", transform.GetOutputs()),
},
@@ -474,10 +473,10 @@ func TestUnmarshalPlan(t *testing.T) {
}
func TestNewBuilder(t *testing.T) {
- descriptor := fnpb.ProcessBundleDescriptor_builder{
+ descriptor := fnpb.ProcessBundleDescriptor{
Id: "",
Transforms: map[string]*pipepb.PTransform{},
- }.Build()
+ }
tests := []struct {
name string
inputDesc *fnpb.ProcessBundleDescriptor
@@ -486,9 +485,9 @@ func TestNewBuilder(t *testing.T) {
}{
{
name: "test_1",
- inputDesc: descriptor,
+ inputDesc: &descriptor,
outputBuilder: &builder{
- desc: descriptor,
+ desc: &descriptor,
coders: graphx.NewCoderUnmarshaller(descriptor.GetCoders()),
prev: make(map[string]int),
succ: make(map[string][]linkID),
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/coder.go b/sdks/go/pkg/beam/core/runtime/graphx/coder.go
index 4ea77dee183..99ca5517d3d 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/coder.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/coder.go
@@ -454,12 +454,12 @@ func (b *CoderMarshaller) Add(c *coder.Coder) (string, error) {
if err != nil {
return "", errors.Wrapf(err, "failed to marshal custom coder %v", c)
}
- inner := b.internCoder(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ inner := b.internCoder(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urnCustomCoder,
Payload: []byte(data),
- }.Build(),
- }.Build())
+ },
+ })
return b.internBuiltInCoder(urnLengthPrefixCoder, inner), nil
case coder.KV:
@@ -596,22 +596,22 @@ func (b *CoderMarshaller) Build() map[string]*pipepb.Coder {
}
func (b *CoderMarshaller) internBuiltInCoder(urn string, components ...string) string {
- return b.internCoder(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ return b.internCoder(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urn,
- }.Build(),
+ },
ComponentCoderIds: components,
- }.Build())
+ })
}
func (b *CoderMarshaller) internRowCoder(schema *pipepb.Schema) string {
payload := protox.MustEncode(schema)
- return b.internCoder(pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ return b.internCoder(&pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: urnRowCoder,
Payload: payload,
- }.Build(),
- }.Build())
+ },
+ })
}
func (b *CoderMarshaller) internCoder(coder *pipepb.Coder) string {
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
index 2a23e364228..0d44e68285b 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
@@ -375,12 +375,12 @@ func (r *Registry) fromType(ot reflect.Type) (*pipepb.Schema, error) {
schm := ftype.GetRowType().GetSchema()
schm = proto.Clone(schm).(*pipepb.Schema)
if ot.Kind() == reflect.Ptr {
- schm.SetOptions(append(schm.GetOptions(), optGoNillable()))
+ schm.Options = append(schm.Options, optGoNillable())
}
if lID != "" {
- schm.SetOptions(append(schm.GetOptions(), logicalOption(lID)))
+ schm.Options = append(schm.Options, logicalOption(lID))
}
- schm.SetId(getUUID(ot))
+ schm.Id = getUUID(ot)
r.typeToSchema[ot] = schm
r.idToType[schm.GetId()] = ot
return schm, nil
@@ -395,8 +395,8 @@ func (r *Registry) fromType(ot reflect.Type) (*pipepb.Schema, error) {
// Cache the pointer type here with it's own id.
pt := reflect.PtrTo(t)
schm = proto.Clone(schm).(*pipepb.Schema)
- schm.SetId(getUUID(pt))
- schm.SetOptions(append(schm.GetOptions(), optGoNillable()))
+ schm.Id = getUUID(pt)
+ schm.Options = append(schm.Options, optGoNillable())
r.idToType[schm.GetId()] = pt
r.typeToSchema[pt] = schm
@@ -428,17 +428,23 @@ func optGoEmbedded() *pipepb.Option {
// is not set if the toggle isn't true, so the value is always
// true.
func newToggleOption(urn string) *pipepb.Option {
- return pipepb.Option_builder{
+ return &pipepb.Option{
Name: urn,
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BOOLEAN.Enum(),
- }.Build(),
- Value: pipepb.FieldValue_builder{
- AtomicValue: pipepb.AtomicTypeValue_builder{
- Boolean: proto.Bool(true),
- }.Build(),
- }.Build(),
- }.Build()
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BOOLEAN,
+ },
+ },
+ Value: &pipepb.FieldValue{
+ FieldValue: &pipepb.FieldValue_AtomicValue{
+ AtomicValue: &pipepb.AtomicTypeValue{
+ Value: &pipepb.AtomicTypeValue_Boolean{
+ Boolean: true,
+ },
+ },
+ },
+ },
+ }
}
func checkOptions(opts []*pipepb.Option, urn string) *pipepb.Option {
@@ -459,20 +465,26 @@ func nillableFromOptions(opts []*pipepb.Option, t reflect.Type) reflect.Type {
return nil
}
-var optGoLogicalType = pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
-}.Build()
+var optGoLogicalType = &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+}
func logicalOption(lID string) *pipepb.Option {
- return pipepb.Option_builder{
+ return &pipepb.Option{
Name: optGoLogicalUrn,
Type: optGoLogicalType,
- Value: pipepb.FieldValue_builder{
- AtomicValue: pipepb.AtomicTypeValue_builder{
- String: proto.String(lID),
- }.Build(),
- }.Build(),
- }.Build()
+ Value: &pipepb.FieldValue{
+ FieldValue: &pipepb.FieldValue_AtomicValue{
+ AtomicValue: &pipepb.AtomicTypeValue{
+ Value: &pipepb.AtomicTypeValue_String_{
+ String_: lID,
+ },
+ },
+ },
+ },
+ }
}
// fromLogicalOption returns the logical type id of this top
@@ -501,8 +513,8 @@ func (r *Registry) structToSchema(t reflect.Type) (*pipepb.Schema, error) {
if ftype != nil {
schm := ftype.GetRowType().GetSchema()
schm = proto.Clone(schm).(*pipepb.Schema)
- schm.SetOptions(append(schm.GetOptions(), logicalOption(lID)))
- schm.SetId(getUUID(t))
+ schm.Options = append(schm.Options, logicalOption(lID))
+ schm.Id = getUUID(t)
r.typeToSchema[t] = schm
r.idToType[schm.GetId()] = t
return schm, nil
@@ -524,15 +536,15 @@ func (r *Registry) structToSchema(t reflect.Type) (*pipepb.Schema, error) {
}
if isAnon {
f = proto.Clone(f).(*pipepb.Field)
- f.SetOptions(append(f.GetOptions(), optGoEmbedded()))
+ f.Options = append(f.Options, optGoEmbedded())
}
fields = append(fields, f)
}
- schm := pipepb.Schema_builder{
+ schm := &pipepb.Schema{
Fields: fields,
Id: getUUID(t),
- }.Build()
+ }
r.idToType[schm.GetId()] = t
r.typeToSchema[t] = schm
return schm, nil
@@ -547,10 +559,10 @@ func (r *Registry) structFieldToField(sf reflect.StructField) (*pipepb.Field, er
if err != nil {
return nil, err
}
- return pipepb.Field_builder{
+ return &pipepb.Field{
Name: name,
Type: ftype,
- }.Build(), nil
+ }, nil
}
func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, error) {
@@ -559,13 +571,15 @@ func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, e
return nil, err
}
if ftype != nil {
- return pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: lID,
- Representation: ftype,
- // TODO(BEAM-9615): Handle type Arguments.
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: lID,
+ Representation: ftype,
+ // TODO(BEAM-9615): Handle type Arguments.
+ },
+ },
+ }, nil
}
t := ot
@@ -575,7 +589,7 @@ func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, e
if err != nil {
return nil, errors.Wrapf(err, "unable to convert key of %v to schema field", ot)
}
- vt.SetNullable(true)
+ vt.Nullable = true
return vt, nil
case reflect.Map:
kt, err := r.reflectTypeToFieldType(t.Key())
@@ -586,45 +600,55 @@ func (r *Registry) reflectTypeToFieldType(ot reflect.Type) (*pipepb.FieldType, e
if err != nil {
return nil, errors.Wrapf(err, "unable to convert value of %v to schema field", ot)
}
- return pipepb.FieldType_builder{
- MapType: pipepb.MapType_builder{
- KeyType: kt,
- ValueType: vt,
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_MapType{
+ MapType: &pipepb.MapType{
+ KeyType: kt,
+ ValueType: vt,
+ },
+ },
+ }, nil
case reflect.Struct:
sch, err := r.structToSchema(t)
if err != nil {
return nil, errors.Wrapf(err, "unable to convert %v to schema field", ot)
}
- return pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: sch,
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: sch,
+ },
+ },
+ }, nil
case reflect.Slice, reflect.Array:
// Special handling for []byte
if t == reflectx.ByteSlice {
- return pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BYTES.Enum(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BYTES,
+ },
+ }, nil
}
vt, err := r.reflectTypeToFieldType(t.Elem())
if err != nil {
return nil, errors.Wrapf(err, "unable to convert element type of %v to schema field", ot)
}
- return pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: vt,
- }.Build(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: vt,
+ },
+ },
+ }, nil
case reflect.Interface, reflect.Func, reflect.Chan, reflect.UnsafePointer, reflect.Complex128, reflect.Complex64, reflect.Invalid:
return nil, errors.Errorf("unable to convert unsupported type %v to schema", ot)
default: // must be an atomic type
if enum, ok := reflectTypeToAtomicTypeMap[t.Kind()]; ok {
- return pipepb.FieldType_builder{
- AtomicType: enum.Enum(),
- }.Build(), nil
+ return &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: enum,
+ },
+ }, nil
}
return nil, errors.Errorf("unable to map %v to pipepb.AtomicType", t)
}
@@ -689,7 +713,7 @@ func (r *Registry) toType(s *pipepb.Schema) (reflect.Type, error) {
if err != nil {
return nil, errors.Wrapf(err, "cannot convert schema field %v to field", sf.GetName())
}
- if checkOptions(sf.GetOptions(), optGoEmbeddedUrn) != nil {
+ if checkOptions(sf.Options, optGoEmbeddedUrn) != nil {
rf.Anonymous = true
}
fields = append(fields, rf)
@@ -706,7 +730,7 @@ func (r *Registry) toType(s *pipepb.Schema) (reflect.Type, error) {
func (r *Registry) fieldToStructField(sf *pipepb.Field) (reflect.StructField, error) {
name := sf.GetName()
- rt, err := r.fieldTypeToReflectType(sf.GetType(), sf.GetOptions())
+ rt, err := r.fieldTypeToReflectType(sf.GetType(), sf.Options)
if err != nil {
return reflect.StructField{}, err
}
@@ -736,19 +760,19 @@ var atomicTypeToReflectType = map[pipepb.AtomicType]reflect.Type{
func (r *Registry) fieldTypeToReflectType(sft *pipepb.FieldType, opts []*pipepb.Option) (reflect.Type, error) {
var t reflect.Type
- switch sft.WhichTypeInfo() {
- case pipepb.FieldType_AtomicType_case:
+ switch sft.GetTypeInfo().(type) {
+ case *pipepb.FieldType_AtomicType:
var ok bool
if t, ok = atomicTypeToReflectType[sft.GetAtomicType()]; !ok {
return nil, errors.Errorf("unknown atomic type: %v", sft.GetAtomicType())
}
- case pipepb.FieldType_ArrayType_case:
+ case *pipepb.FieldType_ArrayType:
rt, err := r.fieldTypeToReflectType(sft.GetArrayType().GetElementType(), nil)
if err != nil {
return nil, errors.Wrap(err, "unable to convert array element type")
}
t = reflect.SliceOf(rt)
- case pipepb.FieldType_MapType_case:
+ case *pipepb.FieldType_MapType:
kt, err := r.fieldTypeToReflectType(sft.GetMapType().GetKeyType(), nil)
if err != nil {
return nil, errors.Wrap(err, "unable to convert map key type")
@@ -758,7 +782,7 @@ func (r *Registry) fieldTypeToReflectType(sft *pipepb.FieldType, opts []*pipepb.
return nil, errors.Wrap(err, "unable to convert map value type")
}
t = reflect.MapOf(kt, vt) // Panics for invalid map keys (slices/iterables)
- case pipepb.FieldType_RowType_case:
+ case *pipepb.FieldType_RowType:
rt, err := r.toType(sft.GetRowType().GetSchema())
if err != nil {
return nil, errors.Wrapf(err, "unable to convert row type: %v", sft.GetRowType().GetSchema().GetId())
@@ -767,7 +791,7 @@ func (r *Registry) fieldTypeToReflectType(sft *pipepb.FieldType, opts []*pipepb.
// case *pipepb.FieldType_IterableType:
// TODO(BEAM-9615): handle IterableTypes (eg. CoGBK values)
- case pipepb.FieldType_LogicalType_case:
+ case *pipepb.FieldType_LogicalType:
lst := sft.GetLogicalType()
identifier := lst.GetUrn()
lt, ok := r.logicalTypes[identifier]
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go
index ebe3d3898ab..367d70e81d1 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go
@@ -107,532 +107,650 @@ func TestSchemaConversion(t *testing.T) {
rt reflect.Type
}{
{
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "firstField",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
FirstField int32 `beam:"firstField"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "stringField",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "intPtrField",
- Type: pipepb.FieldType_builder{
- Nullable: true,
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ Nullable: true,
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
StringField string `beam:"stringField"`
IntPtrField *int32 `beam:"intPtrField"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "cypher",
- Type: pipepb.FieldType_builder{
- MapType: pipepb.MapType_builder{
- KeyType: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BOOLEAN.Enum(),
- }.Build(),
- ValueType: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_FLOAT.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_MapType{
+ MapType: &pipepb.MapType{
+ KeyType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BOOLEAN,
+ },
+ },
+ ValueType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_FLOAT,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: hasMapType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "wrapper",
- Type: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{pipepb.Field_builder{
- Name: "threshold",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_DOUBLE.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{{
+ Name: "threshold",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_DOUBLE,
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
Wrapper struct {
Threshold float64 `beam:"threshold"`
} `beam:"wrapper"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "payloads",
- Type: pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BYTES.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BYTES,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
Payloads [][]byte `beam:"payloads"`
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "AString",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "AnIntPtr",
- Type: pipepb.FieldType_builder{
- Nullable: true,
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ Nullable: true,
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
AString string
AnIntPtr *int32
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "A",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "B",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "C",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_BOOLEAN.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_BOOLEAN,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(registeredType{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "D",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(sRegisteredType{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ &pipepb.Field{
Name: "A",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "B",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
- pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
+ &pipepb.Field{
Name: "C",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(justAType{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Q",
- Type: pipepb.FieldType_builder{
- MapType: pipepb.MapType_builder{
- KeyType: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- ValueType: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(), pipepb.Field_builder{
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_MapType{
+ MapType: &pipepb.MapType{
+ KeyType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ ValueType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }, {
Name: "T",
- Type: pipepb.FieldType_builder{
- ArrayType: pipepb.ArrayType_builder{
- ElementType: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_ArrayType{
+ ArrayType: &pipepb.ArrayType{
+ ElementType: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
Q map[int]int
T []int
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "SuperNES",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
},
Options: []*pipepb.Option{optGoNillable()},
- }.Build(),
+ },
rt: reflect.TypeOf(&struct {
SuperNES int16
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Options: []*pipepb.Option{
logicalOption("schema.unexportedFields"),
},
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "D",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "uint64",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "uint64",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: unexportedFieldsType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "G",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.unexportedFields",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "D",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "uint64",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.unexportedFields",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "D",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "uint64",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct{ G unexportedFields }{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "H",
- Type: pipepb.FieldType_builder{
+ Type: &pipepb.FieldType{
Nullable: true,
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.unexportedFields",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "D",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "uint64",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.unexportedFields",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "D",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "uint64",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct{ H *unexportedFields }{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "E",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
},
Options: []*pipepb.Option{optGoNillable(), logicalOption("*schema.exportedFunc")},
- }.Build(),
+ },
rt: exportedFuncType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Q",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.myInt",
- Representation: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.myInt",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: anotherType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Exported",
Options: []*pipepb.Option{optGoEmbedded()},
- Type: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "G",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.myInt",
- Representation: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "G",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.myInt",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: hasEmbeddedType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "Exported",
Options: []*pipepb.Option{optGoEmbedded()},
- Type: pipepb.FieldType_builder{
+ Type: &pipepb.FieldType{
Nullable: true,
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "G",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.myInt",
- Representation: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "int",
- Representation: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT64.Enum(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "G",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.myInt",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "int",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT64,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: hasEmbeddedPtrType,
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "T",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_STRING.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_STRING,
+ },
+ },
+ },
},
Options: []*pipepb.Option{optGoNillable()},
- }.Build(),
+ },
rt: reflect.TypeOf(&struct {
myInt
T string
i int
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Options: []*pipepb.Option{
logicalOption("schema.exportedFunc"),
},
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "V",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(exportedFunc{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "U",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.exportedFunc",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "V",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT16.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.exportedFunc",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "V",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT16,
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
U exportedFunc
}{}),
}, {
- st: pipepb.Schema_builder{
+ st: &pipepb.Schema{
Fields: []*pipepb.Field{
- pipepb.Field_builder{
+ {
Name: "U",
- Type: pipepb.FieldType_builder{
- LogicalType: pipepb.LogicalType_builder{
- Urn: "schema.nonRegisteredLogical",
- Representation: pipepb.FieldType_builder{
- RowType: pipepb.RowType_builder{
- Schema: pipepb.Schema_builder{
- Fields: []*pipepb.Field{
- pipepb.Field_builder{
- Name: "K",
- Type: pipepb.FieldType_builder{
- AtomicType: pipepb.AtomicType_INT32.Enum(),
- }.Build(),
- }.Build(),
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_LogicalType{
+ LogicalType: &pipepb.LogicalType{
+ Urn: "schema.nonRegisteredLogical",
+ Representation: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_RowType{
+ RowType: &pipepb.RowType{
+ Schema: &pipepb.Schema{
+ Fields: []*pipepb.Field{
+ {
+ Name: "K",
+ Type: &pipepb.FieldType{
+ TypeInfo: &pipepb.FieldType_AtomicType{
+ AtomicType: pipepb.AtomicType_INT32,
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
- }.Build(),
+ },
+ },
+ },
+ },
+ },
+ },
},
- }.Build(),
+ },
rt: reflect.TypeOf(struct {
U nonRegisteredLogical
}{}),
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/translate.go b/sdks/go/pkg/beam/core/runtime/graphx/translate.go
index 41dd4f0b351..1e30d425850 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/translate.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/translate.go
@@ -126,27 +126,27 @@ func CreateEnvironment(ctx context.Context, urn string, extractEnvironmentConfig
return nil, errors.Errorf("unsupported environment %v", urn)
case URNEnvExternal:
config := extractEnvironmentConfig(ctx)
- payload := pipepb.ExternalPayload_builder{Endpoint: pipepb.ApiServiceDescriptor_builder{Url: config}.Build()}.Build()
+ payload := &pipepb.ExternalPayload{Endpoint: &pipepb.ApiServiceDescriptor{Url: config}}
serializedPayload = protox.MustEncode(payload)
case URNEnvDocker:
fallthrough
default:
config := extractEnvironmentConfig(ctx)
- payload := pipepb.DockerPayload_builder{ContainerImage: config}.Build()
+ payload := &pipepb.DockerPayload{ContainerImage: config}
serializedPayload = protox.MustEncode(payload)
}
- return pipepb.Environment_builder{
+ return &pipepb.Environment{
Urn: urn,
Payload: serializedPayload,
Capabilities: goCapabilities(),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: URNArtifactFileType,
TypePayload: protox.MustEncode(&pipepb.ArtifactFilePayload{}),
RoleUrn: URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build(), nil
+ }, nil
}
// TODO(https://github.com/apache/beam/issues/23893): Along with scoped resource hints,
@@ -195,10 +195,10 @@ func Marshal(edges []*graph.MultiEdge, opt *Options) (*pipepb.Pipeline, error) {
}
}
- p := pipepb.Pipeline_builder{
+ p := &pipepb.Pipeline{
Components: m.build(),
Requirements: m.getRequirements(),
- }.Build()
+ }
p, err := pipelinex.Normalize(p)
if err != nil {
@@ -248,13 +248,13 @@ func newMarshaller(opt *Options) *marshaller {
}
func (m *marshaller) build() *pipepb.Components {
- return pipepb.Components_builder{
+ return &pipepb.Components{
Transforms: m.transforms,
Pcollections: m.pcollections,
WindowingStrategies: m.windowing,
Environments: m.environments,
Coders: m.coders.Build(),
- }.Build()
+ }
}
func (m *marshaller) getRequirements() []string {
@@ -292,13 +292,13 @@ func (m *marshaller) addScopeTree(s *ScopeTree) (string, error) {
metadata := m.opt.GetContextReg().ExtractTransformMetadata(s.Scope.Scope.Context)
- transform := pipepb.PTransform_builder{
+ transform := &pipepb.PTransform{
UniqueName: s.Scope.Name,
Subtransforms: subtransforms,
EnvironmentId: m.addDefaultEnv(),
Annotations: metadata.Annotations,
// DisplayData: metadata.DisplayData,
- }.Build()
+ }
if err := m.updateIfCombineComposite(s, transform); err != nil {
return "", errors.Wrapf(err, "failed to add scope tree: %v", s)
@@ -330,14 +330,14 @@ func (m *marshaller) updateIfCombineComposite(s *ScopeTree, transform *pipepb.PT
if err != nil {
return errors.Wrapf(err, "failed to update PTransform spec: %v", transform)
}
- payload := pipepb.CombinePayload_builder{
- CombineFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.CombinePayload{
+ CombineFn: &pipepb.FunctionSpec{
Urn: URNDoFn,
Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
+ },
AccumulatorCoderId: acID,
- }.Build()
- transform.SetSpec(pipepb.FunctionSpec_builder{Urn: URNCombinePerKey, Payload: protox.MustEncode(payload)}.Build())
+ }
+ transform.Spec = &pipepb.FunctionSpec{Urn: URNCombinePerKey, Payload: protox.MustEncode(payload)}
return nil
}
@@ -414,7 +414,7 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
var spec *pipepb.FunctionSpec
switch edge.Edge.Op {
case graph.Impulse:
- spec = pipepb.FunctionSpec_builder{Urn: URNImpulse}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNImpulse}
case graph.ParDo:
si := make(map[string]*pipepb.SideInput)
@@ -432,18 +432,18 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
return nil, err
}
- si[fmt.Sprintf("i%v", i)] = pipepb.SideInput_builder{
- AccessPattern: pipepb.FunctionSpec_builder{
+ si[fmt.Sprintf("i%v", i)] = &pipepb.SideInput{
+ AccessPattern: &pipepb.FunctionSpec{
Urn: URNIterableSideInput,
- }.Build(),
- ViewFn: pipepb.FunctionSpec_builder{
+ },
+ ViewFn: &pipepb.FunctionSpec{
Urn: "foo",
- }.Build(),
- WindowMappingFn: pipepb.FunctionSpec_builder{
+ },
+ WindowMappingFn: &pipepb.FunctionSpec{
Urn: mappingUrn,
- Payload: siWSpec.GetPayload(),
- }.Build(),
- }.Build()
+ Payload: siWSpec.Payload,
+ },
+ }
case graph.Map, graph.MultiMap:
// Already in a MultiMap form, don't need to add a fixed key.
@@ -456,18 +456,18 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
return nil, err
}
- si[fmt.Sprintf("i%v", i)] = pipepb.SideInput_builder{
- AccessPattern: pipepb.FunctionSpec_builder{
+ si[fmt.Sprintf("i%v", i)] = &pipepb.SideInput{
+ AccessPattern: &pipepb.FunctionSpec{
Urn: URNMultimapSideInput,
- }.Build(),
- ViewFn: pipepb.FunctionSpec_builder{
+ },
+ ViewFn: &pipepb.FunctionSpec{
Urn: "foo",
- }.Build(),
- WindowMappingFn: pipepb.FunctionSpec_builder{
+ },
+ WindowMappingFn: &pipepb.FunctionSpec{
Urn: mappingUrn,
- Payload: siWSpec.GetPayload(),
- }.Build(),
- }.Build()
+ Payload: siWSpec.Payload,
+ },
+ }
default:
return nil, errors.Errorf("unexpected input kind: %v", edge)
}
@@ -478,23 +478,23 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
return handleErr(err)
}
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNDoFn,
Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
+ },
SideInputs: si,
- }.Build()
+ }
if edge.Edge.DoFn.IsSplittable() {
coderID, err := m.coders.Add(edge.Edge.RestrictionCoder)
if err != nil {
return handleErr(err)
}
- payload.SetRestrictionCoderId(coderID)
+ payload.RestrictionCoderId = coderID
m.requirements[URNRequiresSplittableDoFn] = true
}
if _, ok := edge.Edge.DoFn.ProcessElementFn().BundleFinalization(); ok {
- payload.SetRequestsFinalization(true)
+ payload.RequestsFinalization = true
m.requirements[URNRequiresBundleFinalization] = true
}
if _, ok := edge.Edge.DoFn.ProcessElementFn().StateProvider(); ok {
@@ -520,23 +520,27 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
}
switch ps.StateType() {
case state.TypeValue:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- ReadModifyWriteSpec: pipepb.ReadModifyWriteStateSpec_builder{
- CoderId: coderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_ReadModifyWriteSpec{
+ ReadModifyWriteSpec: &pipepb.ReadModifyWriteStateSpec{
+ CoderId: coderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNBagUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeBag:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- BagSpec: pipepb.BagStateSpec_builder{
- ElementCoderId: coderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_BagSpec{
+ BagSpec: &pipepb.BagStateSpec{
+ ElementCoderId: coderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNBagUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeCombining:
cps := ps.(state.CombiningPipelineState).GetCombineFn()
f, err := graph.NewFn(cps)
@@ -555,42 +559,48 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
if err != nil {
return handleErr(err)
}
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- CombiningSpec: pipepb.CombiningStateSpec_builder{
- AccumulatorCoderId: coderID,
- CombineFn: pipepb.FunctionSpec_builder{
- Urn: "beam:combinefn:gosdk:v1",
- Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_CombiningSpec{
+ CombiningSpec: &pipepb.CombiningStateSpec{
+ AccumulatorCoderId: coderID,
+ CombineFn: &pipepb.FunctionSpec{
+ Urn: "beam:combinefn:gosdk:v1",
+ Payload: []byte(mustEncodeMultiEdge),
+ },
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNBagUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeMap:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- MapSpec: pipepb.MapStateSpec_builder{
- KeyCoderId: keyCoderID,
- ValueCoderId: coderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_MapSpec{
+ MapSpec: &pipepb.MapStateSpec{
+ KeyCoderId: keyCoderID,
+ ValueCoderId: coderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNMultiMapUserState,
- }.Build(),
- }.Build()
+ },
+ }
case state.TypeSet:
- stateSpecs[ps.StateKey()] = pipepb.StateSpec_builder{
- SetSpec: pipepb.SetStateSpec_builder{
- ElementCoderId: keyCoderID,
- }.Build(),
- Protocol: pipepb.FunctionSpec_builder{
+ stateSpecs[ps.StateKey()] = &pipepb.StateSpec{
+ Spec: &pipepb.StateSpec_SetSpec{
+ SetSpec: &pipepb.SetStateSpec{
+ ElementCoderId: keyCoderID,
+ },
+ },
+ Protocol: &pipepb.FunctionSpec{
Urn: URNMultiMapUserState,
- }.Build(),
- }.Build()
+ },
+ }
default:
return nil, errors.Errorf("State type %v not recognized for state %v", ps.StateKey(), ps)
}
}
- payload.SetStateSpecs(stateSpecs)
+ payload.StateSpecs = stateSpecs
}
if _, ok := edge.Edge.DoFn.ProcessElementFn().TimerProvider(); ok {
m.requirements[URNRequiresStatefulProcessing] = true
@@ -600,7 +610,7 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
// All timers for a single DoFn have the same key and window coders, that match the input PCollection.
mainInputID := inputs["i0"]
pCol := m.pcollections[mainInputID]
- kvCoder := m.coders.coders[pCol.GetCoderId()]
+ kvCoder := m.coders.coders[pCol.CoderId]
if kvCoder.GetSpec().GetUrn() != urnKVCoder {
return nil, errors.Errorf("timer using DoFn %v doesn't use a KV as PCollection input. Unable to extract key coder for timers, got %v", edge.Name, kvCoder.GetSpec().GetUrn())
}
@@ -614,15 +624,15 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
for _, pt := range pipelineTimers {
for timerFamilyID, timeDomain := range pt.Timers() {
- timerSpecs[timerFamilyID] = pipepb.TimerFamilySpec_builder{
+ timerSpecs[timerFamilyID] = &pipepb.TimerFamilySpec{
TimeDomain: pipepb.TimeDomain_Enum(timeDomain),
TimerFamilyCoderId: timerCoderID,
- }.Build()
+ }
}
}
- payload.SetTimerFamilySpecs(timerSpecs)
+ payload.TimerFamilySpecs = timerSpecs
}
- spec = pipepb.FunctionSpec_builder{Urn: URNParDo, Payload: protox.MustEncode(payload)}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNParDo, Payload: protox.MustEncode(payload)}
annotations = edge.Edge.DoFn.Annotations()
case graph.Combine:
@@ -630,33 +640,33 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
if err != nil {
return handleErr(err)
}
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNDoFn,
Payload: []byte(mustEncodeMultiEdge),
- }.Build(),
- }.Build()
- spec = pipepb.FunctionSpec_builder{Urn: URNParDo, Payload: protox.MustEncode(payload)}.Build()
+ },
+ }
+ spec = &pipepb.FunctionSpec{Urn: URNParDo, Payload: protox.MustEncode(payload)}
case graph.Flatten:
- spec = pipepb.FunctionSpec_builder{Urn: URNFlatten}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNFlatten}
case graph.CoGBK:
- spec = pipepb.FunctionSpec_builder{Urn: URNGBK}.Build()
+ spec = &pipepb.FunctionSpec{Urn: URNGBK}
case graph.WindowInto:
windowFn, err := makeWindowFn(edge.Edge.WindowFn)
if err != nil {
return handleErr(err)
}
- payload := pipepb.WindowIntoPayload_builder{
+ payload := &pipepb.WindowIntoPayload{
WindowFn: windowFn,
- }.Build()
- spec = pipepb.FunctionSpec_builder{Urn: URNWindow, Payload: protox.MustEncode(payload)}.Build()
+ }
+ spec = &pipepb.FunctionSpec{Urn: URNWindow, Payload: protox.MustEncode(payload)}
case graph.External:
pyld := edge.Edge.Payload
- spec = pipepb.FunctionSpec_builder{Urn: pyld.URN, Payload: pyld.Data}.Build()
+ spec = &pipepb.FunctionSpec{Urn: pyld.URN, Payload: pyld.Data}
if len(pyld.InputsMap) != 0 {
if got, want := len(pyld.InputsMap), len(edge.Edge.Input); got != want {
@@ -690,18 +700,18 @@ func (m *marshaller) addMultiEdge(edge NamedEdge) ([]string, error) {
}
var transformEnvID = ""
- if !(spec.GetUrn() == URNGBK || spec.GetUrn() == URNImpulse) {
+ if !(spec.Urn == URNGBK || spec.Urn == URNImpulse) {
transformEnvID = m.addDefaultEnv()
}
- transform := pipepb.PTransform_builder{
+ transform := &pipepb.PTransform{
UniqueName: edge.Name,
Spec: spec,
Inputs: inputs,
Outputs: outputs,
EnvironmentId: transformEnvID,
Annotations: annotations,
- }.Build()
+ }
m.transforms[id] = transform
allPIds = append(allPIds, id)
return allPIds, nil
@@ -724,22 +734,22 @@ func (m *marshaller) expandCrossLanguage(namedEdge NamedEdge) (string, error) {
inputs[tag] = nodeID(n)
}
- spec := pipepb.FunctionSpec_builder{
+ spec := &pipepb.FunctionSpec{
Urn: edge.External.Urn,
Payload: edge.External.Payload,
- }.Build()
+ }
- transform := pipepb.PTransform_builder{
+ transform := &pipepb.PTransform{
UniqueName: namedEdge.Name,
Spec: spec,
Inputs: inputs,
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
// Add the coders for output in the marshaller even if expanded is nil
// for output coder field in expansion request.
// We need this specifically for Python External Transforms.
- names := strings.Split(spec.GetUrn(), ":")
+ names := strings.Split(spec.Urn, ":")
if len(names) > 2 && names[2] == "python" {
for _, out := range edge.Output {
id, err := m.coders.Add(out.To.Coder)
@@ -761,12 +771,12 @@ func (m *marshaller) expandCrossLanguage(namedEdge NamedEdge) (string, error) {
}
outputs[fmt.Sprintf("i%v", i)] = nodeID(out.To)
}
- transform.SetOutputs(outputs)
+ transform.Outputs = outputs
environment, err := ExpandedTransform(edge.External.Expanded)
if err != nil {
return "", errors.Wrapf(err, "failed to expand cross language transform for edge: %v", namedEdge)
}
- transform.SetEnvironmentId(environment.GetEnvironmentId())
+ transform.EnvironmentId = environment.EnvironmentId
}
m.transforms[id] = transform
@@ -815,25 +825,25 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
// Inject(i)
injectID := fmt.Sprintf("%v_inject%v", id, i)
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNInject,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNInject,
Inject: &v1pb.InjectPayload{N: (int32)(i)},
})),
- }.Build(),
- }.Build()
- inject := pipepb.PTransform_builder{
+ },
+ }
+ inject := &pipepb.PTransform{
UniqueName: injectID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(payload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": nodeID(in.From)},
Outputs: map[string]string{"i0": out},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[injectID] = inject
subtransforms = append(subtransforms, injectID)
@@ -850,13 +860,13 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
}
flattenID := fmt.Sprintf("%v_flatten", id)
- flatten := pipepb.PTransform_builder{
+ flatten := &pipepb.PTransform{
UniqueName: flattenID,
- Spec: pipepb.FunctionSpec_builder{Urn: URNFlatten}.Build(),
+ Spec: &pipepb.FunctionSpec{Urn: URNFlatten},
Inputs: inputs,
Outputs: map[string]string{"i0": out},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[flattenID] = flatten
subtransforms = append(subtransforms, flattenID)
@@ -868,12 +878,12 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
}
gbkID := fmt.Sprintf("%v_gbk", id)
- gbk := pipepb.PTransform_builder{
+ gbk := &pipepb.PTransform{
UniqueName: gbkID,
- Spec: pipepb.FunctionSpec_builder{Urn: URNGBK}.Build(),
+ Spec: &pipepb.FunctionSpec{Urn: URNGBK},
Inputs: map[string]string{"i0": out},
Outputs: map[string]string{"i0": gbkOut},
- }.Build()
+ }
m.transforms[gbkID] = gbk
subtransforms = append(subtransforms, gbkID)
@@ -884,35 +894,35 @@ func (m *marshaller) expandCoGBK(edge NamedEdge) (string, error) {
}
expandID := fmt.Sprintf("%v_expand", id)
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNExpand,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNExpand,
})),
- }.Build(),
- }.Build()
- expand := pipepb.PTransform_builder{
+ },
+ }
+ expand := &pipepb.PTransform{
UniqueName: expandID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(payload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": gbkOut},
Outputs: map[string]string{"i0": nodeID(outNode)},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[id] = expand
subtransforms = append(subtransforms, id)
// Add composite for visualization
cogbkID := fmt.Sprintf("%v_cogbk", id)
- m.transforms[cogbkID] = pipepb.PTransform_builder{
+ m.transforms[cogbkID] = &pipepb.PTransform{
UniqueName: edge.Name,
Subtransforms: subtransforms,
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
return cogbkID, nil
}
@@ -992,14 +1002,16 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
if err != nil {
return handleErr(err)
}
- m.pcollections[postReify].SetWindowingStrategyId(
- m.internWindowingStrategy(pipepb.WindowingStrategy_builder{
+ m.pcollections[postReify].WindowingStrategyId =
+ m.internWindowingStrategy(&pipepb.WindowingStrategy{
// Not segregated by time...
WindowFn: windowFn,
// ...output after every element is received...
- Trigger: pipepb.Trigger_builder{
- Always: &pipepb.Trigger_Always{},
- }.Build(),
+ Trigger: &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Always_{
+ Always: &pipepb.Trigger_Always{},
+ },
+ },
// ...and after outputing, discard the output elements...
AccumulationMode: pipepb.AccumulationMode_DISCARDING,
// ...and since every pane should have 1 element,
@@ -1013,14 +1025,14 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
ClosingBehavior: pipepb.ClosingBehavior_EMIT_IF_NONEMPTY,
AllowedLateness: int64(in.From.WindowingStrategy().AllowedLateness),
OnTimeBehavior: pipepb.OnTimeBehavior_FIRE_ALWAYS,
- }.Build()))
+ })
}
// Inputs (i)
inputID := fmt.Sprintf("%v_reifyts", id)
- payload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ payload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNReshuffleInput,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNReshuffleInput,
@@ -1029,18 +1041,18 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
CoderPayloads: coderPayloads,
},
})),
- }.Build(),
- }.Build()
- input := pipepb.PTransform_builder{
+ },
+ }
+ input := &pipepb.PTransform{
UniqueName: inputID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(payload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": nodeID(in.From)},
Outputs: map[string]string{"i0": postReify},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[inputID] = input
subtransforms = append(subtransforms, inputID)
@@ -1054,12 +1066,12 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
}
gbkID := fmt.Sprintf("%v_gbk", id)
- gbk := pipepb.PTransform_builder{
+ gbk := &pipepb.PTransform{
UniqueName: gbkID,
- Spec: pipepb.FunctionSpec_builder{Urn: URNGBK}.Build(),
+ Spec: &pipepb.FunctionSpec{Urn: URNGBK},
Inputs: map[string]string{"i0": postReify},
Outputs: map[string]string{"i0": gbkOut},
- }.Build()
+ }
m.transforms[gbkID] = gbk
subtransforms = append(subtransforms, gbkID)
@@ -1069,11 +1081,11 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
if err != nil {
return handleErr(err)
}
- m.pcollections[outPCol].SetWindowingStrategyId(preservedWSID)
+ m.pcollections[outPCol].WindowingStrategyId = preservedWSID
outputID := fmt.Sprintf("%v_unreify", id)
- outputPayload := pipepb.ParDoPayload_builder{
- DoFn: pipepb.FunctionSpec_builder{
+ outputPayload := &pipepb.ParDoPayload{
+ DoFn: &pipepb.FunctionSpec{
Urn: URNReshuffleOutput,
Payload: []byte(protox.MustEncodeBase64(&v1pb.TransformPayload{
Urn: URNReshuffleOutput,
@@ -1082,31 +1094,31 @@ func (m *marshaller) expandReshuffle(edge NamedEdge) (string, error) {
CoderPayloads: coderPayloads,
},
})),
- }.Build(),
- }.Build()
- output := pipepb.PTransform_builder{
+ },
+ }
+ output := &pipepb.PTransform{
UniqueName: outputID,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNParDo,
Payload: protox.MustEncode(outputPayload),
- }.Build(),
+ },
Inputs: map[string]string{"i0": gbkOut},
Outputs: map[string]string{"i0": nodeID(outNode)},
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
m.transforms[id] = output
subtransforms = append(subtransforms, id)
// Add composite for visualization, or runner optimization
reshuffleID := fmt.Sprintf("%v_reshuffle", id)
- m.transforms[reshuffleID] = pipepb.PTransform_builder{
+ m.transforms[reshuffleID] = &pipepb.PTransform{
UniqueName: edge.Name,
Subtransforms: subtransforms,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNReshuffle,
- }.Build(),
+ },
EnvironmentId: m.addDefaultEnv(),
- }.Build()
+ }
return reshuffleID, nil
}
@@ -1152,12 +1164,12 @@ func (m *marshaller) makeNode(id, cid string, n *graph.Node) (string, error) {
return "", errors.Wrapf(err, "failed to make node %v with node id %v", n, id)
}
- col := pipepb.PCollection_builder{
+ col := &pipepb.PCollection{
UniqueName: id,
CoderId: cid,
IsBounded: boolToBounded(n.Bounded()),
WindowingStrategyId: windowingStrategyId,
- }.Build()
+ }
m.pcollections[id] = col
return id, nil
}
@@ -1182,7 +1194,7 @@ func (m *marshaller) addDefaultEnv() string {
// Add the pipeline level resource hints here for now.
// TODO(https://github.com/apache/beam/issues/23893) move to a better place for
// scoped hints in next pass, which affect number of environments set by Go pipelines.
- env.SetResourceHints(m.opt.PipelineResourceHints.Payloads())
+ env.ResourceHints = m.opt.PipelineResourceHints.Payloads()
m.environments[defaultEnvId] = env
}
return defaultEnvId
@@ -1193,7 +1205,7 @@ func (m *marshaller) addWindowingStrategy(w *window.WindowingStrategy) (string,
if err != nil {
return "", errors.Wrapf(err, "failed to add window strategy %v", w)
}
- ws.SetEnvironmentId(m.addDefaultEnv())
+ ws.EnvironmentId = m.addDefaultEnv()
return m.internWindowingStrategy(ws), nil
}
@@ -1231,7 +1243,7 @@ func MarshalWindowingStrategy(c *CoderMarshaller, w *window.WindowingStrategy) (
mergeStat = pipepb.MergeStatus_NON_MERGING
}
- ws := pipepb.WindowingStrategy_builder{
+ ws := &pipepb.WindowingStrategy{
WindowFn: windowFn,
MergeStatus: mergeStat,
WindowCoderId: windowCoderID,
@@ -1241,7 +1253,7 @@ func MarshalWindowingStrategy(c *CoderMarshaller, w *window.WindowingStrategy) (
ClosingBehavior: pipepb.ClosingBehavior_EMIT_IF_NONEMPTY,
AllowedLateness: int64(w.AllowedLateness),
OnTimeBehavior: pipepb.OnTimeBehavior_FIRE_IF_NONEMPTY,
- }.Build()
+ }
return ws, nil
}
@@ -1263,25 +1275,33 @@ func makeAccumulationMode(m window.AccumulationMode) pipepb.AccumulationMode_Enu
func makeTrigger(t trigger.Trigger) *pipepb.Trigger {
switch t := t.(type) {
case *trigger.DefaultTrigger:
- return pipepb.Trigger_builder{
- Default: &pipepb.Trigger_Default{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Default_{
+ Default: &pipepb.Trigger_Default{},
+ },
+ }
case *trigger.AlwaysTrigger:
- return pipepb.Trigger_builder{
- Always: &pipepb.Trigger_Always{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Always_{
+ Always: &pipepb.Trigger_Always{},
+ },
+ }
case *trigger.AfterAnyTrigger:
- return pipepb.Trigger_builder{
- AfterAny: pipepb.Trigger_AfterAny_builder{
- Subtriggers: extractSubtriggers(t.SubTriggers()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterAny_{
+ AfterAny: &pipepb.Trigger_AfterAny{
+ Subtriggers: extractSubtriggers(t.SubTriggers()),
+ },
+ },
+ }
case *trigger.AfterAllTrigger:
- return pipepb.Trigger_builder{
- AfterAll: pipepb.Trigger_AfterAll_builder{
- Subtriggers: extractSubtriggers(t.SubTriggers()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterAll_{
+ AfterAll: &pipepb.Trigger_AfterAll{
+ Subtriggers: extractSubtriggers(t.SubTriggers()),
+ },
+ },
+ }
case *trigger.AfterProcessingTimeTrigger:
if len(t.TimestampTransforms()) == 0 {
panic("AfterProcessingTime trigger set without a delay or alignment.")
@@ -1291,66 +1311,88 @@ func makeTrigger(t trigger.Trigger) *pipepb.Trigger {
var ttp *pipepb.TimestampTransform
switch tt := tt.(type) {
case trigger.DelayTransform:
- ttp = pipepb.TimestampTransform_builder{
- Delay: pipepb.TimestampTransform_Delay_builder{DelayMillis: tt.Delay}.Build()}.Build()
+ ttp = &pipepb.TimestampTransform{
+ TimestampTransform: &pipepb.TimestampTransform_Delay_{
+ Delay: &pipepb.TimestampTransform_Delay{DelayMillis: tt.Delay},
+ }}
case trigger.AlignToTransform:
- ttp = pipepb.TimestampTransform_builder{
- AlignTo: pipepb.TimestampTransform_AlignTo_builder{
- Period: tt.Period,
- Offset: tt.Offset,
- }.Build()}.Build()
+ ttp = &pipepb.TimestampTransform{
+ TimestampTransform: &pipepb.TimestampTransform_AlignTo_{
+ AlignTo: &pipepb.TimestampTransform_AlignTo{
+ Period: tt.Period,
+ Offset: tt.Offset,
+ },
+ }}
}
tts = append(tts, ttp)
}
- return pipepb.Trigger_builder{
- AfterProcessingTime: pipepb.Trigger_AfterProcessingTime_builder{
- TimestampTransforms: tts,
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterProcessingTime_{
+ AfterProcessingTime: &pipepb.Trigger_AfterProcessingTime{
+ TimestampTransforms: tts,
+ },
+ },
+ }
case *trigger.AfterCountTrigger:
- return pipepb.Trigger_builder{
- ElementCount: pipepb.Trigger_ElementCount_builder{ElementCount: t.ElementCount()}.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_ElementCount_{
+ ElementCount: &pipepb.Trigger_ElementCount{ElementCount: t.ElementCount()},
+ },
+ }
case *trigger.AfterEndOfWindowTrigger:
var lateTrigger *pipepb.Trigger
if t.Late() != nil {
lateTrigger = makeTrigger(t.Late())
}
- return pipepb.Trigger_builder{
- AfterEndOfWindow: pipepb.Trigger_AfterEndOfWindow_builder{
- EarlyFirings: makeTrigger(t.Early()),
- LateFirings: lateTrigger,
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterEndOfWindow_{
+ AfterEndOfWindow: &pipepb.Trigger_AfterEndOfWindow{
+ EarlyFirings: makeTrigger(t.Early()),
+ LateFirings: lateTrigger,
+ },
+ },
+ }
case *trigger.RepeatTrigger:
- return pipepb.Trigger_builder{
- Repeat: pipepb.Trigger_Repeat_builder{Subtrigger: makeTrigger(t.SubTrigger())}.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Repeat_{
+ Repeat: &pipepb.Trigger_Repeat{Subtrigger: makeTrigger(t.SubTrigger())},
+ },
+ }
case *trigger.NeverTrigger:
- return pipepb.Trigger_builder{
- Never: &pipepb.Trigger_Never{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Never_{
+ Never: &pipepb.Trigger_Never{},
+ },
+ }
case *trigger.AfterSynchronizedProcessingTimeTrigger:
- return pipepb.Trigger_builder{
- AfterSynchronizedProcessingTime: &pipepb.Trigger_AfterSynchronizedProcessingTime{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterSynchronizedProcessingTime_{
+ AfterSynchronizedProcessingTime: &pipepb.Trigger_AfterSynchronizedProcessingTime{},
+ },
+ }
case *trigger.OrFinallyTrigger:
- return pipepb.Trigger_builder{
- OrFinally: pipepb.Trigger_OrFinally_builder{
- Main: makeTrigger(t.Main()),
- Finally: makeTrigger(t.Finally()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_OrFinally_{
+ OrFinally: &pipepb.Trigger_OrFinally{
+ Main: makeTrigger(t.Main()),
+ Finally: makeTrigger(t.Finally()),
+ },
+ },
+ }
case *trigger.AfterEachTrigger:
- return pipepb.Trigger_builder{
- AfterEach: pipepb.Trigger_AfterEach_builder{
- Subtriggers: extractSubtriggers(t.Subtriggers()),
- }.Build(),
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_AfterEach_{
+ AfterEach: &pipepb.Trigger_AfterEach{
+ Subtriggers: extractSubtriggers(t.Subtriggers()),
+ },
+ },
+ }
default:
- return pipepb.Trigger_builder{
- Default: &pipepb.Trigger_Default{},
- }.Build()
+ return &pipepb.Trigger{
+ Trigger: &pipepb.Trigger_Default_{
+ Default: &pipepb.Trigger_Default{},
+ },
+ }
}
}
@@ -1369,37 +1411,37 @@ func extractSubtriggers(t []trigger.Trigger) []*pipepb.Trigger {
func makeWindowFn(w *window.Fn) (*pipepb.FunctionSpec, error) {
switch w.Kind {
case window.GlobalWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNGlobalWindowsWindowFn,
- }.Build(), nil
+ }, nil
case window.FixedWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNFixedWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.FixedWindowsPayload_builder{
+ &pipepb.FixedWindowsPayload{
Size: durationpb.New(w.Size),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.SlidingWindows:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNSlidingWindowsWindowFn,
Payload: protox.MustEncode(
- pipepb.SlidingWindowsPayload_builder{
+ &pipepb.SlidingWindowsPayload{
Size: durationpb.New(w.Size),
Period: durationpb.New(w.Period),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
case window.Sessions:
- return pipepb.FunctionSpec_builder{
+ return &pipepb.FunctionSpec{
Urn: URNSessionsWindowFn,
Payload: protox.MustEncode(
- pipepb.SessionWindowsPayload_builder{
+ &pipepb.SessionWindowsPayload{
GapSize: durationpb.New(w.Gap),
- }.Build(),
+ },
),
- }.Build(), nil
+ }, nil
default:
return nil, errors.Errorf("unexpected windowing strategy: %v", w)
}
@@ -1449,11 +1491,11 @@ func UpdateDefaultEnvWorkerType(typeUrn string, pyld []byte, p *pipepb.Pipeline)
return errors.Errorf("unable to find default Go environment with ID %q", defaultEnvId)
}
for _, dep := range env.GetDependencies() {
- if dep.GetRoleUrn() != URNArtifactGoWorkerRole {
+ if dep.RoleUrn != URNArtifactGoWorkerRole {
continue
}
- dep.SetTypeUrn(typeUrn)
- dep.SetTypePayload(pyld)
+ dep.TypeUrn = typeUrn
+ dep.TypePayload = pyld
return nil
}
return errors.Errorf("unable to find dependency with %q role in environment with ID %q,", URNArtifactGoWorkerRole, defaultEnvId)
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go b/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go
index 2f52818200d..e18a5f97796 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/translate_test.go
@@ -170,12 +170,12 @@ func TestMarshal(t *testing.T) {
t.Fatalf("got %v edges, want %v", got, want)
}
- payload, err := proto.Marshal(pipepb.DockerPayload_builder{ContainerImage: "foo"}.Build())
+ payload, err := proto.Marshal(&pipepb.DockerPayload{ContainerImage: "foo"})
if err != nil {
t.Fatal(err)
}
p, err := graphx.Marshal(edges,
- &graphx.Options{Environment: pipepb.Environment_builder{Urn: "beam:env:docker:v1", Payload: payload}.Build()})
+ &graphx.Options{Environment: &pipepb.Environment{Urn: "beam:env:docker:v1", Payload: payload}})
if err != nil {
t.Fatal(err)
}
@@ -236,12 +236,12 @@ func TestMarshal_PTransformAnnotations(t *testing.T) {
t.Fatal(err)
}
- payload, err := proto.Marshal(pipepb.DockerPayload_builder{ContainerImage: "foo"}.Build())
+ payload, err := proto.Marshal(&pipepb.DockerPayload{ContainerImage: "foo"})
if err != nil {
t.Fatal(err)
}
p, err := graphx.Marshal(edges,
- &graphx.Options{Environment: pipepb.Environment_builder{Urn: "beam:env:docker:v1", Payload: payload}.Build(), ContextReg: &creg})
+ &graphx.Options{Environment: &pipepb.Environment{Urn: "beam:env:docker:v1", Payload: payload}, ContextReg: &creg})
if err != nil {
t.Fatal(err)
}
@@ -317,19 +317,19 @@ func TestCreateEnvironment(t *testing.T) {
name: "external",
urn: graphx.URNEnvExternal,
payload: func(name string) []byte {
- return protox.MustEncode(pipepb.ExternalPayload_builder{
- Endpoint: pipepb.ApiServiceDescriptor_builder{
+ return protox.MustEncode(&pipepb.ExternalPayload{
+ Endpoint: &pipepb.ApiServiceDescriptor{
Url: name,
- }.Build(),
- }.Build())
+ },
+ })
},
}, {
name: "docker",
urn: graphx.URNEnvDocker,
payload: func(name string) []byte {
- return protox.MustEncode(pipepb.DockerPayload_builder{
+ return protox.MustEncode(&pipepb.DockerPayload{
ContainerImage: name,
- }.Build())
+ })
},
},
}
@@ -340,17 +340,17 @@ func TestCreateEnvironment(t *testing.T) {
if err != nil {
t.Errorf("CreateEnvironment(%v) = %v error, want nil", test.urn, err)
}
- want := pipepb.Environment_builder{
+ want := &pipepb.Environment{
Urn: test.urn,
Payload: test.payload(test.name),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: graphx.URNArtifactFileType,
TypePayload: protox.MustEncode(&pipepb.ArtifactFilePayload{}),
RoleUrn: graphx.URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build()
+ }
opts := []cmp.Option{
protocmp.Transform(),
// Ignore the capabilities field, since we can't access that method here.
@@ -365,87 +365,87 @@ func TestCreateEnvironment(t *testing.T) {
func TestUpdateDefaultEnvWorkerType(t *testing.T) {
t.Run("noEnvs", func(t *testing.T) {
- if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, pipepb.Pipeline_builder{
+ if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, &pipepb.Pipeline{
Components: &pipepb.Components{},
- }.Build()); err == nil {
+ }); err == nil {
t.Error("UpdateDefaultEnvWorkerType(<emptyEnv>) no error, want err")
}
})
t.Run("noGoEnvs", func(t *testing.T) {
- if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: map[string]*pipepb.Environment{
- "java": pipepb.Environment_builder{Urn: "java"}.Build(),
- "python": pipepb.Environment_builder{Urn: "python"}.Build(),
- "typescript": pipepb.Environment_builder{Urn: "typescript"}.Build(),
+ "java": {Urn: "java"},
+ "python": {Urn: "python"},
+ "typescript": {Urn: "typescript"},
},
- }.Build(),
- }.Build()); err == nil {
+ },
+ }); err == nil {
t.Error("UpdateDefaultEnvWorkerType(<noGoEnvs>) no error, want err")
}
})
t.Run("badGoEnv", func(t *testing.T) {
- if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ if err := graphx.UpdateDefaultEnvWorkerType("unused", nil, &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: map[string]*pipepb.Environment{
- "java": pipepb.Environment_builder{Urn: "java"}.Build(),
- "python": pipepb.Environment_builder{Urn: "python"}.Build(),
- "typescript": pipepb.Environment_builder{Urn: "typescript"}.Build(),
- "go": pipepb.Environment_builder{
+ "java": {Urn: "java"},
+ "python": {Urn: "python"},
+ "typescript": {Urn: "typescript"},
+ "go": {
Urn: "test",
Payload: []byte("test"),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
RoleUrn: "unset",
- }.Build(),
+ },
},
- }.Build(),
+ },
},
- }.Build(),
- }.Build()); err == nil {
+ },
+ }); err == nil {
t.Error("UpdateDefaultEnvWorkerType(<badGoEnv>) no error, want err")
}
})
t.Run("goEnv", func(t *testing.T) {
wantUrn := graphx.URNArtifactFileType
- wantPyld := protox.MustEncode(pipepb.ArtifactFilePayload_builder{
+ wantPyld := protox.MustEncode(&pipepb.ArtifactFilePayload{
Path: "good",
- }.Build())
- p := pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ })
+ p := &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: map[string]*pipepb.Environment{
- "java": pipepb.Environment_builder{Urn: "java"}.Build(),
- "python": pipepb.Environment_builder{Urn: "python"}.Build(),
- "typescript": pipepb.Environment_builder{Urn: "typescript"}.Build(),
- "go": pipepb.Environment_builder{
+ "java": {Urn: "java"},
+ "python": {Urn: "python"},
+ "typescript": {Urn: "typescript"},
+ "go": {
Urn: "test",
Payload: []byte("test"),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: "to be removed",
TypePayload: nil,
RoleUrn: graphx.URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build(),
+ },
},
- }.Build(),
- }.Build()
+ },
+ }
if err := graphx.UpdateDefaultEnvWorkerType(wantUrn, wantPyld, p); err != nil {
t.Errorf("UpdateDefaultEnvWorkerType(<goEnv>) = %v, want nil", err)
}
got := p.GetComponents().GetEnvironments()["go"]
- want := pipepb.Environment_builder{
+ want := &pipepb.Environment{
Urn: "test",
Payload: []byte("test"),
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: wantUrn,
TypePayload: wantPyld,
RoleUrn: graphx.URNArtifactGoWorkerRole,
- }.Build(),
+ },
},
- }.Build()
+ }
opts := []cmp.Option{
protocmp.Transform(),
}
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/xlang.go b/sdks/go/pkg/beam/core/runtime/graphx/xlang.go
index b02af765c37..baee3c00093 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/xlang.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/xlang.go
@@ -34,7 +34,7 @@ func mergeExpandedWithPipeline(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
exp := e.External.Expanded
id := fmt.Sprintf("e%v", e.ID())
- p.SetRequirements(append(p.GetRequirements(), exp.Requirements...))
+ p.Requirements = append(p.Requirements, exp.Requirements...)
// Adding components of the Expanded Transforms to the current Pipeline
components, err := ExpandedComponents(exp)
@@ -42,16 +42,16 @@ func mergeExpandedWithPipeline(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
panic(err)
}
for k, v := range components.GetTransforms() {
- p.GetComponents().GetTransforms()[k] = v
+ p.Components.Transforms[k] = v
}
for k, v := range components.GetPcollections() {
- p.GetComponents().GetPcollections()[k] = v
+ p.Components.Pcollections[k] = v
}
for k, v := range components.GetWindowingStrategies() {
- p.GetComponents().GetWindowingStrategies()[k] = v
+ p.Components.WindowingStrategies[k] = v
}
for k, v := range components.GetCoders() {
- p.GetComponents().GetCoders()[k] = v
+ p.Components.Coders[k] = v
}
for k, v := range components.GetEnvironments() {
if k == defaultEnvId {
@@ -61,14 +61,14 @@ func mergeExpandedWithPipeline(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
// using unique namespace prevents collision.
continue
}
- p.GetComponents().GetEnvironments()[k] = v
+ p.Components.Environments[k] = v
}
transform, err := ExpandedTransform(exp)
if err != nil {
panic(err)
}
- p.GetComponents().GetTransforms()[id] = transform
+ p.Components.Transforms[id] = transform
}
}
@@ -102,7 +102,7 @@ func purgeOutputInput(edges []*graph.MultiEdge, p *pipepb.Pipeline) {
}
idxMap[nodeID] = pcolID
- delete(components.GetPcollections(), nodeID)
+ delete(components.Pcollections, nodeID)
}
}
@@ -200,13 +200,13 @@ func AddFakeImpulses(p *pipepb.Pipeline) {
key := fmt.Sprintf("%s_%s", "impulse", tag)
output := map[string]string{"out": id}
- impulse := pipepb.PTransform_builder{
+ impulse := &pipepb.PTransform{
UniqueName: key,
- Spec: pipepb.FunctionSpec_builder{
+ Spec: &pipepb.FunctionSpec{
Urn: URNImpulse,
- }.Build(),
+ },
Outputs: output,
- }.Build()
+ }
transforms[key] = impulse
}
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go b/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go
index 283e49e691c..7b68a770768 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go
@@ -129,17 +129,17 @@ func TestExternalOutputs(t *testing.T) {
}
func newTransform(name string) *pipepb.PTransform {
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
UniqueName: name,
- }.Build()
+ }
}
func newComponents(ts []string) *pipepb.Components {
components := &pipepb.Components{}
- components.SetTransforms(make(map[string]*pipepb.PTransform))
+ components.Transforms = make(map[string]*pipepb.PTransform)
for id, t := range ts {
- components.GetTransforms()[fmt.Sprint(id)] = newTransform(t)
+ components.Transforms[fmt.Sprint(id)] = newTransform(t)
}
return components
@@ -205,23 +205,23 @@ var testExternal = graph.ExternalTransform{
Expanded: &testExpanded,
}
-var testComponents = pipepb.Components_builder{
- Transforms: map[string]*pipepb.PTransform{"transform_id1": pipepb.PTransform_builder{UniqueName: "test_components_transform"}.Build()},
- Pcollections: map[string]*pipepb.PCollection{"pcollection_id1": pipepb.PCollection_builder{UniqueName: "test_components_pcollection"}.Build()},
- WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id1": pipepb.WindowingStrategy_builder{WindowCoderId: "test_components_windowing"}.Build()},
- Coders: map[string]*pipepb.Coder{"coder_id1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "test_components_coder"}.Build()}.Build()},
- Environments: map[string]*pipepb.Environment{"environment_id1": pipepb.Environment_builder{Urn: "test_components_environment"}.Build()},
-}.Build()
+var testComponents = pipepb.Components{
+ Transforms: map[string]*pipepb.PTransform{"transform_id1": {UniqueName: "test_components_transform"}},
+ Pcollections: map[string]*pipepb.PCollection{"pcollection_id1": {UniqueName: "test_components_pcollection"}},
+ WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id1": {WindowCoderId: "test_components_windowing"}},
+ Coders: map[string]*pipepb.Coder{"coder_id1": {Spec: &pipepb.FunctionSpec{Urn: "test_components_coder"}}},
+ Environments: map[string]*pipepb.Environment{"environment_id1": {Urn: "test_components_environment"}},
+}
var testRequirements = []string{"test_requirement1", "test_requirement2"}
-var testTransform = pipepb.PTransform_builder{
+var testTransform = pipepb.PTransform{
UniqueName: "test_transform",
-}.Build()
+}
var testExpanded = graph.ExpandedTransform{
- Components: testComponents,
- Transform: testTransform,
+ Components: &testComponents,
+ Transform: &testTransform,
Requirements: testRequirements,
}
@@ -232,23 +232,23 @@ var testExternal2 = graph.ExternalTransform{
Expanded: &testExpanded2,
}
-var testComponents2 = pipepb.Components_builder{
- Transforms: map[string]*pipepb.PTransform{"transform_id2": pipepb.PTransform_builder{UniqueName: "test_components2_transform"}.Build()},
- Pcollections: map[string]*pipepb.PCollection{"pcollection_id2": pipepb.PCollection_builder{UniqueName: "test_components2_pcollection"}.Build()},
- WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id2": pipepb.WindowingStrategy_builder{WindowCoderId: "test_components2_windowing"}.Build()},
- Coders: map[string]*pipepb.Coder{"coder_id2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "test_components2_coder"}.Build()}.Build()},
- Environments: map[string]*pipepb.Environment{"environment_id2": pipepb.Environment_builder{Urn: "test_components2_environment"}.Build()},
-}.Build()
+var testComponents2 = pipepb.Components{
+ Transforms: map[string]*pipepb.PTransform{"transform_id2": {UniqueName: "test_components2_transform"}},
+ Pcollections: map[string]*pipepb.PCollection{"pcollection_id2": {UniqueName: "test_components2_pcollection"}},
+ WindowingStrategies: map[string]*pipepb.WindowingStrategy{"windowing_id2": {WindowCoderId: "test_components2_windowing"}},
+ Coders: map[string]*pipepb.Coder{"coder_id2": {Spec: &pipepb.FunctionSpec{Urn: "test_components2_coder"}}},
+ Environments: map[string]*pipepb.Environment{"environment_id2": {Urn: "test_components2_environment"}},
+}
var testRequirements2 = []string{"test_requirement2", "test_requirement3"}
-var testTransform2 = pipepb.PTransform_builder{
+var testTransform2 = pipepb.PTransform{
UniqueName: "test_transform2",
-}.Build()
+}
var testExpanded2 = graph.ExpandedTransform{
- Components: testComponents2,
- Transform: testTransform2,
+ Components: &testComponents2,
+ Transform: &testTransform2,
Requirements: testRequirements2,
}
@@ -314,13 +314,13 @@ func TestMergeExpandedWithPipeline(t *testing.T) {
g := graph.New()
edges := test.makeEdges(g)
var p pipepb.Pipeline
- p.SetComponents(pipepb.Components_builder{
+ p.Components = &pipepb.Components{
Transforms: make(map[string]*pipepb.PTransform),
Pcollections: make(map[string]*pipepb.PCollection),
WindowingStrategies: make(map[string]*pipepb.WindowingStrategy),
Coders: make(map[string]*pipepb.Coder),
Environments: make(map[string]*pipepb.Environment),
- }.Build())
+ }
mergeExpandedWithPipeline(edges, &p)
// Check that all wanted expanded components have been added to
@@ -371,7 +371,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
// Transforms
for k, wantVal := range wantComps.GetTransforms() {
- gotVal, ok := gotComps.GetTransforms()[k]
+ gotVal, ok := gotComps.Transforms[k]
if !ok {
t.Errorf("Pipeline components missing expected transform with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -380,7 +380,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// PCollections
for k, wantVal := range wantComps.GetPcollections() {
- gotVal, ok := gotComps.GetPcollections()[k]
+ gotVal, ok := gotComps.Pcollections[k]
if !ok {
t.Errorf("Pipeline components missing expected PCollection with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -389,7 +389,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// Windowing Strategies
for k, wantVal := range wantComps.GetWindowingStrategies() {
- gotVal, ok := gotComps.GetWindowingStrategies()[k]
+ gotVal, ok := gotComps.WindowingStrategies[k]
if !ok {
t.Errorf("Pipeline components missing expected windowing strategy with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -398,7 +398,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// Coders
for k, wantVal := range wantComps.GetCoders() {
- gotVal, ok := gotComps.GetCoders()[k]
+ gotVal, ok := gotComps.Coders[k]
if !ok {
t.Errorf("Pipeline components missing expected coder with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
@@ -407,7 +407,7 @@ func validateComponents(t *testing.T, wantComps, gotComps *pipepb.Components) {
}
// Environments
for k, wantVal := range wantComps.GetEnvironments() {
- gotVal, ok := gotComps.GetEnvironments()[k]
+ gotVal, ok := gotComps.Environments[k]
if !ok {
t.Errorf("Pipeline components missing expected environment with key \"%v\": %v", k, wantVal)
} else if d := cmp.Diff(wantVal, gotVal, protocmp.Transform()); d != "" {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
index dc7733a3111..6385098d013 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
@@ -594,17 +594,17 @@ func (w *dataWriter) Close() error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
delete(w.ch.writers[w.id.instID], w.id.ptransformID)
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Data: []*fnpb.Elements_Data{
- fnpb.Elements_Data_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
// TODO(https://github.com/apache/beam/issues/21164): Set IsLast true on final flush instead of w/empty sentinel?
// Empty data == sentinel
IsLast: true,
- }.Build(),
+ },
},
- }.Build()
+ }
return w.send(msg)
}
@@ -617,15 +617,15 @@ func (w *dataWriter) Flush() error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Data: []*fnpb.Elements_Data{
- fnpb.Elements_Data_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
Data: w.buf,
- }.Build(),
+ },
},
- }.Build()
+ }
if l := len(w.buf); l > largeBufferNotificationThreshold {
log.Infof(context.TODO(), "dataWriter[%v;%v].Flush flushed large buffer of length %d", w.id, w.ch.id, l)
}
@@ -707,16 +707,16 @@ func (w *timerWriter) Close() error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
delete(w.ch.timerWriters[w.id.instID], timerKey{w.id.ptransformID, w.timerFamilyID})
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
- fnpb.Elements_Timers_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
TimerFamilyId: w.timerFamilyID,
IsLast: true,
- }.Build(),
+ },
},
- }.Build()
+ }
return w.send(msg)
}
@@ -724,16 +724,16 @@ func (w *timerWriter) writeTimers(p []byte) error {
w.ch.mu.Lock()
defer w.ch.mu.Unlock()
- msg := fnpb.Elements_builder{
+ msg := &fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
- fnpb.Elements_Timers_builder{
+ {
InstructionId: string(w.id.instID),
TransformId: w.id.ptransformID,
TimerFamilyId: w.timerFamilyID,
Timers: p,
- }.Build(),
+ },
},
- }.Build()
+ }
return w.send(msg)
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
index dc197e00fd0..9f6f8a986a3 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
@@ -52,20 +52,20 @@ func (f *fakeDataClient) Recv() (*fnpb.Elements, error) {
f.calls++
data := []byte{1, 2, 3, 4, 1, 2, 3, 4}
- elemData := fnpb.Elements_Data_builder{
+ elemData := fnpb.Elements_Data{
InstructionId: "inst_ref",
Data: data,
TransformId: "ptr",
- }.Build()
+ }
if f.isLastCall == f.calls {
- elemData.SetIsLast(true)
+ elemData.IsLast = true
}
- msg := &fnpb.Elements{}
+ msg := fnpb.Elements{}
// Send extraData more than the number of elements buffered in the channel.
for i := 0; i < bufElements+extraData; i++ {
- msg.SetData(append(msg.GetData(), elemData))
+ msg.Data = append(msg.Data, &elemData)
}
// The first two calls fill up the buffer completely to stimulate the deadlock
@@ -73,16 +73,16 @@ func (f *fakeDataClient) Recv() (*fnpb.Elements, error) {
// Subsequent calls return no data.
switch f.calls {
case 1:
- return msg, f.err
+ return &msg, f.err
case 2:
- return msg, f.err
+ return &msg, f.err
case 3:
- elemData.SetData([]byte{})
- msg.SetData([]*fnpb.Elements_Data{elemData})
+ elemData.Data = []byte{}
+ msg.Data = []*fnpb.Elements_Data{&elemData}
// Broadcasting done here means that this code providing messages
// has not been blocked by the bug blocking the dataReader
// from getting more messages.
- return msg, f.err
+ return &msg, f.err
default:
f.done <- true
return nil, io.EOF
@@ -165,16 +165,16 @@ func TestElementChan(t *testing.T) {
}
timerElm := func(val byte, isLast bool) *fnpb.Elements_Timers {
- return fnpb.Elements_Timers_builder{InstructionId: instID, TransformId: timerID, Timers: []byte{val}, IsLast: isLast, TimerFamilyId: timerFamily}.Build()
+ return &fnpb.Elements_Timers{InstructionId: instID, TransformId: timerID, Timers: []byte{val}, IsLast: isLast, TimerFamilyId: timerFamily}
}
dataElm := func(val byte, isLast bool) *fnpb.Elements_Data {
- return fnpb.Elements_Data_builder{InstructionId: instID, TransformId: dataID, Data: []byte{val}, IsLast: isLast}.Build()
+ return &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{val}, IsLast: isLast}
}
noTimerElm := func() *fnpb.Elements_Timers {
- return fnpb.Elements_Timers_builder{InstructionId: instID, TransformId: timerID, Timers: []byte{}, IsLast: true}.Build()
+ return &fnpb.Elements_Timers{InstructionId: instID, TransformId: timerID, Timers: []byte{}, IsLast: true}
}
noDataElm := func() *fnpb.Elements_Data {
- return fnpb.Elements_Data_builder{InstructionId: instID, TransformId: dataID, Data: []byte{}, IsLast: true}.Build()
+ return &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{}, IsLast: true}
}
openChan := func(ctx context.Context, t *testing.T, c *DataChannel, timers ...string) <-chan exec.Elements {
t.Helper()
@@ -196,13 +196,13 @@ func TestElementChan(t *testing.T) {
name: "ReadThenData_singleRecv",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(1, false),
dataElm(2, false),
dataElm(3, true),
},
- }.Build())
+ })
return elms
},
wantSum: 6, wantCount: 3,
@@ -210,9 +210,9 @@ func TestElementChan(t *testing.T) {
name: "ReadThenData_multipleRecv",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(1, false)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(2, false)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(2, false)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
@@ -220,14 +220,14 @@ func TestElementChan(t *testing.T) {
name: "ReadThenNoData",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
return elms
},
wantSum: 0, wantCount: 0,
}, {
name: "NoDataThenRead",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
elms := openChan(ctx, t, c)
return elms
},
@@ -235,7 +235,7 @@ func TestElementChan(t *testing.T) {
}, {
name: "NoDataInstEndsThenRead",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
c.removeInstruction(instID)
elms := openChan(ctx, t, c)
return elms
@@ -245,18 +245,18 @@ func TestElementChan(t *testing.T) {
name: "ReadThenDataAndTimers",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(1, false)}}.Build())
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
}, {
name: "AllDataAndTimersThenRead",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(1, false)}}.Build())
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}}.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(1, false)}})
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(2, true)}})
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
elms := openChan(ctx, t, c, timerID)
return elms
},
@@ -264,30 +264,30 @@ func TestElementChan(t *testing.T) {
}, {
name: "DataThenReaderThenLast",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(1, false),
dataElm(2, false),
},
- }.Build())
+ })
elms := openChan(ctx, t, c)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
}, {
name: "PartialTimersAllDataReadThenLastTimer",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
timerElm(1, false),
timerElm(2, false),
},
- }.Build())
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ })
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(3, true)}})
return elms
},
@@ -295,16 +295,16 @@ func TestElementChan(t *testing.T) {
}, {
name: "AllTimerThenReaderThenDataClose",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Timers: []*fnpb.Elements_Timers{
timerElm(1, false),
timerElm(2, false),
timerElm(3, true),
},
- }.Build())
+ })
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
return elms
},
@@ -312,28 +312,28 @@ func TestElementChan(t *testing.T) {
}, {
name: "NoTimersThenReaderThenNoData",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{noTimerElm()}}.Build())
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{noTimerElm()}})
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{noDataElm()}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{noDataElm()}})
return elms
},
wantSum: 0, wantCount: 0,
}, {
name: "SomeTimersThenReaderThenAData",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)}}.Build())
+ client.Send(&fnpb.Elements{Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)}})
elms := openChan(ctx, t, c, timerID)
- client.Send(fnpb.Elements_builder{Data: []*fnpb.Elements_Data{dataElm(3, true)}}.Build())
+ client.Send(&fnpb.Elements{Data: []*fnpb.Elements_Data{dataElm(3, true)}})
return elms
},
wantSum: 6, wantCount: 3,
}, {
name: "SomeTimersAndADataThenReader",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)},
Data: []*fnpb.Elements_Data{dataElm(3, true)},
- }.Build())
+ })
elms := openChan(ctx, t, c, timerID)
return elms
},
@@ -341,12 +341,12 @@ func TestElementChan(t *testing.T) {
}, {
name: "PartialReadThenEndInstruction",
sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(1, false),
dataElm(2, false),
},
- }.Build())
+ })
elms := openChan(ctx, t, c)
var sum int
e := <-elms
@@ -362,12 +362,12 @@ func TestElementChan(t *testing.T) {
c.removeInstruction(instID)
// Instruction is ended, so further data for this instruction is ignored.
- client.Send(fnpb.Elements_builder{
+ client.Send(&fnpb.Elements{
Data: []*fnpb.Elements_Data{
dataElm(3, false),
dataElm(4, true),
},
- }.Build())
+ })
elms = openChan(ctx, t, c)
return elms
@@ -410,12 +410,12 @@ func BenchmarkElementChan(b *testing.B) {
if err != nil {
b.Errorf("Unexpected error from OpenElementChan(%v, %v, nil): %v", dataID, instID, err)
}
- e := fnpb.Elements_Data_builder{InstructionId: instID, TransformId: dataID, Data: []byte{1}, IsLast: false}.Build()
+ e := &fnpb.Elements_Data{InstructionId: instID, TransformId: dataID, Data: []byte{1}, IsLast: false}
es := make([]*fnpb.Elements_Data, 0, bench.size)
for i := 0; i < bench.size; i++ {
es = append(es, e)
}
- batch := fnpb.Elements_builder{Data: es}.Build()
+ batch := &fnpb.Elements{Data: es}
var wg sync.WaitGroup
wg.Add(1)
go func() {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness.go b/sdks/go/pkg/beam/core/runtime/harness/harness.go
index 2b32efc02eb..20aad81c123 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/harness.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/harness.go
@@ -40,7 +40,6 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/grpcx"
"golang.org/x/sync/singleflight"
"google.golang.org/grpc"
- "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -110,7 +109,7 @@ func MainWithOptions(ctx context.Context, loggingEndpoint, controlEndpoint strin
client := fnpb.NewBeamFnControlClient(conn)
lookupDesc := func(id bundleDescriptorID) (*fnpb.ProcessBundleDescriptor, error) {
- return client.GetProcessBundleDescriptor(ctx, fnpb.GetProcessBundleDescriptorRequest_builder{ProcessBundleDescriptorId: string(id)}.Build())
+ return client.GetProcessBundleDescriptor(ctx, &fnpb.GetProcessBundleDescriptorRequest{ProcessBundleDescriptorId: string(id)})
}
stub, err := client.Control(ctx)
@@ -375,10 +374,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
}
c.mu.Unlock()
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- Register: &fnpb.RegisterResponse{},
- }.Build()
+ Response: &fnpb.InstructionResponse_Register{
+ Register: &fnpb.RegisterResponse{},
+ },
+ }
case req.GetProcessBundle() != nil:
msg := req.GetProcessBundle()
@@ -466,15 +467,15 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
if len(checkpoints) > 0 {
for _, cp := range checkpoints {
for _, r := range cp.SR.RS {
- rRoots = append(rRoots, fnpb.DelayedBundleApplication_builder{
- Application: fnpb.BundleApplication_builder{
+ rRoots = append(rRoots, &fnpb.DelayedBundleApplication{
+ Application: &fnpb.BundleApplication{
TransformId: cp.SR.TId,
InputId: cp.SR.InId,
Element: r,
OutputWatermarks: cp.SR.OW,
- }.Build(),
+ },
RequestedTimeDelay: durationpb.New(cp.Reapply),
- }.Build())
+ })
}
}
}
@@ -490,15 +491,17 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
if err != nil {
return fail(ctx, instID, "process bundle failed for instruction %v using plan %v : %v", instID, bdID, err)
}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- ProcessBundle: fnpb.ProcessBundleResponse_builder{
- ResidualRoots: rRoots,
- MonitoringData: pylds,
- MonitoringInfos: mons,
- RequiresFinalization: requiresFinalization,
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_ProcessBundle{
+ ProcessBundle: &fnpb.ProcessBundleResponse{
+ ResidualRoots: rRoots,
+ MonitoringData: pylds,
+ MonitoringInfos: mons,
+ RequiresFinalization: requiresFinalization,
+ },
+ },
+ }
case req.GetFinalizeBundle() != nil:
msg := req.GetFinalizeBundle()
@@ -518,10 +521,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
c.plans[af.bdID] = append(c.plans[af.bdID], af.plan)
delete(c.awaitingFinalization, ref)
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- FinalizeBundle: &fnpb.FinalizeBundleResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_FinalizeBundle{
+ FinalizeBundle: &fnpb.FinalizeBundleResponse{},
+ },
+ }
case req.GetProcessBundleProgress() != nil:
msg := req.GetProcessBundleProgress()
@@ -533,22 +538,26 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
return resp
}
if plan == nil && resp == nil {
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- ProcessBundleProgress: &fnpb.ProcessBundleProgressResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_ProcessBundleProgress{
+ ProcessBundleProgress: &fnpb.ProcessBundleProgressResponse{},
+ },
+ }
}
mons, pylds, consumingReceivedData := monitoring(plan, store, c.runnerCapabilities[URNMonitoringInfoShortID])
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- ProcessBundleProgress: fnpb.ProcessBundleProgressResponse_builder{
- MonitoringData: pylds,
- MonitoringInfos: mons,
- ConsumingReceivedData: &consumingReceivedData,
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_ProcessBundleProgress{
+ ProcessBundleProgress: &fnpb.ProcessBundleProgressResponse{
+ MonitoringData: pylds,
+ MonitoringInfos: mons,
+ ConsumingReceivedData: &consumingReceivedData,
+ },
+ },
+ }
case req.GetProcessBundleSplit() != nil:
msg := req.GetProcessBundleSplit()
@@ -562,10 +571,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
return resp
}
if plan == nil {
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_ProcessBundleSplit{
+ ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
+ },
+ }
}
// Get the desired splits for the root FnAPI read operation.
@@ -586,10 +597,12 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
// Unsuccessful splits without errors indicate we should return an empty response,
// as processing can continue.
if sr.Unsuccessful {
- return fnpb.InstructionResponse_builder{
- InstructionId: string(instID),
- ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
- }.Build()
+ return &fnpb.InstructionResponse{
+ InstructionId: string(instID),
+ Response: &fnpb.InstructionResponse_ProcessBundleSplit{
+ ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{},
+ },
+ }
}
var pRoots []*fnpb.BundleApplication
@@ -597,53 +610,59 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
if len(sr.PS) > 0 && len(sr.RS) > 0 {
pRoots = make([]*fnpb.BundleApplication, len(sr.PS))
for i, p := range sr.PS {
- pRoots[i] = fnpb.BundleApplication_builder{
+ pRoots[i] = &fnpb.BundleApplication{
TransformId: sr.TId,
InputId: sr.InId,
Element: p,
- }.Build()
+ }
}
rRoots = make([]*fnpb.DelayedBundleApplication, len(sr.RS))
for i, r := range sr.RS {
- rRoots[i] = fnpb.DelayedBundleApplication_builder{
- Application: fnpb.BundleApplication_builder{
+ rRoots[i] = &fnpb.DelayedBundleApplication{
+ Application: &fnpb.BundleApplication{
TransformId: sr.TId,
InputId: sr.InId,
Element: r,
OutputWatermarks: sr.OW,
- }.Build(),
- }.Build()
+ },
+ }
}
}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- ProcessBundleSplit: fnpb.ProcessBundleSplitResponse_builder{
- ChannelSplits: []*fnpb.ProcessBundleSplitResponse_ChannelSplit{fnpb.ProcessBundleSplitResponse_ChannelSplit_builder{
- TransformId: plan.SourcePTransformID(),
- LastPrimaryElement: sr.PI,
- FirstResidualElement: sr.RI,
- }.Build()},
- PrimaryRoots: pRoots,
- ResidualRoots: rRoots,
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_ProcessBundleSplit{
+ ProcessBundleSplit: &fnpb.ProcessBundleSplitResponse{
+ ChannelSplits: []*fnpb.ProcessBundleSplitResponse_ChannelSplit{{
+ TransformId: plan.SourcePTransformID(),
+ LastPrimaryElement: sr.PI,
+ FirstResidualElement: sr.RI,
+ }},
+ PrimaryRoots: pRoots,
+ ResidualRoots: rRoots,
+ },
+ },
+ }
case req.GetMonitoringInfos() != nil:
msg := req.GetMonitoringInfos()
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- MonitoringInfos: fnpb.MonitoringInfosMetadataResponse_builder{
- MonitoringInfo: shortIdsToInfos(msg.GetMonitoringInfoId()),
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_MonitoringInfos{
+ MonitoringInfos: &fnpb.MonitoringInfosMetadataResponse{
+ MonitoringInfo: shortIdsToInfos(msg.GetMonitoringInfoId()),
+ },
+ },
+ }
case req.GetHarnessMonitoringInfos() != nil:
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- HarnessMonitoringInfos: fnpb.HarnessMonitoringInfosResponse_builder{
- // TODO(BEAM-11092): Populate with non-bundle metrics data.
- MonitoringData: map[string][]byte{},
- }.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_HarnessMonitoringInfos{
+ HarnessMonitoringInfos: &fnpb.HarnessMonitoringInfosResponse{
+ // TODO(BEAM-11092): Populate with non-bundle metrics data.
+ MonitoringData: map[string][]byte{},
+ },
+ },
+ }
case req.GetSampleData() != nil:
msg := req.GetSampleData()
var samples = make(map[string]*fnpb.SampleDataResponse_ElementList)
@@ -652,20 +671,22 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
for pid, elements := range elementsMap {
var elementList fnpb.SampleDataResponse_ElementList
for i := range elements {
- var sampledElement = fnpb.SampledElement_builder{
+ var sampledElement = &fnpb.SampledElement{
Element: elements[i].Element,
SampleTimestamp: timestamppb.New(elements[i].Timestamp),
- }.Build()
- elementList.SetElements(append(elementList.GetElements(), sampledElement))
+ }
+ elementList.Elements = append(elementList.Elements, sampledElement)
}
samples[pid] = &elementList
}
}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(instID),
- SampleData: fnpb.SampleDataResponse_builder{ElementSamples: samples}.Build(),
- }.Build()
+ Response: &fnpb.InstructionResponse_SampleData{
+ SampleData: &fnpb.SampleDataResponse{ElementSamples: samples},
+ },
+ }
default:
return fail(ctx, instID, "Unexpected request: %v", req)
}
@@ -709,11 +730,11 @@ func fail(ctx context.Context, id instructionID, format string, args ...any) *fn
log.Output(ctx, log.SevError, 1, fmt.Sprintf(format, args...))
dummy := &fnpb.InstructionResponse_Register{Register: &fnpb.RegisterResponse{}}
- return fnpb.InstructionResponse_builder{
+ return &fnpb.InstructionResponse{
InstructionId: string(id),
Error: fmt.Sprintf(format, args...),
- Register: proto.ValueOrDefault(dummy.Register),
- }.Build()
+ Response: dummy,
+ }
}
// dial to the specified endpoint. if timeout <=0, call blocks until
diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness_test.go b/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
index e52ca9b1c11..8c25db613eb 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
@@ -29,62 +29,62 @@ import (
// validDescriptor describes a valid pipeline with a source and a sink, but doesn't do anything else.
func validDescriptor(t *testing.T) *fnpb.ProcessBundleDescriptor {
t.Helper()
- port := fnpb.RemoteGrpcPort_builder{
+ port := &fnpb.RemoteGrpcPort{
CoderId: "c1",
- ApiServiceDescriptor: pipepb.ApiServiceDescriptor_builder{
+ ApiServiceDescriptor: &pipepb.ApiServiceDescriptor{
Url: "hostname:port",
- }.Build(),
- }.Build()
+ },
+ }
portBytes, err := proto.Marshal(port)
if err != nil {
t.Fatalf("bad port: %v", err)
}
- return fnpb.ProcessBundleDescriptor_builder{
+ return &fnpb.ProcessBundleDescriptor{
Id: "test",
Transforms: map[string]*pipepb.PTransform{
- "source": pipepb.PTransform_builder{
- Spec: pipepb.FunctionSpec_builder{
+ "source": &pipepb.PTransform{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:runner:source:v1",
Payload: portBytes,
- }.Build(),
+ },
Outputs: map[string]string{
"o1": "p1",
},
- }.Build(),
- "sink": pipepb.PTransform_builder{
- Spec: pipepb.FunctionSpec_builder{
+ },
+ "sink": &pipepb.PTransform{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:runner:sink:v1",
Payload: portBytes,
- }.Build(),
+ },
Inputs: map[string]string{
"i1": "p1",
},
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p1": pipepb.PCollection_builder{
+ "p1": &pipepb.PCollection{
CoderId: "c1",
- }.Build(),
+ },
},
Coders: map[string]*pipepb.Coder{
- "c1": pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ "c1": &pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:coder:windowed_value:v1",
- }.Build(),
+ },
ComponentCoderIds: []string{"c2", "c3"},
- }.Build(),
- "c2": pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ },
+ "c2": &pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:coder:varint:v1",
- }.Build(),
- }.Build(),
- "c3": pipepb.Coder_builder{
- Spec: pipepb.FunctionSpec_builder{
+ },
+ },
+ "c3": &pipepb.Coder{
+ Spec: &pipepb.FunctionSpec{
Urn: "beam:coder:global_window:v1",
- }.Build(),
- }.Build(),
+ },
+ },
},
- }.Build()
+ }
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/logging.go b/sdks/go/pkg/beam/core/runtime/harness/logging.go
index f5d6d18b622..20fe7660828 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/logging.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/logging.go
@@ -38,16 +38,16 @@ type logger struct {
func (l *logger) Log(ctx context.Context, sev log.Severity, calldepth int, msg string) {
now := timestamppb.New(time.Now())
- entry := fnpb.LogEntry_builder{
+ entry := &fnpb.LogEntry{
Timestamp: now,
Severity: convertSeverity(sev),
Message: msg,
- }.Build()
+ }
if _, file, line, ok := runtime.Caller(calldepth + 1); ok {
- entry.SetLogLocation(fmt.Sprintf("%v:%v", file, line))
+ entry.LogLocation = fmt.Sprintf("%v:%v", file, line)
}
- entry.SetInstructionId(metrics.GetBundleID(ctx))
- entry.SetTransformId(metrics.GetTransformID(ctx))
+ entry.InstructionId = metrics.GetBundleID(ctx)
+ entry.TransformId = metrics.GetTransformID(ctx)
select {
case l.out <- entry:
@@ -174,7 +174,7 @@ func (w *remoteWriter) connect(ctx context.Context, makeClient func(ctx context.
return errBuffClosed
}
msgs = append(msgs, newMsg)
- flush = newMsg.GetSeverity() >= fnpb.LogEntry_Severity_CRITICAL
+ flush = newMsg.Severity >= fnpb.LogEntry_Severity_CRITICAL
}
// If there are still messages in the buffer, drain them out to batch them to a cap.
@@ -185,12 +185,12 @@ func (w *remoteWriter) connect(ctx context.Context, makeClient func(ctx context.
return errBuffClosed
}
msgs = append(msgs, newMsg)
- flush = newMsg.GetSeverity() >= fnpb.LogEntry_Severity_CRITICAL
+ flush = newMsg.Severity >= fnpb.LogEntry_Severity_CRITICAL
}
- list := fnpb.LogEntry_List_builder{
+ list := &fnpb.LogEntry_List{
LogEntries: msgs,
- }.Build()
+ }
if err := client.Send(list); err != nil {
if err == io.EOF {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/logging_test.go b/sdks/go/pkg/beam/core/runtime/harness/logging_test.go
index 0087bb6cbf0..b2383111a4d 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/logging_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/logging_test.go
@@ -103,7 +103,7 @@ func TestLogger_connect(t *testing.T) {
var count int
for _, batch := range cacher.logs {
- count += len(batch.GetLogEntries())
+ count += len(batch.LogEntries)
}
if got, want := count, 10; got != want {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/monitoring.go b/sdks/go/pkg/beam/core/runtime/harness/monitoring.go
index ab5c091c3db..08b87514732 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/monitoring.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/monitoring.go
@@ -69,11 +69,11 @@ func (c *shortIDCache) getShortID(l metrics.Labels, urn metricsx.Urn) string {
}
s = c.getNextShortID()
c.labels2ShortIds[k] = s
- c.shortIds2Infos[s] = pipepb.MonitoringInfo_builder{
+ c.shortIds2Infos[s] = &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(urn),
Type: metricsx.UrnToType(urn),
Labels: l.Map(),
- }.Build()
+ }
return s
}
@@ -120,12 +120,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(l, metricsx.UrnUserSumInt64)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnUserSumInt64),
Type: metricsx.UrnToType(metricsx.UrnUserSumInt64),
Labels: l.Map(),
Payload: payload,
- }.Build())
+ })
}
},
DistributionInt64: func(l metrics.Labels, count, sum, min, max int64) {
@@ -136,12 +136,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(l, metricsx.UrnUserDistInt64)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnUserDistInt64),
Type: metricsx.UrnToType(metricsx.UrnUserDistInt64),
Labels: l.Map(),
Payload: payload,
- }.Build())
+ })
}
},
GaugeInt64: func(l metrics.Labels, v int64, t time.Time) {
@@ -152,12 +152,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(l, metricsx.UrnUserLatestMsInt64)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnUserLatestMsInt64),
Type: metricsx.UrnToType(metricsx.UrnUserLatestMsInt64),
Labels: l.Map(),
Payload: payload,
- }.Build())
+ })
}
},
MsecsInt64: func(l string, states *[4]metrics.ExecutionState) {
@@ -171,12 +171,12 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(metrics.PTransformLabels(l), ul)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(ul),
Type: metricsx.UrnToType(ul),
Labels: label,
Payload: payload,
- }.Build())
+ })
}
}
},
@@ -199,14 +199,14 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnElementCount),
Type: metricsx.UrnToType(metricsx.UrnElementCount),
Labels: map[string]string{
"PCOLLECTION": pcol.ID,
},
Payload: payload,
- }.Build())
+ })
}
// Skip pcollections without size
if pcol.SizeCount != 0 {
@@ -218,14 +218,14 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnSampledByteSize),
Type: metricsx.UrnToType(metricsx.UrnSampledByteSize),
Labels: map[string]string{
"PCOLLECTION": pcol.ID,
},
Payload: payload,
- }.Build())
+ })
}
}
}
@@ -238,14 +238,14 @@ func monitoring(p *exec.Plan, store *metrics.Store, supportShortID bool) ([]*pip
payloads[getShortID(metrics.PTransformLabels(snapshot.Source.ID), metricsx.UrnDataChannelReadIndex)] = payload
if !supportShortID {
monitoringInfo = append(monitoringInfo,
- pipepb.MonitoringInfo_builder{
+ &pipepb.MonitoringInfo{
Urn: metricsx.UrnToString(metricsx.UrnDataChannelReadIndex),
Type: metricsx.UrnToType(metricsx.UrnDataChannelReadIndex),
Labels: map[string]string{
"PTRANSFORM": snapshot.Source.ID,
},
Payload: payload,
- }.Build())
+ })
}
return monitoringInfo, payloads, snapshot.Source.ConsumingReceivedData
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go b/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go
index 7535ad4d5c3..23f6fbf2461 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/statecache/statecache_test.go
@@ -122,13 +122,15 @@ func TestSetCache_CacheableCase(t *testing.T) {
}
func makeRequest(transformID, sideInputID string, t token) *fnpb.ProcessBundleRequest_CacheToken {
- return fnpb.ProcessBundleRequest_CacheToken_builder{
+ return &fnpb.ProcessBundleRequest_CacheToken{
Token: []byte(t),
- SideInput: fnpb.ProcessBundleRequest_CacheToken_SideInput_builder{
- TransformId: transformID,
- SideInputId: sideInputID,
- }.Build(),
- }.Build()
+ Type: &fnpb.ProcessBundleRequest_CacheToken_SideInput_{
+ SideInput: &fnpb.ProcessBundleRequest_CacheToken_SideInput{
+ TransformId: transformID,
+ SideInputId: sideInputID,
+ },
+ },
+ }
}
func makeCacheToken(transformID, sideInputID string, tok token) cacheToken {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/statemgr.go b/sdks/go/pkg/beam/core/runtime/harness/statemgr.go
index e082d9806de..4c1bc0b55fe 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/statemgr.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/statemgr.go
@@ -228,14 +228,16 @@ type stateKeyWriter struct {
}
func newSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID string, instID instructionID, k, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- MultimapSideInput: fnpb.StateKey_MultimapSideInput_builder{
- TransformId: id.PtransformID,
- SideInputId: sideInputID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapSideInput_{
+ MultimapSideInput: &fnpb.StateKey_MultimapSideInput{
+ TransformId: id.PtransformID,
+ SideInputId: sideInputID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -244,13 +246,15 @@ func newSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID string,
}
func newIterableSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID string, instID instructionID, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- IterableSideInput: fnpb.StateKey_IterableSideInput_builder{
- TransformId: id.PtransformID,
- SideInputId: sideInputID,
- Window: w,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_IterableSideInput_{
+ IterableSideInput: &fnpb.StateKey_IterableSideInput{
+ TransformId: id.PtransformID,
+ SideInputId: sideInputID,
+ Window: w,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
@@ -260,11 +264,13 @@ func newIterableSideInputReader(ch *StateChannel, id exec.StreamID, sideInputID
}
func newRunnerReader(ch *StateChannel, instID instructionID, k []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- Runner: fnpb.StateKey_Runner_builder{
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_Runner_{
+ Runner: &fnpb.StateKey_Runner{
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -273,14 +279,16 @@ func newRunnerReader(ch *StateChannel, instID instructionID, k []byte) *stateKey
}
func newBagUserStateReader(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- BagUserState: fnpb.StateKey_BagUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_BagUserState_{
+ BagUserState: &fnpb.StateKey_BagUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -289,14 +297,16 @@ func newBagUserStateReader(ch *StateChannel, id exec.StreamID, instID instructio
}
func newBagUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, wt writeTypeEnum) *stateKeyWriter {
- key := fnpb.StateKey_builder{
- BagUserState: fnpb.StateKey_BagUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_BagUserState_{
+ BagUserState: &fnpb.StateKey_BagUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyWriter{
instID: instID,
key: key,
@@ -306,15 +316,17 @@ func newBagUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructio
}
func newMultimapUserStateReader(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, mk []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- MultimapUserState: fnpb.StateKey_MultimapUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- MapKey: mk,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapUserState_{
+ MultimapUserState: &fnpb.StateKey_MultimapUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ MapKey: mk,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -323,15 +335,17 @@ func newMultimapUserStateReader(ch *StateChannel, id exec.StreamID, instID instr
}
func newMultimapUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, mk []byte, wt writeTypeEnum) *stateKeyWriter {
- key := fnpb.StateKey_builder{
- MultimapUserState: fnpb.StateKey_MultimapUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- MapKey: mk,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapUserState_{
+ MultimapUserState: &fnpb.StateKey_MultimapUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ MapKey: mk,
+ },
+ },
+ }
return &stateKeyWriter{
instID: instID,
key: key,
@@ -341,14 +355,16 @@ func newMultimapUserStateWriter(ch *StateChannel, id exec.StreamID, instID instr
}
func newMultimapKeysUserStateReader(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte) *stateKeyReader {
- key := fnpb.StateKey_builder{
- MultimapKeysUserState: fnpb.StateKey_MultimapKeysUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapKeysUserState_{
+ MultimapKeysUserState: &fnpb.StateKey_MultimapKeysUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyReader{
instID: instID,
key: key,
@@ -357,14 +373,16 @@ func newMultimapKeysUserStateReader(ch *StateChannel, id exec.StreamID, instID i
}
func newMultimapKeysUserStateWriter(ch *StateChannel, id exec.StreamID, instID instructionID, userStateID string, k []byte, w []byte, wt writeTypeEnum) *stateKeyWriter {
- key := fnpb.StateKey_builder{
- MultimapKeysUserState: fnpb.StateKey_MultimapKeysUserState_builder{
- TransformId: id.PtransformID,
- UserStateId: userStateID,
- Window: w,
- Key: k,
- }.Build(),
- }.Build()
+ key := &fnpb.StateKey{
+ Type: &fnpb.StateKey_MultimapKeysUserState_{
+ MultimapKeysUserState: &fnpb.StateKey_MultimapKeysUserState{
+ TransformId: id.PtransformID,
+ UserStateId: userStateID,
+ Window: w,
+ Key: k,
+ },
+ },
+ }
return &stateKeyWriter{
instID: instID,
key: key,
@@ -389,14 +407,16 @@ func (r *stateKeyReader) Read(buf []byte) (int, error) {
localChannel := r.ch
r.mu.Unlock()
- req := fnpb.StateRequest_builder{
+ req := &fnpb.StateRequest{
// Id: set by StateChannel
InstructionId: string(r.instID),
StateKey: r.key,
- Get: fnpb.StateGetRequest_builder{
- ContinuationToken: r.token,
- }.Build(),
- }.Build()
+ Request: &fnpb.StateRequest_Get{
+ Get: &fnpb.StateGetRequest{
+ ContinuationToken: r.token,
+ },
+ },
+ }
resp, err := localChannel.Send(req)
if err != nil {
r.Close()
@@ -457,21 +477,25 @@ func (r *stateKeyWriter) Write(buf []byte) (int, error) {
var req *fnpb.StateRequest
switch r.writeType {
case writeTypeAppend:
- req = fnpb.StateRequest_builder{
+ req = &fnpb.StateRequest{
// Id: set by StateChannel
InstructionId: string(r.instID),
StateKey: r.key,
- Append: fnpb.StateAppendRequest_builder{
- Data: toSend,
- }.Build(),
- }.Build()
+ Request: &fnpb.StateRequest_Append{
+ Append: &fnpb.StateAppendRequest{
+ Data: toSend,
+ },
+ },
+ }
case writeTypeClear:
- req = fnpb.StateRequest_builder{
+ req = &fnpb.StateRequest{
// ID: set by StateChannel
InstructionId: string(r.instID),
StateKey: r.key,
- Clear: &fnpb.StateClearRequest{},
- }.Build()
+ Request: &fnpb.StateRequest_Clear{
+ Clear: &fnpb.StateClearRequest{},
+ },
+ }
default:
return 0, errors.Errorf("Unknown write type %v", r.writeType)
}
@@ -607,8 +631,8 @@ func (c *StateChannel) read(ctx context.Context) {
}
c.mu.Lock()
- ch, ok := c.responses[msg.GetId()]
- delete(c.responses, msg.GetId())
+ ch, ok := c.responses[msg.Id]
+ delete(c.responses, msg.Id)
c.mu.Unlock()
if !ok {
// This can happen if Send returns an error that write handles, but
@@ -638,7 +662,7 @@ func (c *StateChannel) write(ctx context.Context) {
}
err = c.client.Send(req)
if err != nil {
- id = req.GetId()
+ id = req.Id
break // non-nil errors mean the stream is broken and can't be re-used.
}
}
@@ -664,14 +688,14 @@ func (c *StateChannel) write(ctx context.Context) {
c.terminateStreamOnError(err)
if ok {
- ch <- fnpb.StateResponse_builder{Id: id, Error: fmt.Sprintf("StateChannel[%v].write failed to send: %v", c.id, err)}.Build()
+ ch <- &fnpb.StateResponse{Id: id, Error: fmt.Sprintf("StateChannel[%v].write failed to send: %v", c.id, err)}
}
}
// Send sends a state request and returns the response.
func (c *StateChannel) Send(req *fnpb.StateRequest) (*fnpb.StateResponse, error) {
id := fmt.Sprintf("r%v", atomic.AddInt32(&c.nextRequestNo, 1))
- req.SetId(id)
+ req.Id = id
ch := make(chan *fnpb.StateResponse, 1)
c.mu.Lock()
@@ -692,8 +716,8 @@ func (c *StateChannel) Send(req *fnpb.StateRequest) (*fnpb.StateResponse, error)
defer c.mu.Unlock()
return nil, errors.Wrapf(c.closedErr, "StateChannel[%v].Send(%v): context canceled", c.id, id)
}
- if resp.GetError() != "" {
- return nil, errors.New(resp.GetError())
+ if resp.Error != "" {
+ return nil, errors.New(resp.Error)
}
return resp, nil
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go b/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go
index 81b1f2ca11e..53c4ca05ee1 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/statemgr_test.go
@@ -93,21 +93,25 @@ func TestStateChannel(t *testing.T) {
for i := 0; i < count; i++ {
go func() {
req := <-client.send
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(), // Ids need to match up to ensure routing can occur properly.
- Get: fnpb.StateGetResponse_builder{
- ContinuationToken: req.GetGet().GetContinuationToken(),
- }.Build(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id, // Ids need to match up to ensure routing can occur properly.
+ Response: &fnpb.StateResponse_Get{
+ Get: &fnpb.StateGetResponse{
+ ContinuationToken: req.GetGet().GetContinuationToken(),
+ },
+ },
+ }
}()
}
for i := 0; i < count; i++ {
token := []byte(fmt.Sprintf("%d", i))
- resp, err := c.Send(fnpb.StateRequest_builder{
- Get: fnpb.StateGetRequest_builder{
- ContinuationToken: token,
- }.Build(),
- }.Build())
+ resp, err := c.Send(&fnpb.StateRequest{
+ Request: &fnpb.StateRequest_Get{
+ Get: &fnpb.StateGetRequest{
+ ContinuationToken: token,
+ },
+ },
+ })
if err != nil {
t.Fatalf("unexpected error from Send: %v", err)
}
@@ -124,9 +128,9 @@ func TestStateChannel(t *testing.T) {
req := <-client.send // Send should succeed.
client.setRecvErr(io.EOF)
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
}()
_, err := c.Send(&fnpb.StateRequest{})
return err
@@ -140,9 +144,9 @@ func TestStateChannel(t *testing.T) {
req := <-client.send // Send should succeed.
client.setRecvErr(expectedError)
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
}()
_, err := c.Send(&fnpb.StateRequest{})
return err
@@ -156,13 +160,13 @@ func TestStateChannel(t *testing.T) {
req := <-client.send // Send should succeed.
c.mu.Lock()
- ch := c.responses[req.GetId()]
- delete(c.responses, req.GetId())
+ ch := c.responses[req.Id]
+ delete(c.responses, req.Id)
c.mu.Unlock()
- resp := fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ resp := &fnpb.StateResponse{
+ Id: req.Id,
+ }
client.recv <- resp
// unblock Send.
ch <- resp
@@ -179,9 +183,9 @@ func TestStateChannel(t *testing.T) {
// This can be plumbed through on either side, write or read,
// the important part is that we get it.
client.setRecvErr(expectedError)
- client.recv <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ client.recv <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
}()
_, err := c.Send(&fnpb.StateRequest{})
return err
@@ -231,7 +235,7 @@ func TestStateChannel(t *testing.T) {
client.setRecvErr(nil)
// Drain the next send, and ensure the response is unblocked.
req := <-client.send
- client.recv <- fnpb.StateResponse_builder{Id: req.GetId()}.Build() // Ids need to match up to ensure routing can occur properly.
+ client.recv <- &fnpb.StateResponse{Id: req.Id} // Ids need to match up to ensure routing can occur properly.
}()
if _, err := c.Send(&fnpb.StateRequest{}); !contains(err, test.expectedErr) {
t.Errorf("Unexpected error from Send: got %v, want %v", err, test.expectedErr)
@@ -333,9 +337,9 @@ func TestStateKeyReader(t *testing.T) {
go func() {
if test.noGet {
req := <-ch.requests
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ }
return
}
for i, buflen := range test.buflens {
@@ -350,13 +354,15 @@ func TestStateKeyReader(t *testing.T) {
}
req := <-ch.requests
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Get: fnpb.StateGetResponse_builder{
- ContinuationToken: token,
- Data: buf,
- }.Build(),
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Get{
+ Get: &fnpb.StateGetResponse{
+ ContinuationToken: token,
+ Data: buf,
+ },
+ },
+ }
}
}()
@@ -457,30 +463,30 @@ func TestStateKeyWriter(t *testing.T) {
if !ok {
t.Errorf("Append write: got %v, want data of type StateRequest_Append", req.Request)
} else {
- data := sra.Append.GetData()
+ data := sra.Append.Data
if !bytes.Equal(data, test.data) {
t.Errorf("Expected request data: got %v, want %v", data, test.data)
}
}
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Append: &fnpb.StateAppendResponse{},
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Append{},
+ }
case writeTypeClear:
_, ok := req.Request.(*fnpb.StateRequest_Clear)
if !ok {
t.Errorf("Append write: got %v, want data of type StateRequest_Append", req.Request)
}
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Clear: &fnpb.StateClearResponse{},
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Clear{},
+ }
default:
// Still return response so that write doesn't hang.
- ch.responses[req.GetId()] <- fnpb.StateResponse_builder{
- Id: req.GetId(),
- Append: &fnpb.StateAppendResponse{},
- }.Build()
+ ch.responses[req.Id] <- &fnpb.StateResponse{
+ Id: req.Id,
+ Response: &fnpb.StateResponse_Append{},
+ }
}
}()
diff --git a/sdks/go/pkg/beam/core/runtime/harness/worker_status.go b/sdks/go/pkg/beam/core/runtime/harness/worker_status.go
index 9b53f3cf065..0ec7e1f805c 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/worker_status.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/worker_status.go
@@ -129,7 +129,7 @@ func (w *workerStatusHandler) reader(ctx context.Context, stub fnpb.BeamFnWorker
goroutineDump(statusInfo)
buildInfo(statusInfo)
- response := fnpb.WorkerStatusResponse_builder{Id: req.GetId(), StatusInfo: statusInfo.String()}.Build()
+ response := &fnpb.WorkerStatusResponse{Id: req.GetId(), StatusInfo: statusInfo.String()}
if err := stub.Send(response); err != nil && err != io.EOF {
log.Errorf(ctx, "workerStatus.Writer: Failed to respond: %v", err)
}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go b/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go
index dffe0e26fb2..9cc2e533079 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/worker_status_test.go
@@ -37,7 +37,7 @@ type BeamFnWorkerStatusServicer struct {
}
func (w *BeamFnWorkerStatusServicer) WorkerStatus(b fnpb.BeamFnWorkerStatus_WorkerStatusServer) error {
- b.Send(fnpb.WorkerStatusRequest_builder{Id: "1"}.Build())
+ b.Send(&fnpb.WorkerStatusRequest{Id: "1"})
resp, err := b.Recv()
if err != nil {
return fmt.Errorf("error receiving response b.recv: %v", err)
diff --git a/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go b/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
index 06d58290346..95b92f929c0 100644
--- a/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
+++ b/sdks/go/pkg/beam/core/runtime/metricsx/metricsx_test.go
@@ -46,12 +46,12 @@ func TestFromMonitoringInfos_Counters(t *testing.T) {
"NAME": "customCounter",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnUserSumInt64),
Type: UrnToType(UrnUserSumInt64),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -111,30 +111,30 @@ func TestFromMonitoringInfos_Msec(t *testing.T) {
if err != nil {
t.Fatalf("Failed to encode Int64Counter: %v", err)
}
- mStartBundleInfo := pipepb.MonitoringInfo_builder{
+ mStartBundleInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(0)),
Type: UrnToType(ExecutionMsecUrn(0)),
Labels: labels,
Payload: startValue,
- }.Build()
- mProcessBundleInfo := pipepb.MonitoringInfo_builder{
+ }
+ mProcessBundleInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(1)),
Type: UrnToType(ExecutionMsecUrn(1)),
Labels: labels,
Payload: processValue,
- }.Build()
- mFinishBundleInfo := pipepb.MonitoringInfo_builder{
+ }
+ mFinishBundleInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(2)),
Type: UrnToType(ExecutionMsecUrn(2)),
Labels: labels,
Payload: finishValue,
- }.Build()
- mTotalTimeInfo := pipepb.MonitoringInfo_builder{
+ }
+ mTotalTimeInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(ExecutionMsecUrn(3)),
Type: UrnToType(ExecutionMsecUrn(3)),
Labels: labels,
Payload: totalValue,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mStartBundleInfo, mProcessBundleInfo, mFinishBundleInfo, mTotalTimeInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -174,12 +174,12 @@ func TestFromMonitoringInfos_PColCounters(t *testing.T) {
"NAME": "customCounter",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnElementCount),
Type: UrnToType(UrnElementCount),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -224,12 +224,12 @@ func TestFromMonitoringInfos_SampledByteSize(t *testing.T) {
"NAME": "customCounter",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnSampledByteSize),
Type: UrnToType(UrnSampledByteSize),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -274,12 +274,12 @@ func TestFromMonitoringInfos_Distributions(t *testing.T) {
"NAME": "customDist",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnUserDistInt64),
Type: UrnToType(UrnUserDistInt64),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
@@ -324,12 +324,12 @@ func TestFromMonitoringInfos_Gauges(t *testing.T) {
"NAME": "customGauge",
}
- mInfo := pipepb.MonitoringInfo_builder{
+ mInfo := &pipepb.MonitoringInfo{
Urn: UrnToString(UrnUserLatestMsInt64),
Type: UrnToType(UrnUserLatestMsInt64),
Labels: labels,
Payload: payload,
- }.Build()
+ }
attempted := []*pipepb.MonitoringInfo{mInfo}
committed := []*pipepb.MonitoringInfo{}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go b/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go
index 443853b0769..7745cec942e 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/clone.go
@@ -21,10 +21,10 @@ import (
)
func shallowClonePipeline(p *pipepb.Pipeline) *pipepb.Pipeline {
- ret := pipepb.Pipeline_builder{
+ ret := &pipepb.Pipeline{
Components: shallowCloneComponents(p.GetComponents()),
Requirements: reflectx.ShallowClone(p.GetRequirements()).([]string),
- }.Build()
+ }
ret.RootTransformIds, _ = reflectx.ShallowClone(p.GetRootTransformIds()).([]string)
return ret
}
@@ -45,15 +45,15 @@ func ShallowClonePTransform(t *pipepb.PTransform) *pipepb.PTransform {
return nil
}
- ret := pipepb.PTransform_builder{
- UniqueName: t.GetUniqueName(),
- Spec: t.GetSpec(),
- DisplayData: t.GetDisplayData(),
- Annotations: t.GetAnnotations(),
- }.Build()
- ret.Subtransforms, _ = reflectx.ShallowClone(t.GetSubtransforms()).([]string)
- ret.Inputs, _ = reflectx.ShallowClone(t.GetInputs()).(map[string]string)
- ret.Outputs, _ = reflectx.ShallowClone(t.GetOutputs()).(map[string]string)
- ret.SetEnvironmentId(t.GetEnvironmentId())
+ ret := &pipepb.PTransform{
+ UniqueName: t.UniqueName,
+ Spec: t.Spec,
+ DisplayData: t.DisplayData,
+ Annotations: t.Annotations,
+ }
+ ret.Subtransforms, _ = reflectx.ShallowClone(t.Subtransforms).([]string)
+ ret.Inputs, _ = reflectx.ShallowClone(t.Inputs).(map[string]string)
+ ret.Outputs, _ = reflectx.ShallowClone(t.Outputs).(map[string]string)
+ ret.EnvironmentId = t.EnvironmentId
return ret
}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go b/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go
index cc674b96979..b58a3098379 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/clone_test.go
@@ -26,11 +26,11 @@ import (
func TestShallowClonePTransform(t *testing.T) {
tests := []*pipepb.PTransform{
{},
- pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- pipepb.PTransform_builder{Spec: pipepb.FunctionSpec_builder{Urn: "foo"}.Build()}.Build(),
- pipepb.PTransform_builder{Subtransforms: []string{"a", "b"}}.Build(),
- pipepb.PTransform_builder{Inputs: map[string]string{"a": "b"}}.Build(),
- pipepb.PTransform_builder{Outputs: map[string]string{"a": "b"}}.Build(),
+ {UniqueName: "a"},
+ {Spec: &pipepb.FunctionSpec{Urn: "foo"}},
+ {Subtransforms: []string{"a", "b"}},
+ {Inputs: map[string]string{"a": "b"}},
+ {Outputs: map[string]string{"a": "b"}},
}
for _, test := range tests {
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go b/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go
index 89e671f37ad..9e527f2fd32 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/replace.go
@@ -36,11 +36,11 @@ import (
// modified.
func Update(p *pipepb.Pipeline, values *pipepb.Components) (*pipepb.Pipeline, error) {
ret := shallowClonePipeline(p)
- reflectx.UpdateMap(ret.GetComponents().GetTransforms(), values.GetTransforms())
- reflectx.UpdateMap(ret.GetComponents().GetPcollections(), values.GetPcollections())
- reflectx.UpdateMap(ret.GetComponents().GetWindowingStrategies(), values.GetWindowingStrategies())
- reflectx.UpdateMap(ret.GetComponents().GetCoders(), values.GetCoders())
- reflectx.UpdateMap(ret.GetComponents().GetEnvironments(), values.GetEnvironments())
+ reflectx.UpdateMap(ret.Components.Transforms, values.Transforms)
+ reflectx.UpdateMap(ret.Components.Pcollections, values.Pcollections)
+ reflectx.UpdateMap(ret.Components.WindowingStrategies, values.WindowingStrategies)
+ reflectx.UpdateMap(ret.Components.Coders, values.Coders)
+ reflectx.UpdateMap(ret.Components.Environments, values.Environments)
return Normalize(ret)
}
@@ -60,12 +60,12 @@ func Normalize(p *pipepb.Pipeline) (*pipepb.Pipeline, error) {
ret := shallowClonePipeline(p)
if IdempotentNormalize {
- ret.GetComponents().SetTransforms(ensureUniqueNames(ret.GetComponents().GetTransforms()))
+ ret.Components.Transforms = ensureUniqueNames(ret.Components.Transforms)
} else {
- ret.GetComponents().SetTransforms(ensureUniqueNamesLegacy(ret.GetComponents().GetTransforms()))
+ ret.Components.Transforms = ensureUniqueNamesLegacy(ret.Components.Transforms)
}
- ret.GetComponents().SetTransforms(computeCompositeInputOutput(ret.GetComponents().GetTransforms()))
- ret.SetRootTransformIds(computeRoots(ret.GetComponents().GetTransforms()))
+ ret.Components.Transforms = computeCompositeInputOutput(ret.Components.Transforms)
+ ret.RootTransformIds = computeRoots(ret.Components.Transforms)
return ret, nil
}
@@ -85,7 +85,7 @@ func walkCoders(coders, accum map[string]*pipepb.Coder, id string) {
c := coders[id]
accum[id] = c
- for _, sub := range c.GetComponentCoderIds() {
+ for _, sub := range c.ComponentCoderIds {
walkCoders(coders, accum, sub)
}
}
@@ -106,7 +106,7 @@ func computeRoots(xforms map[string]*pipepb.PTransform) []string {
func makeParentMap(xforms map[string]*pipepb.PTransform) map[string]string {
parent := make(map[string]string)
for id, t := range xforms {
- for _, key := range t.GetSubtransforms() {
+ for _, key := range t.Subtransforms {
parent[key] = id
}
}
@@ -138,7 +138,7 @@ func computeCompositeInputOutput(xforms map[string]*pipepb.PTransform) map[strin
// maps of composite transforms. Update the transform map.
func walk(id string, ret map[string]*pipepb.PTransform, seen map[string]bool, primitiveXformsForInput map[string][]string) {
t := ret[id]
- if seen[id] || len(t.GetSubtransforms()) == 0 {
+ if seen[id] || len(t.Subtransforms) == 0 {
return
}
@@ -150,7 +150,7 @@ func walk(id string, ret map[string]*pipepb.PTransform, seen map[string]bool, pr
in := make(map[string]bool)
out := make(map[string]bool)
local := map[string]bool{id: true}
- for _, sid := range t.GetSubtransforms() {
+ for _, sid := range t.Subtransforms {
walk(sid, ret, seen, primitiveXformsForInput)
inout(ret[sid], in, out)
local[sid] = true
@@ -164,9 +164,9 @@ func walk(id string, ret map[string]*pipepb.PTransform, seen map[string]bool, pr
externalIns(local, primitiveXformsForInput, extIn, out)
upd := ShallowClonePTransform(t)
- upd.SetInputs(diff(in, out))
- upd.SetOutputs(diffAndMerge(out, in, extIn))
- upd.SetSubtransforms(TopologicalSort(ret, upd.GetSubtransforms()))
+ upd.Inputs = diff(in, out)
+ upd.Outputs = diffAndMerge(out, in, extIn)
+ upd.Subtransforms = TopologicalSort(ret, upd.Subtransforms)
ret[id] = upd
seen[id] = true
@@ -312,9 +312,9 @@ func ensureUniqueNames(xforms map[string]*pipepb.PTransform) map[string]*pipepb.
name := findFreeName(seen, base)
seen[name] = true
- if name != t.GetUniqueName() {
+ if name != t.UniqueName {
upd := ShallowClonePTransform(t)
- upd.SetUniqueName(name)
+ upd.UniqueName = name
ret[id] = upd
}
return name
@@ -373,12 +373,12 @@ func ensureUniqueNamesLegacy(xforms map[string]*pipepb.PTransform) map[string]*p
seen := make(map[string]bool)
for _, id := range ordering {
t := xforms[id]
- name := findFreeName(seen, t.GetUniqueName())
+ name := findFreeName(seen, t.UniqueName)
seen[name] = true
- if name != t.GetUniqueName() {
+ if name != t.UniqueName {
upd := ShallowClonePTransform(t)
- upd.SetUniqueName(name)
+ upd.UniqueName = name
ret[id] = upd
}
}
@@ -426,12 +426,12 @@ func ApplySdkImageOverrides(p *pipepb.Pipeline, patterns map[string]string) erro
for re, replacement := range regexes {
newImg := re.ReplaceAllLiteralString(oldImg, replacement)
if newImg != oldImg {
- payload.SetContainerImage(newImg)
+ payload.ContainerImage = newImg
pl, err := proto.Marshal(&payload)
if err != nil {
return err
}
- env.SetPayload(pl)
+ env.Payload = pl
break // Apply at most one override to each environment.
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go b/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go
index 02f4d660f5f..3024787e616 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/replace_test.go
@@ -33,130 +33,130 @@ func TestEnsureUniqueName(t *testing.T) {
{
name: "AlreadyUnique",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "b"},
+ "3": {UniqueName: "c"},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "b"},
+ "3": {UniqueName: "c"},
},
},
{
name: "NeedsUniqueLeaves",
in: map[string]*pipepb.PTransform{
- "2": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
+ "2": {UniqueName: "a"},
+ "1": {UniqueName: "a"},
+ "3": {UniqueName: "a"},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "a'1"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a'2"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "a'1"},
+ "3": {UniqueName: "a'2"},
},
},
{
name: "StripUniqueLeaves",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "a'1"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a'2"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "a'1"},
+ "3": {UniqueName: "a'2"},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "a'1"}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "a'2"}.Build(),
+ "1": {UniqueName: "a"},
+ "2": {UniqueName: "a'1"},
+ "3": {UniqueName: "a'2"},
},
},
{
name: "NonTopologicalIdOrder",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s1"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2"}}.Build(), // root
+ "e1": {UniqueName: "a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a", Subtransforms: []string{"s1"}},
+ "s3": {UniqueName: "a", Subtransforms: []string{"s2"}}, // root
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a/a/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a/a/a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a/a", Subtransforms: []string{"s1"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2"}}.Build(), // root
+ "e1": {UniqueName: "a/a/a/a"},
+ "s1": {UniqueName: "a/a/a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a/a", Subtransforms: []string{"s1"}},
+ "s3": {UniqueName: "a", Subtransforms: []string{"s2"}}, // root
},
},
{
name: "UniqueComps",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a"},
+ "e2": {UniqueName: "a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a", Subtransforms: []string{"e2"}},
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a'1/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a'1", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a/a"},
+ "e2": {UniqueName: "a'1/a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a'1", Subtransforms: []string{"e2"}},
},
},
{
name: "StripComps",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a'1/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a'1", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a/a"},
+ "e2": {UniqueName: "a'1/a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a'1", Subtransforms: []string{"e2"}},
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a'1/a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a'1", Subtransforms: []string{"e2"}}.Build(),
+ "e1": {UniqueName: "a/a"},
+ "e2": {UniqueName: "a'1/a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s2": {UniqueName: "a'1", Subtransforms: []string{"e2"}},
},
},
{
name: "large",
in: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e3": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e4": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e5": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e6": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e7": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e8": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e9": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e10": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e11": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "e12": pipepb.PTransform_builder{UniqueName: "a"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2", "s3"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s4", "s5"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s6", "s7"}}.Build(),
- "s4": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e1"}}.Build(),
- "s5": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e2", "e3"}}.Build(),
- "s6": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e4", "e5", "e6"}}.Build(),
- "s7": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}}.Build(),
+ "e1": {UniqueName: "a"},
+ "e2": {UniqueName: "a"},
+ "e3": {UniqueName: "a"},
+ "e4": {UniqueName: "a"},
+ "e5": {UniqueName: "a"},
+ "e6": {UniqueName: "a"},
+ "e7": {UniqueName: "a"},
+ "e8": {UniqueName: "a"},
+ "e9": {UniqueName: "a"},
+ "e10": {UniqueName: "a"},
+ "e11": {UniqueName: "a"},
+ "e12": {UniqueName: "a"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"s2", "s3"}},
+ "s2": {UniqueName: "a", Subtransforms: []string{"s4", "s5"}},
+ "s3": {UniqueName: "a", Subtransforms: []string{"s6", "s7"}},
+ "s4": {UniqueName: "a", Subtransforms: []string{"e1"}},
+ "s5": {UniqueName: "a", Subtransforms: []string{"e2", "e3"}},
+ "s6": {UniqueName: "a", Subtransforms: []string{"e4", "e5", "e6"}},
+ "s7": {UniqueName: "a", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}},
},
exp: map[string]*pipepb.PTransform{
- "e1": pipepb.PTransform_builder{UniqueName: "a/a/a/a"}.Build(),
- "e2": pipepb.PTransform_builder{UniqueName: "a/a/a'1/a"}.Build(),
- "e3": pipepb.PTransform_builder{UniqueName: "a/a/a'1/a'1"}.Build(),
- "e4": pipepb.PTransform_builder{UniqueName: "a/a'1/a/a"}.Build(),
- "e5": pipepb.PTransform_builder{UniqueName: "a/a'1/a/a'1"}.Build(),
- "e6": pipepb.PTransform_builder{UniqueName: "a/a'1/a/a'2"}.Build(),
- "e7": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a"}.Build(),
- "e8": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'1"}.Build(),
- "e9": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'2"}.Build(),
- "e10": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'3"}.Build(),
- "e11": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'4"}.Build(),
- "e12": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1/a'5"}.Build(),
- "s1": pipepb.PTransform_builder{UniqueName: "a", Subtransforms: []string{"s2", "s3"}}.Build(),
- "s2": pipepb.PTransform_builder{UniqueName: "a/a", Subtransforms: []string{"s4", "s5"}}.Build(),
- "s3": pipepb.PTransform_builder{UniqueName: "a/a'1", Subtransforms: []string{"s6", "s7"}}.Build(),
- "s4": pipepb.PTransform_builder{UniqueName: "a/a/a", Subtransforms: []string{"e1"}}.Build(),
- "s5": pipepb.PTransform_builder{UniqueName: "a/a/a'1", Subtransforms: []string{"e2", "e3"}}.Build(),
- "s6": pipepb.PTransform_builder{UniqueName: "a/a'1/a", Subtransforms: []string{"e4", "e5", "e6"}}.Build(),
- "s7": pipepb.PTransform_builder{UniqueName: "a/a'1/a'1", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}}.Build(),
+ "e1": {UniqueName: "a/a/a/a"},
+ "e2": {UniqueName: "a/a/a'1/a"},
+ "e3": {UniqueName: "a/a/a'1/a'1"},
+ "e4": {UniqueName: "a/a'1/a/a"},
+ "e5": {UniqueName: "a/a'1/a/a'1"},
+ "e6": {UniqueName: "a/a'1/a/a'2"},
+ "e7": {UniqueName: "a/a'1/a'1/a"},
+ "e8": {UniqueName: "a/a'1/a'1/a'1"},
+ "e9": {UniqueName: "a/a'1/a'1/a'2"},
+ "e10": {UniqueName: "a/a'1/a'1/a'3"},
+ "e11": {UniqueName: "a/a'1/a'1/a'4"},
+ "e12": {UniqueName: "a/a'1/a'1/a'5"},
+ "s1": {UniqueName: "a", Subtransforms: []string{"s2", "s3"}},
+ "s2": {UniqueName: "a/a", Subtransforms: []string{"s4", "s5"}},
+ "s3": {UniqueName: "a/a'1", Subtransforms: []string{"s6", "s7"}},
+ "s4": {UniqueName: "a/a/a", Subtransforms: []string{"e1"}},
+ "s5": {UniqueName: "a/a/a'1", Subtransforms: []string{"e2", "e3"}},
+ "s6": {UniqueName: "a/a'1/a", Subtransforms: []string{"e4", "e5", "e6"}},
+ "s7": {UniqueName: "a/a'1/a'1", Subtransforms: []string{"e7", "e8", "e9", "e10", "e11", "e12"}},
},
},
}
@@ -179,124 +179,124 @@ func TestComputeInputOutput(t *testing.T) {
{
name: "singleton composite",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Inputs: map[string]string{"i0": "p1"},
Outputs: map[string]string{"i0": "p2"},
- }.Build(),
+ },
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
Inputs: map[string]string{"p1": "p1"},
Outputs: map[string]string{"p2": "p2"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Inputs: map[string]string{"i0": "p1"},
Outputs: map[string]string{"i0": "p2"},
- }.Build(),
+ },
},
},
{
name: "closed composite",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2", "3"},
- }.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "2": {UniqueName: "b", Outputs: map[string]string{"i0": "p1"}},
+ "3": {UniqueName: "c", Inputs: map[string]string{"i0": "p1"}},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2", "3"},
- }.Build(),
- "2": pipepb.PTransform_builder{UniqueName: "b", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "2": {UniqueName: "b", Outputs: map[string]string{"i0": "p1"}},
+ "3": {UniqueName: "c", Inputs: map[string]string{"i0": "p1"}},
},
},
{
name: "nested composites",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"3", "7", "8"},
- }.Build(),
- "3": pipepb.PTransform_builder{
+ },
+ "3": {
UniqueName: "c",
Subtransforms: []string{"4", "5", "6"},
- }.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}}.Build(),
- "6": pipepb.PTransform_builder{UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}}.Build(),
- "7": pipepb.PTransform_builder{UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}}.Build(),
- "8": pipepb.PTransform_builder{UniqueName: "h", Inputs: map[string]string{"i0": "p7"}}.Build(),
+ },
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}},
+ "6": {UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}},
+ "7": {UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}},
+ "8": {UniqueName: "h", Inputs: map[string]string{"i0": "p7"}},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"2"},
Inputs: map[string]string{"p1": "p1", "p5": "p5", "p8": "p8"},
Outputs: map[string]string{"p3": "p3"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"3", "7", "8"},
Inputs: map[string]string{"p1": "p1", "p5": "p5", "p8": "p8"},
Outputs: map[string]string{"p3": "p3"},
- }.Build(),
- "3": pipepb.PTransform_builder{
+ },
+ "3": {
UniqueName: "c",
Subtransforms: []string{"4", "6", "5"}, // topologically sorted.
Inputs: map[string]string{"p1": "p1", "p5": "p5"},
Outputs: map[string]string{"p4": "p4", "p6": "p6", "p3": "p3"},
- }.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}}.Build(),
- "6": pipepb.PTransform_builder{UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}}.Build(),
- "7": pipepb.PTransform_builder{UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}}.Build(),
- "8": pipepb.PTransform_builder{UniqueName: "h", Inputs: map[string]string{"i0": "p7"}}.Build(),
+ },
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}, Outputs: map[string]string{"i0": "p2"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p2"}, Outputs: map[string]string{"i0": "p3", "i1": "p4"}},
+ "6": {UniqueName: "f", Inputs: map[string]string{"i0": "p2", "i1": "p5"}, Outputs: map[string]string{"i0": "p6"}},
+ "7": {UniqueName: "g", Inputs: map[string]string{"i0": "p4", "i1": "p6", "i2": "p8"}, Outputs: map[string]string{"i0": "p7"}},
+ "8": {UniqueName: "h", Inputs: map[string]string{"i0": "p7"}},
},
}, {
name: "sibling composite",
in: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"3", "4"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"5"},
- }.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "3": {UniqueName: "c", Outputs: map[string]string{"i0": "p1"}},
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p1"}},
},
exp: map[string]*pipepb.PTransform{
- "1": pipepb.PTransform_builder{
+ "1": {
UniqueName: "a",
Subtransforms: []string{"3", "4"},
Outputs: map[string]string{"p1": "p1"},
- }.Build(),
- "2": pipepb.PTransform_builder{
+ },
+ "2": {
UniqueName: "b",
Subtransforms: []string{"5"},
Inputs: map[string]string{"p1": "p1"},
- }.Build(),
- "3": pipepb.PTransform_builder{UniqueName: "c", Outputs: map[string]string{"i0": "p1"}}.Build(),
- "4": pipepb.PTransform_builder{UniqueName: "d", Inputs: map[string]string{"i0": "p1"}}.Build(),
- "5": pipepb.PTransform_builder{UniqueName: "e", Inputs: map[string]string{"i0": "p1"}}.Build(),
+ },
+ "3": {UniqueName: "c", Outputs: map[string]string{"i0": "p1"}},
+ "4": {UniqueName: "d", Inputs: map[string]string{"i0": "p1"}},
+ "5": {UniqueName: "e", Inputs: map[string]string{"i0": "p1"}},
},
},
}
@@ -317,15 +317,15 @@ func BenchmarkComputeInputOutput(b *testing.B) {
for i := 0; i < 3000; i++ {
compositeID := fmt.Sprintf("x%d", i)
primitiveID := fmt.Sprintf("y%d", i)
- in[compositeID] = pipepb.PTransform_builder{
+ in[compositeID] = &pipepb.PTransform{
UniqueName: compositeID,
Subtransforms: []string{primitiveID},
- }.Build()
- in[primitiveID] = pipepb.PTransform_builder{
+ }
+ in[primitiveID] = &pipepb.PTransform{
UniqueName: primitiveID,
Inputs: map[string]string{"i0": fmt.Sprintf("p%d", i)},
Outputs: map[string]string{"i0": fmt.Sprintf("p%d", i+1)},
- }.Build()
+ }
}
b.ResetTimer()
b.ReportAllocs()
@@ -395,11 +395,11 @@ func TestApplySdkImageOverrides(t *testing.T) {
wantEnvs[id] = env
}
- p := pipepb.Pipeline_builder{
- Components: pipepb.Components_builder{
+ p := &pipepb.Pipeline{
+ Components: &pipepb.Components{
Environments: envs,
- }.Build(),
- }.Build()
+ },
+ }
if err := ApplySdkImageOverrides(p, test.patterns); err != nil {
t.Fatalf("ApplySdkImageOverrides failed: %v", err)
}
@@ -412,16 +412,16 @@ func TestApplySdkImageOverrides(t *testing.T) {
func buildEnvironment(t *testing.T, containerImg string) *pipepb.Environment {
t.Helper()
- env := pipepb.Environment_builder{
+ env := &pipepb.Environment{
Urn: "alpha",
- DisplayData: []*pipepb.DisplayData{pipepb.DisplayData_builder{Urn: "beta"}.Build()},
+ DisplayData: []*pipepb.DisplayData{{Urn: "beta"}},
Capabilities: []string{"delta", "gamma"},
- }.Build()
- pl := pipepb.DockerPayload_builder{ContainerImage: containerImg}.Build()
- plb, err := proto.Marshal(pl)
+ }
+ pl := pipepb.DockerPayload{ContainerImage: containerImg}
+ plb, err := proto.Marshal(&pl)
if err != nil {
t.Fatalf("Failed to marshal DockerPayload with container image %v: %v", containerImg, err)
}
- env.SetPayload(plb)
+ env.Payload = plb
return env
}
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/util.go b/sdks/go/pkg/beam/core/runtime/pipelinex/util.go
index 8ba87ff1e65..4735e7b77d2 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/util.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/util.go
@@ -25,7 +25,7 @@ import (
// Bounded returns true iff all PCollections are bounded.
func Bounded(p *pipepb.Pipeline) bool {
for _, col := range p.GetComponents().GetPcollections() {
- if col.GetIsBounded() == pipepb.IsBounded_UNBOUNDED {
+ if col.IsBounded == pipepb.IsBounded_UNBOUNDED {
return false
}
}
@@ -41,7 +41,7 @@ func ContainerImages(p *pipepb.Pipeline) []string {
for _, t := range p.GetComponents().GetEnvironments() {
var payload pipepb.DockerPayload
proto.Unmarshal(t.GetPayload(), &payload)
- ret = append(ret, payload.GetContainerImage())
+ ret = append(ret, payload.ContainerImage)
}
return ret
}
@@ -75,7 +75,7 @@ func newVisiter(xforms map[string]*pipepb.PTransform, ids []string) *visiter {
next: make(map[string][]string),
}
for _, id := range ids {
- for _, in := range xforms[id].GetInputs() {
+ for _, in := range xforms[id].Inputs {
ret.next[in] = append(ret.next[in], id)
}
}
@@ -91,8 +91,8 @@ func (v *visiter) visit(xforms map[string]*pipepb.PTransform, id string) {
}
v.seen[id] = true
// Deterministically iterate through the output keys.
- outputKeys := make([]string, 0, len(xforms[id].GetOutputs()))
- for _, k := range xforms[id].GetOutputs() {
+ outputKeys := make([]string, 0, len(xforms[id].Outputs))
+ for _, k := range xforms[id].Outputs {
outputKeys = append(outputKeys, k)
}
sort.Strings(outputKeys)
diff --git a/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go b/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go
index 795e0fb7679..bdadd669629 100644
--- a/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go
+++ b/sdks/go/pkg/beam/core/runtime/pipelinex/util_test.go
@@ -145,23 +145,23 @@ func ptSink(input string, sides ...string) *pipepb.PTransform {
for i, s := range sides {
ins[fmt.Sprintf("i%d", i+1)] = s
}
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Inputs: ins,
- }.Build()
+ }
}
// ptComp generates a composite PCollection. Unlike the other helpers, it takes in
// the *subtransform ids* instead of the input/output pcollections.
func ptComp(subs ...string) *pipepb.PTransform {
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Subtransforms: subs,
- }.Build()
+ }
}
func ptImpulse(output string) *pipepb.PTransform {
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Outputs: map[string]string{"o0": output},
- }.Build()
+ }
}
func ptNoSide(input string, outputs ...string) *pipepb.PTransform {
@@ -169,10 +169,10 @@ func ptNoSide(input string, outputs ...string) *pipepb.PTransform {
for i, o := range outputs {
outs[fmt.Sprintf("o%d", i)] = o
}
- return pipepb.PTransform_builder{
+ return &pipepb.PTransform{
Inputs: map[string]string{"i0": input},
Outputs: outs,
- }.Build()
+ }
}
// validateSortForTest ensures that in the sorted ids, outputs are declared before
@@ -186,7 +186,7 @@ func validateSortForTest(t *testing.T, xforms map[string]*pipepb.PTransform, sor
for _, id := range sorted {
pt := xforms[id]
- for _, in := range pt.GetInputs() {
+ for _, in := range pt.Inputs {
if !seenPCollections[in] {
t.Errorf("out of order pcollection %v required by %v", in, id)
missingPCollections[in] = true
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/expand.go b/sdks/go/pkg/beam/core/runtime/xlangx/expand.go
index 4aae6062bbc..94dda75e805 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/expand.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/expand.go
@@ -62,7 +62,7 @@ func Expand(edge *graph.MultiEdge, ext *graph.ExternalTransform) error {
// should be removed from proto before submitting expansion request.
extTransformID := p.GetRootTransformIds()[0]
extTransform := transforms[extTransformID]
- for extTransform.GetUniqueName() != "External" {
+ for extTransform.UniqueName != "External" {
delete(transforms, extTransformID)
p, err = pipelinex.Normalize(p) // Update root transform IDs.
if err != nil {
@@ -136,12 +136,12 @@ func expand(
}
return h(ctx, &HandlerParams{
Config: config,
- Req: jobpb.ExpansionRequest_builder{
+ Req: &jobpb.ExpansionRequest{
Components: comps,
Transform: transform,
Namespace: ext.Namespace,
OutputCoderRequests: outputCoderID,
- }.Build(),
+ },
edge: edge,
ext: ext,
})
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go b/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go
index 20d2acc7eb5..723d2e25cce 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/namespace.go
@@ -27,27 +27,27 @@ func addCoderID(c *pipepb.Components, idMap map[string]string, cid string, newID
return idMap[cid]
}
- coder, exists := c.GetCoders()[cid]
+ coder, exists := c.Coders[cid]
if !exists {
- panic(errors.Errorf("attempted to add namespace to missing coder id: %v not in %v", cid, c.GetCoders()))
+ panic(errors.Errorf("attempted to add namespace to missing coder id: %v not in %v", cid, c.Coders))
}
// Updating ComponentCoderIDs of Coder
if coder.GetComponentCoderIds() != nil {
var updatedComponentCoderIDs []string
- updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.GetComponentCoderIds()...)
+ updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.ComponentCoderIds...)
- for i, ccid := range coder.GetComponentCoderIds() {
+ for i, ccid := range coder.ComponentCoderIds {
updatedComponentCoderIDs[i] = addCoderID(c, idMap, ccid, newID)
}
- coder.SetComponentCoderIds(updatedComponentCoderIDs)
+ coder.ComponentCoderIds = updatedComponentCoderIDs
}
idMap[cid] = newID(cid)
// Updating Coders map
- c.GetCoders()[idMap[cid]] = coder
- delete(c.GetCoders(), cid)
+ c.Coders[idMap[cid]] = coder
+ delete(c.Coders, cid)
return idMap[cid]
}
@@ -57,21 +57,21 @@ func addWindowingStrategyID(c *pipepb.Components, idMap map[string]string, wid s
return idMap[wid]
}
- windowingStrategy, exists := c.GetWindowingStrategies()[wid]
+ windowingStrategy, exists := c.WindowingStrategies[wid]
if !exists {
- panic(errors.Errorf("attempted to add namespace to missing windowing strategy id: %v not in %v", wid, c.GetWindowingStrategies()))
+ panic(errors.Errorf("attempted to add namespace to missing windowing strategy id: %v not in %v", wid, c.WindowingStrategies))
}
// Updating WindowCoderID of WindowingStrategy
- if windowingStrategy.GetWindowCoderId() != "" {
- windowingStrategy.SetWindowCoderId(addCoderID(c, idMap, windowingStrategy.GetWindowCoderId(), newID))
+ if windowingStrategy.WindowCoderId != "" {
+ windowingStrategy.WindowCoderId = addCoderID(c, idMap, windowingStrategy.WindowCoderId, newID)
}
idMap[wid] = newID(wid)
// Updating WindowingStrategies map
- c.GetWindowingStrategies()[idMap[wid]] = windowingStrategy
- delete(c.GetWindowingStrategies(), wid)
+ c.WindowingStrategies[idMap[wid]] = windowingStrategy
+ delete(c.WindowingStrategies, wid)
return idMap[wid]
}
@@ -88,14 +88,14 @@ func addNamespace(t *pipepb.PTransform, c *pipepb.Components, namespace string)
// environment. If multiple Go SDK environments become possible, then
// namespacing of non-default environments should happen here.
- for _, pcolsMap := range []map[string]string{t.GetInputs(), t.GetOutputs()} {
+ for _, pcolsMap := range []map[string]string{t.Inputs, t.Outputs} {
for _, pid := range pcolsMap {
- if pcol, exists := c.GetPcollections()[pid]; exists {
+ if pcol, exists := c.Pcollections[pid]; exists {
// Update Coder ID of PCollection
- pcol.SetCoderId(addCoderID(c, idMap, pcol.GetCoderId(), newID))
+ pcol.CoderId = addCoderID(c, idMap, pcol.CoderId, newID)
// Update WindowingStrategyID of PCollection
- pcol.SetWindowingStrategyId(addWindowingStrategyID(c, idMap, pcol.GetWindowingStrategyId(), newID))
+ pcol.WindowingStrategyId = addWindowingStrategyID(c, idMap, pcol.WindowingStrategyId, newID)
}
}
}
@@ -106,32 +106,32 @@ func addNamespace(t *pipepb.PTransform, c *pipepb.Components, namespace string)
continue
}
var updatedComponentCoderIDs []string
- updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.GetComponentCoderIds()...)
+ updatedComponentCoderIDs = append(updatedComponentCoderIDs, coder.ComponentCoderIds...)
for i, ccid := range coder.GetComponentCoderIds() {
if _, exists := idMap[ccid]; exists {
updatedComponentCoderIDs[i] = idMap[ccid]
}
}
- coder.SetComponentCoderIds(updatedComponentCoderIDs)
+ coder.ComponentCoderIds = updatedComponentCoderIDs
}
- sourceName := t.GetUniqueName()
- for _, t := range c.GetTransforms() {
- if t.GetUniqueName() != sourceName {
- if id, exists := idMap[t.GetEnvironmentId()]; exists {
- t.SetEnvironmentId(id)
+ sourceName := t.UniqueName
+ for _, t := range c.Transforms {
+ if t.UniqueName != sourceName {
+ if id, exists := idMap[t.EnvironmentId]; exists {
+ t.EnvironmentId = id
}
- for _, pcolsMap := range []map[string]string{t.GetInputs(), t.GetOutputs()} {
+ for _, pcolsMap := range []map[string]string{t.Inputs, t.Outputs} {
for _, pid := range pcolsMap {
- if pcol, exists := c.GetPcollections()[pid]; exists {
+ if pcol, exists := c.Pcollections[pid]; exists {
// Update Coder ID of PCollection
- if id, exists := idMap[pcol.GetCoderId()]; exists {
- pcol.SetCoderId(id)
+ if id, exists := idMap[pcol.CoderId]; exists {
+ pcol.CoderId = id
}
// Update WindowingStrategyID of PCollection
- if id, exists := idMap[pcol.GetWindowingStrategyId()]; exists {
- pcol.SetWindowingStrategyId(id)
+ if id, exists := idMap[pcol.WindowingStrategyId]; exists {
+ pcol.WindowingStrategyId = id
}
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go b/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go
index eb98d0d7e7d..e2ee43780bc 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/namespace_test.go
@@ -41,515 +41,515 @@ func TestAddNamespace(t *testing.T) {
}{
{
name: "[Correctness] Single Input Multi Output",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p1": {CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Correctness] Single Input Single Output",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t1",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p1": {CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p3": {CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Correctness] Multi Input Single Output",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3", WindowingStrategyId: "w1"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w0"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
+ "p3": {CoderId: "c3", WindowingStrategyId: "w1"},
+ "p4": {CoderId: "c2", WindowingStrategyId: "w0"},
+ "p5": {CoderId: "c2", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t3",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1", "t0o1": "p2"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p3"},
EnvironmentId: "e1",
- }.Build(),
- "t2": pipepb.PTransform_builder{
+ },
+ "t2": {
UniqueName: "t2",
Inputs: map[string]string{"t2i0": "p2"},
Outputs: map[string]string{"t2o0": "p4"},
EnvironmentId: "e0",
- }.Build(),
- "t3": pipepb.PTransform_builder{
+ },
+ "t3": {
UniqueName: "t3",
Inputs: map[string]string{"t3i0": "p3", "t3i1": "p4"},
Outputs: map[string]string{"t3o0": "p5"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p3": pipepb.PCollection_builder{CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p4": pipepb.PCollection_builder{CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"}.Build(),
- "p5": pipepb.PCollection_builder{CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p3": {CoderId: "c3@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p4": {CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"},
+ "p5": {CoderId: "c2@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0@daASxQwenJ": {WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Correctness] Component Coders",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build(), ComponentCoderIds: []string{"c2"}}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}, ComponentCoderIds: []string{"c2"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
- want: pipepb.Components_builder{
+ want: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"}.Build(),
+ "p0": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w0@daASxQwenJ"},
+ "p1": {CoderId: "c1@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
+ "p2": {CoderId: "c0@daASxQwenJ", WindowingStrategyId: "w1@daASxQwenJ"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"}.Build(),
- "w1@daASxQwenJ": pipepb.WindowingStrategy_builder{WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"}.Build(),
+ "w0@daASxQwenJ": {WindowCoderId: "c3@daASxQwenJ", EnvironmentId: "e0"},
+ "w1@daASxQwenJ": {WindowCoderId: "c4@daASxQwenJ", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build(), ComponentCoderIds: []string{"c2@daASxQwenJ"}}.Build(),
- "c1@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
- "c2@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c2"}.Build()}.Build(),
- "c3@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c3"}.Build()}.Build(),
- "c4@daASxQwenJ": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c4"}.Build()}.Build(),
+ "c0@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c0"}, ComponentCoderIds: []string{"c2@daASxQwenJ"}},
+ "c1@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
+ "c2@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c2"}},
+ "c3@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c3"}},
+ "c4@daASxQwenJ": {Spec: &pipepb.FunctionSpec{Urn: "c4"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
},
{
name: "[Consistency] Missing EnvironmentID",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
},
Environments: map[string]*pipepb.Environment{
// Missing "e0"
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
err: "trying to add an Environment whose key is absent should panic",
},
{
name: "[Consistency] Missing WindowingStrategyID",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
// Missing w0
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
- "c0": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c0"}.Build()}.Build(),
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
+ "c0": {Spec: &pipepb.FunctionSpec{Urn: "c0"}},
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
err: "trying to add an WindowingStrategy whose key is absent should panic",
},
{
name: "[Consistency] Missing CoderID",
- init: pipepb.Components_builder{
+ init: &pipepb.Components{
Transforms: map[string]*pipepb.PTransform{
- "t0": pipepb.PTransform_builder{
+ "t0": {
UniqueName: "t0",
Inputs: map[string]string{"t0i0": "p0"},
Outputs: map[string]string{"t0o0": "p1"},
EnvironmentId: "e0",
- }.Build(),
- "t1": pipepb.PTransform_builder{
+ },
+ "t1": {
UniqueName: "t1",
Inputs: map[string]string{"t1i0": "p1"},
Outputs: map[string]string{"t1o0": "p2"},
EnvironmentId: "e1",
- }.Build(),
+ },
},
Pcollections: map[string]*pipepb.PCollection{
- "p0": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w0"}.Build(),
- "p1": pipepb.PCollection_builder{CoderId: "c1", WindowingStrategyId: "w1"}.Build(),
- "p2": pipepb.PCollection_builder{CoderId: "c0", WindowingStrategyId: "w1"}.Build(),
+ "p0": {CoderId: "c0", WindowingStrategyId: "w0"},
+ "p1": {CoderId: "c1", WindowingStrategyId: "w1"},
+ "p2": {CoderId: "c0", WindowingStrategyId: "w1"},
},
WindowingStrategies: map[string]*pipepb.WindowingStrategy{
- "w0": pipepb.WindowingStrategy_builder{WindowCoderId: "c3", EnvironmentId: "e0"}.Build(),
- "w1": pipepb.WindowingStrategy_builder{WindowCoderId: "c4", EnvironmentId: "e1"}.Build(),
+ "w0": {WindowCoderId: "c3", EnvironmentId: "e0"},
+ "w1": {WindowCoderId: "c4", EnvironmentId: "e1"},
},
Coders: map[string]*pipepb.Coder{
// Missing c0
- "c1": pipepb.Coder_builder{Spec: pipepb.FunctionSpec_builder{Urn: "c1"}.Build()}.Build(),
+ "c1": {Spec: &pipepb.FunctionSpec{Urn: "c1"}},
},
Environments: map[string]*pipepb.Environment{
- "e0": pipepb.Environment_builder{Urn: "e0"}.Build(),
- "e1": pipepb.Environment_builder{Urn: "e1"}.Build(),
+ "e0": {Urn: "e0"},
+ "e1": {Urn: "e1"},
},
- }.Build(),
+ },
namespace: "daASxQwenJ",
transformID: "t0",
err: "trying to add an WindowingStrategy whose key is absent should panic",
@@ -559,7 +559,7 @@ func TestAddNamespace(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if strings.Contains(tt.name, "Correctness") {
- transform := tt.init.GetTransforms()[tt.transformID]
+ transform := tt.init.Transforms[tt.transformID]
addNamespace(transform, tt.init, tt.namespace)
@@ -570,7 +570,7 @@ func TestAddNamespace(t *testing.T) {
if strings.Contains(tt.name, "Consistency") {
defer expectPanic(t, tt.err)
- transform := tt.init.GetTransforms()[tt.transformID]
+ transform := tt.init.Transforms[tt.transformID]
addNamespace(transform, tt.init, tt.namespace)
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/payload.go b/sdks/go/pkg/beam/core/runtime/xlangx/payload.go
index fcab9cd769a..8c01ffbd34f 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/payload.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/payload.go
@@ -52,10 +52,10 @@ func CreateExternalConfigurationPayload(pl any) (*pipepb.ExternalConfigurationPa
}
// Put schema and row into payload proto, and marshal it.
- ecp := pipepb.ExternalConfigurationPayload_builder{
+ ecp := &pipepb.ExternalConfigurationPayload{
Schema: scm,
Payload: buf.Bytes(),
- }.Build()
+ }
return ecp, nil
}
@@ -102,7 +102,7 @@ func DecodeStructPayload(plBytes []byte) (any, error) {
err = errors.WithContextf(err, "creating Row decoder for type %v", rt)
return nil, errors.WithContext(err, "decoding external payload")
}
- buf := bytes.NewBuffer(ecp.GetPayload())
+ buf := bytes.NewBuffer(ecp.Payload)
val, err := dec(buf)
if err != nil {
err = errors.WithContext(err, "decoding Row to payload")
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/registry.go b/sdks/go/pkg/beam/core/runtime/xlangx/registry.go
index d796dee5a15..1957cc61bb9 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/registry.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/registry.go
@@ -78,7 +78,7 @@ type HandlerParams struct {
// CoderMarshaller returns a coder marshaller initialized with the request's namespace.
func (p *HandlerParams) CoderMarshaller() *graphx.CoderMarshaller {
cm := graphx.NewCoderMarshaller()
- cm.Namespace = p.Req.GetNamespace()
+ cm.Namespace = p.Req.Namespace
return cm
}
@@ -134,7 +134,7 @@ func makePCol(node *graph.Node, index int, local, namespace string) PCol {
func (p *HandlerParams) Outputs() []PCol {
out := make([]PCol, 0, len(p.ext.OutputsMap))
for local, i := range p.ext.OutputsMap {
- out = append(out, makePCol(p.edge.Output[i].To, i, local, p.Req.GetNamespace()))
+ out = append(out, makePCol(p.edge.Output[i].To, i, local, p.Req.Namespace))
}
return out
}
@@ -144,7 +144,7 @@ func (p *HandlerParams) Outputs() []PCol {
func (p *HandlerParams) Inputs() []PCol {
out := make([]PCol, 0, len(p.ext.InputsMap))
for local, i := range p.ext.InputsMap {
- out = append(out, makePCol(p.edge.Input[i].From, i, local, p.Req.GetNamespace()))
+ out = append(out, makePCol(p.edge.Input[i].From, i, local, p.Req.Namespace))
}
return out
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go b/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go
index a62025039ac..aabe6eface4 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/resolve.go
@@ -87,7 +87,7 @@ func ResolveArtifactsWithConfig(ctx context.Context, edges []*graph.MultiEdge, c
return nil, errors.WithContextf(err,
"resolving remote artifacts for edge %v", e.Name())
}
- envs := components.GetEnvironments()
+ envs := components.Environments
for eid, env := range envs {
if strings.HasPrefix(eid, "go") {
continue
@@ -111,21 +111,21 @@ func ResolveArtifactsWithConfig(ctx context.Context, edges []*graph.MultiEdge, c
fullSdkPath = cfg.JoinFn(cfg.SdkPath, name)
}
resolvedDeps = append(resolvedDeps,
- pipepb.ArtifactInformation_builder{
+ &pipepb.ArtifactInformation{
TypeUrn: "beam:artifact:type:file:v1",
TypePayload: protox.MustEncode(
- pipepb.ArtifactFilePayload_builder{
+ &pipepb.ArtifactFilePayload{
Path: fullSdkPath,
Sha256: sha256,
- }.Build(),
+ },
),
- RoleUrn: a.GetRoleUrn(),
- RolePayload: a.GetRolePayload(),
- }.Build(),
+ RoleUrn: a.RoleUrn,
+ RolePayload: a.RolePayload,
+ },
)
paths[fullTmpPath] = fullSdkPath
}
- env.SetDependencies(resolvedDeps)
+ env.Dependencies = resolvedDeps
}
}
}
@@ -142,24 +142,24 @@ func UpdateArtifactTypeFromFileToURL(edges []*graph.MultiEdge) {
panic(errors.WithContextf(err,
"updating URL artifacts type for edge %v", e.Name()))
}
- envs := components.GetEnvironments()
+ envs := components.Environments
for _, env := range envs {
deps := env.GetDependencies()
var resolvedDeps []*pipepb.ArtifactInformation
for _, a := range deps {
path, sha256 := artifact.MustExtractFilePayload(a)
if strings.Contains(path, "://") {
- a.SetTypeUrn("beam:artifact:type:url:v1")
- a.SetTypePayload(protox.MustEncode(
- pipepb.ArtifactUrlPayload_builder{
+ a.TypeUrn = "beam:artifact:type:url:v1"
+ a.TypePayload = protox.MustEncode(
+ &pipepb.ArtifactUrlPayload{
Url: path,
Sha256: sha256,
- }.Build(),
- ))
+ },
+ )
}
resolvedDeps = append(resolvedDeps, a)
}
- env.SetDependencies(resolvedDeps)
+ env.Dependencies = resolvedDeps
}
}
}
diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go b/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go
index 1693ad14b22..eec13c451a1 100644
--- a/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go
+++ b/sdks/go/pkg/beam/core/runtime/xlangx/resolve_test.go
@@ -25,30 +25,30 @@ import (
func createExternalEdge(typeUrn string, typePayload []byte) *graph.MultiEdge {
env := map[string]*pipepb.Environment{
- "env_java": pipepb.Environment_builder{
+ "env_java": {
Dependencies: []*pipepb.ArtifactInformation{
- pipepb.ArtifactInformation_builder{
+ {
TypeUrn: typeUrn,
TypePayload: typePayload,
- }.Build(),
+ },
},
- }.Build(),
+ },
}
return &graph.MultiEdge{
Op: graph.External,
External: &graph.ExternalTransform{
Expanded: &graph.ExpandedTransform{
- Components: pipepb.Components_builder{
+ Components: &pipepb.Components{
Environments: env,
- }.Build(),
+ },
},
},
}
}
func TestUpdateFileArtifactWithUrlPath(t *testing.T) {
- payload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: "gs://dummy"}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: "gs://dummy"})
e := createExternalEdge("beam:artifact:type:file:v1", payload)
UpdateArtifactTypeFromFileToURL([]*graph.MultiEdge{e})
expected := createExternalEdge("beam:artifact:type:url:v1", payload)
@@ -63,8 +63,8 @@ func TestUpdateFileArtifactWithUrlPath(t *testing.T) {
}
func TestUpdateFileArtifactWithLocalPath(t *testing.T) {
- payload, _ := proto.Marshal(pipepb.ArtifactFilePayload_builder{
- Path: "/tmp/artifact/dummy"}.Build())
+ payload, _ := proto.Marshal(&pipepb.ArtifactFilePayload{
+ Path: "/tmp/artifact/dummy"})
e := createExternalEdge("beam:artifact:type:file:v1", payload)
UpdateArtifactTypeFromFileToURL([]*graph.MultiEdge{e})
expected := createExternalEdge("beam:artifact:type:file:v1", payload)
diff --git a/sdks/go/pkg/beam/io/pubsubio/pubsubio.go b/sdks/go/pkg/beam/io/pubsubio/pubsubio.go
index 8eef70b7f04..bcf20e89c89 100644
--- a/sdks/go/pkg/beam/io/pubsubio/pubsubio.go
+++ b/sdks/go/pkg/beam/io/pubsubio/pubsubio.go
@@ -64,16 +64,16 @@ type ReadOptions struct {
func Read(s beam.Scope, project, topic string, opts *ReadOptions) beam.PCollection {
s = s.Scope("pubsubio.Read")
- payload := pipepb.PubSubReadPayload_builder{
+ payload := &pipepb.PubSubReadPayload{
Topic: pubsubx.MakeQualifiedTopicName(project, topic),
- }.Build()
+ }
if opts != nil {
- payload.SetIdAttribute(opts.IDAttribute)
- payload.SetTimestampAttribute(opts.TimestampAttribute)
+ payload.IdAttribute = opts.IDAttribute
+ payload.TimestampAttribute = opts.TimestampAttribute
if opts.Subscription != "" {
- payload.SetSubscription(pubsubx.MakeQualifiedSubscriptionName(project, opts.Subscription))
+ payload.Subscription = pubsubx.MakeQualifiedSubscriptionName(project, opts.Subscription)
}
- payload.SetWithAttributes(opts.WithAttributes)
+ payload.WithAttributes = opts.WithAttributes
}
out := beam.External(s, readURN, protox.MustEncode(payload), nil, []beam.FullType{typex.New(reflectx.ByteSlice)}, false)
@@ -118,9 +118,9 @@ var pubSubMessageT = reflect.TypeOf((*pb.PubsubMessage)(nil))
func Write(s beam.Scope, project, topic string, col beam.PCollection) {
s = s.Scope("pubsubio.Write")
- payload := pipepb.PubSubWritePayload_builder{
+ payload := &pipepb.PubSubWritePayload{
Topic: pubsubx.MakeQualifiedTopicName(project, topic),
- }.Build()
+ }
out := col
if col.Type().Type() == reflectx.ByteSlice {
diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
index 3eb826fae93..55ec131df06 100644
--- a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
+++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go
@@ -26,15 +26,13 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
-// protoc-gen-go v1.36.0
+// protoc-gen-go v1.27.1
// protoc v5.27.3
// source: org/apache/beam/model/fn_execution/v1/beam_fn_api.proto
// TODO: Consider consolidating common components in another package
// and language namespaces for re-use with Runner Api.
-//go:build !protoopaque
-
package fnexecution_v1
import (
@@ -46,6 +44,7 @@ import (
structpb "google.golang.org/protobuf/types/known/structpb"
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
reflect "reflect"
+ sync "sync"
)
const (
@@ -131,6 +130,11 @@ func (x FnApiTransforms_Runner) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
+// Deprecated: Use FnApiTransforms_Runner.Descriptor instead.
+func (FnApiTransforms_Runner) EnumDescriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{0, 0}
+}
+
type LogEntry_Severity_Enum int32
const (
@@ -199,19 +203,26 @@ func (x LogEntry_Severity_Enum) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
+// Deprecated: Use LogEntry_Severity_Enum.Descriptor instead.
+func (LogEntry_Severity_Enum) EnumDescriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{36, 1, 0}
+}
+
// Describes transforms necessary to execute Beam over the FnAPI but are
// implementation details rather than part of the core model.
type FnApiTransforms struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- unknownFields protoimpl.UnknownFields
+ state protoimpl.MessageState
sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
}
func (x *FnApiTransforms) Reset() {
*x = FnApiTransforms{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *FnApiTransforms) String() string {
@@ -222,7 +233,7 @@ func (*FnApiTransforms) ProtoMessage() {}
func (x *FnApiTransforms) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -232,16 +243,9 @@ func (x *FnApiTransforms) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-type FnApiTransforms_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
-}
-
-func (b0 FnApiTransforms_builder) Build() *FnApiTransforms {
- m0 := &FnApiTransforms{}
- b, x := &b0, m0
- _, _ = b, x
- return m0
+// Deprecated: Use FnApiTransforms.ProtoReflect.Descriptor instead.
+func (*FnApiTransforms) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{0}
}
// A descriptor for connecting to a remote port using the Beam Fn Data API.
@@ -249,22 +253,25 @@ func (b0 FnApiTransforms_builder) Build() *FnApiTransforms {
// runner and the SDK).
// Stable
type RemoteGrpcPort struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) An API descriptor which describes where to
// connect to including any authentication that is required.
ApiServiceDescriptor *pipeline_v1.ApiServiceDescriptor `protobuf:"bytes,1,opt,name=api_service_descriptor,json=apiServiceDescriptor,proto3" json:"api_service_descriptor,omitempty"`
// (Required) The ID of the Coder that will be used to encode and decode data
// sent over this port.
- CoderId string `protobuf:"bytes,2,opt,name=coder_id,json=coderId,proto3" json:"coder_id,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ CoderId string `protobuf:"bytes,2,opt,name=coder_id,json=coderId,proto3" json:"coder_id,omitempty"`
}
func (x *RemoteGrpcPort) Reset() {
*x = RemoteGrpcPort{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *RemoteGrpcPort) String() string {
@@ -275,7 +282,7 @@ func (*RemoteGrpcPort) ProtoMessage() {}
func (x *RemoteGrpcPort) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -285,6 +292,11 @@ func (x *RemoteGrpcPort) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use RemoteGrpcPort.ProtoReflect.Descriptor instead.
+func (*RemoteGrpcPort) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{1}
+}
+
func (x *RemoteGrpcPort) GetApiServiceDescriptor() *pipeline_v1.ApiServiceDescriptor {
if x != nil {
return x.ApiServiceDescriptor
@@ -299,58 +311,22 @@ func (x *RemoteGrpcPort) GetCoderId() string {
return ""
}
-func (x *RemoteGrpcPort) SetApiServiceDescriptor(v *pipeline_v1.ApiServiceDescriptor) {
- x.ApiServiceDescriptor = v
-}
-
-func (x *RemoteGrpcPort) SetCoderId(v string) {
- x.CoderId = v
-}
-
-func (x *RemoteGrpcPort) HasApiServiceDescriptor() bool {
- if x == nil {
- return false
- }
- return x.ApiServiceDescriptor != nil
-}
-
-func (x *RemoteGrpcPort) ClearApiServiceDescriptor() {
- x.ApiServiceDescriptor = nil
-}
-
-type RemoteGrpcPort_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) An API descriptor which describes where to
- // connect to including any authentication that is required.
- ApiServiceDescriptor *pipeline_v1.ApiServiceDescriptor
- // (Required) The ID of the Coder that will be used to encode and decode data
- // sent over this port.
- CoderId string
-}
-
-func (b0 RemoteGrpcPort_builder) Build() *RemoteGrpcPort {
- m0 := &RemoteGrpcPort{}
- b, x := &b0, m0
- _, _ = b, x
- x.ApiServiceDescriptor = b.ApiServiceDescriptor
- x.CoderId = b.CoderId
- return m0
-}
-
// Requests the ProcessBundleDescriptor with the given id.
type GetProcessBundleDescriptorRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- ProcessBundleDescriptorId string `protobuf:"bytes,1,opt,name=process_bundle_descriptor_id,json=processBundleDescriptorId,proto3" json:"process_bundle_descriptor_id,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ ProcessBundleDescriptorId string `protobuf:"bytes,1,opt,name=process_bundle_descriptor_id,json=processBundleDescriptorId,proto3" json:"process_bundle_descriptor_id,omitempty"`
}
func (x *GetProcessBundleDescriptorRequest) Reset() {
*x = GetProcessBundleDescriptorRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *GetProcessBundleDescriptorRequest) String() string {
@@ -361,7 +337,7 @@ func (*GetProcessBundleDescriptorRequest) ProtoMessage() {}
func (x *GetProcessBundleDescriptorRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -371,6 +347,11 @@ func (x *GetProcessBundleDescriptorRequest) ProtoReflect() protoreflect.Message
return mi.MessageOf(x)
}
+// Deprecated: Use GetProcessBundleDescriptorRequest.ProtoReflect.Descriptor instead.
+func (*GetProcessBundleDescriptorRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{2}
+}
+
func (x *GetProcessBundleDescriptorRequest) GetProcessBundleDescriptorId() string {
if x != nil {
return x.ProcessBundleDescriptorId
@@ -378,36 +359,21 @@ func (x *GetProcessBundleDescriptorRequest) GetProcessBundleDescriptorId() strin
return ""
}
-func (x *GetProcessBundleDescriptorRequest) SetProcessBundleDescriptorId(v string) {
- x.ProcessBundleDescriptorId = v
-}
-
-type GetProcessBundleDescriptorRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- ProcessBundleDescriptorId string
-}
-
-func (b0 GetProcessBundleDescriptorRequest_builder) Build() *GetProcessBundleDescriptorRequest {
- m0 := &GetProcessBundleDescriptorRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.ProcessBundleDescriptorId = b.ProcessBundleDescriptorId
- return m0
-}
-
// A request sent by a runner which the SDK is asked to fulfill.
// For any unsupported request type, an error should be returned with a
// matching instruction id.
// Stable
type InstructionRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) A unique identifier provided by the runner which represents
// this requests execution. The InstructionResponse MUST have the matching id.
InstructionId string `protobuf:"bytes,1,opt,name=instruction_id,json=instructionId,proto3" json:"instruction_id,omitempty"`
// (Required) A request that the SDK Harness needs to interpret.
//
- // Types that are valid to be assigned to Request:
+ // Types that are assignable to Request:
//
// *InstructionRequest_ProcessBundle
// *InstructionRequest_ProcessBundleProgress
@@ -417,16 +383,16 @@ type InstructionRequest struct {
// *InstructionRequest_HarnessMonitoringInfos
// *InstructionRequest_SampleData
// *InstructionRequest_Register
- Request isInstructionRequest_Request `protobuf_oneof:"request"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ Request isInstructionRequest_Request `protobuf_oneof:"request"`
}
func (x *InstructionRequest) Reset() {
*x = InstructionRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *InstructionRequest) String() string {
@@ -437,7 +403,7 @@ func (*InstructionRequest) ProtoMessage() {}
func (x *InstructionRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -447,6 +413,11 @@ func (x *InstructionRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use InstructionRequest.ProtoReflect.Descriptor instead.
+func (*InstructionRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{3}
+}
+
func (x *InstructionRequest) GetInstructionId() string {
if x != nil {
return x.InstructionId
@@ -454,375 +425,69 @@ func (x *InstructionRequest) GetInstructionId() string {
return ""
}
-func (x *InstructionRequest) GetRequest() isInstructionRequest_Request {
- if x != nil {
- return x.Request
+func (m *InstructionRequest) GetRequest() isInstructionRequest_Request {
+ if m != nil {
+ return m.Request
}
return nil
}
func (x *InstructionRequest) GetProcessBundle() *ProcessBundleRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_ProcessBundle); ok {
- return x.ProcessBundle
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_ProcessBundle); ok {
+ return x.ProcessBundle
}
return nil
}
func (x *InstructionRequest) GetProcessBundleProgress() *ProcessBundleProgressRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_ProcessBundleProgress); ok {
- return x.ProcessBundleProgress
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_ProcessBundleProgress); ok {
+ return x.ProcessBundleProgress
}
return nil
}
func (x *InstructionRequest) GetProcessBundleSplit() *ProcessBundleSplitRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_ProcessBundleSplit); ok {
- return x.ProcessBundleSplit
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_ProcessBundleSplit); ok {
+ return x.ProcessBundleSplit
}
return nil
}
func (x *InstructionRequest) GetFinalizeBundle() *FinalizeBundleRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_FinalizeBundle); ok {
- return x.FinalizeBundle
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_FinalizeBundle); ok {
+ return x.FinalizeBundle
}
return nil
}
func (x *InstructionRequest) GetMonitoringInfos() *MonitoringInfosMetadataRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_MonitoringInfos); ok {
- return x.MonitoringInfos
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_MonitoringInfos); ok {
+ return x.MonitoringInfos
}
return nil
}
func (x *InstructionRequest) GetHarnessMonitoringInfos() *HarnessMonitoringInfosRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_HarnessMonitoringInfos); ok {
- return x.HarnessMonitoringInfos
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_HarnessMonitoringInfos); ok {
+ return x.HarnessMonitoringInfos
}
return nil
}
func (x *InstructionRequest) GetSampleData() *SampleDataRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_SampleData); ok {
- return x.SampleData
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_SampleData); ok {
+ return x.SampleData
}
return nil
}
func (x *InstructionRequest) GetRegister() *RegisterRequest {
- if x != nil {
- if x, ok := x.Request.(*InstructionRequest_Register); ok {
- return x.Register
- }
+ if x, ok := x.GetRequest().(*InstructionRequest_Register); ok {
+ return x.Register
}
return nil
}
-func (x *InstructionRequest) SetInstructionId(v string) {
- x.InstructionId = v
-}
-
-func (x *InstructionRequest) SetProcessBundle(v *ProcessBundleRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_ProcessBundle{v}
-}
-
-func (x *InstructionRequest) SetProcessBundleProgress(v *ProcessBundleProgressRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_ProcessBundleProgress{v}
-}
-
-func (x *InstructionRequest) SetProcessBundleSplit(v *ProcessBundleSplitRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_ProcessBundleSplit{v}
-}
-
-func (x *InstructionRequest) SetFinalizeBundle(v *FinalizeBundleRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_FinalizeBundle{v}
-}
-
-func (x *InstructionRequest) SetMonitoringInfos(v *MonitoringInfosMetadataRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_MonitoringInfos{v}
-}
-
-func (x *InstructionRequest) SetHarnessMonitoringInfos(v *HarnessMonitoringInfosRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_HarnessMonitoringInfos{v}
-}
-
-func (x *InstructionRequest) SetSampleData(v *SampleDataRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_SampleData{v}
-}
-
-func (x *InstructionRequest) SetRegister(v *RegisterRequest) {
- if v == nil {
- x.Request = nil
- return
- }
- x.Request = &InstructionRequest_Register{v}
-}
-
-func (x *InstructionRequest) HasRequest() bool {
- if x == nil {
- return false
- }
- return x.Request != nil
-}
-
-func (x *InstructionRequest) HasProcessBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_ProcessBundle)
- return ok
-}
-
-func (x *InstructionRequest) HasProcessBundleProgress() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_ProcessBundleProgress)
- return ok
-}
-
-func (x *InstructionRequest) HasProcessBundleSplit() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_ProcessBundleSplit)
- return ok
-}
-
-func (x *InstructionRequest) HasFinalizeBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_FinalizeBundle)
- return ok
-}
-
-func (x *InstructionRequest) HasMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_MonitoringInfos)
- return ok
-}
-
-func (x *InstructionRequest) HasHarnessMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_HarnessMonitoringInfos)
- return ok
-}
-
-func (x *InstructionRequest) HasSampleData() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_SampleData)
- return ok
-}
-
-func (x *InstructionRequest) HasRegister() bool {
- if x == nil {
- return false
- }
- _, ok := x.Request.(*InstructionRequest_Register)
- return ok
-}
-
-func (x *InstructionRequest) ClearRequest() {
- x.Request = nil
-}
-
-func (x *InstructionRequest) ClearProcessBundle() {
- if _, ok := x.Request.(*InstructionRequest_ProcessBundle); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearProcessBundleProgress() {
- if _, ok := x.Request.(*InstructionRequest_ProcessBundleProgress); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearProcessBundleSplit() {
- if _, ok := x.Request.(*InstructionRequest_ProcessBundleSplit); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearFinalizeBundle() {
- if _, ok := x.Request.(*InstructionRequest_FinalizeBundle); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearMonitoringInfos() {
- if _, ok := x.Request.(*InstructionRequest_MonitoringInfos); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearHarnessMonitoringInfos() {
- if _, ok := x.Request.(*InstructionRequest_HarnessMonitoringInfos); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearSampleData() {
- if _, ok := x.Request.(*InstructionRequest_SampleData); ok {
- x.Request = nil
- }
-}
-
-func (x *InstructionRequest) ClearRegister() {
- if _, ok := x.Request.(*InstructionRequest_Register); ok {
- x.Request = nil
- }
-}
-
-const InstructionRequest_Request_not_set_case case_InstructionRequest_Request = 0
-const InstructionRequest_ProcessBundle_case case_InstructionRequest_Request = 1001
-const InstructionRequest_ProcessBundleProgress_case case_InstructionRequest_Request = 1002
-const InstructionRequest_ProcessBundleSplit_case case_InstructionRequest_Request = 1003
-const InstructionRequest_FinalizeBundle_case case_InstructionRequest_Request = 1004
-const InstructionRequest_MonitoringInfos_case case_InstructionRequest_Request = 1005
-const InstructionRequest_HarnessMonitoringInfos_case case_InstructionRequest_Request = 1006
-const InstructionRequest_SampleData_case case_InstructionRequest_Request = 1007
-const InstructionRequest_Register_case case_InstructionRequest_Request = 1000
-
-func (x *InstructionRequest) WhichRequest() case_InstructionRequest_Request {
- if x == nil {
- return InstructionRequest_Request_not_set_case
- }
- switch x.Request.(type) {
- case *InstructionRequest_ProcessBundle:
- return InstructionRequest_ProcessBundle_case
- case *InstructionRequest_ProcessBundleProgress:
- return InstructionRequest_ProcessBundleProgress_case
- case *InstructionRequest_ProcessBundleSplit:
- return InstructionRequest_ProcessBundleSplit_case
- case *InstructionRequest_FinalizeBundle:
- return InstructionRequest_FinalizeBundle_case
- case *InstructionRequest_MonitoringInfos:
- return InstructionRequest_MonitoringInfos_case
- case *InstructionRequest_HarnessMonitoringInfos:
- return InstructionRequest_HarnessMonitoringInfos_case
- case *InstructionRequest_SampleData:
- return InstructionRequest_SampleData_case
- case *InstructionRequest_Register:
- return InstructionRequest_Register_case
- default:
- return InstructionRequest_Request_not_set_case
- }
-}
-
-type InstructionRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) A unique identifier provided by the runner which represents
- // this requests execution. The InstructionResponse MUST have the matching id.
- InstructionId string
- // (Required) A request that the SDK Harness needs to interpret.
-
- // Fields of oneof Request:
- ProcessBundle *ProcessBundleRequest
- ProcessBundleProgress *ProcessBundleProgressRequest
- ProcessBundleSplit *ProcessBundleSplitRequest
- FinalizeBundle *FinalizeBundleRequest
- MonitoringInfos *MonitoringInfosMetadataRequest
- HarnessMonitoringInfos *HarnessMonitoringInfosRequest
- SampleData *SampleDataRequest
- // DEPRECATED
- Register *RegisterRequest
- // -- end of Request
-}
-
-func (b0 InstructionRequest_builder) Build() *InstructionRequest {
- m0 := &InstructionRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.InstructionId = b.InstructionId
- if b.ProcessBundle != nil {
- x.Request = &InstructionRequest_ProcessBundle{b.ProcessBundle}
- }
- if b.ProcessBundleProgress != nil {
- x.Request = &InstructionRequest_ProcessBundleProgress{b.ProcessBundleProgress}
- }
- if b.ProcessBundleSplit != nil {
- x.Request = &InstructionRequest_ProcessBundleSplit{b.ProcessBundleSplit}
- }
- if b.FinalizeBundle != nil {
- x.Request = &InstructionRequest_FinalizeBundle{b.FinalizeBundle}
- }
- if b.MonitoringInfos != nil {
- x.Request = &InstructionRequest_MonitoringInfos{b.MonitoringInfos}
- }
- if b.HarnessMonitoringInfos != nil {
- x.Request = &InstructionRequest_HarnessMonitoringInfos{b.HarnessMonitoringInfos}
- }
- if b.SampleData != nil {
- x.Request = &InstructionRequest_SampleData{b.SampleData}
- }
- if b.Register != nil {
- x.Request = &InstructionRequest_Register{b.Register}
- }
- return m0
-}
-
-type case_InstructionRequest_Request protoreflect.FieldNumber
-
-func (x case_InstructionRequest_Request) String() string {
- md := file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3].Descriptor()
- if x == 0 {
- return "not set"
- }
- return protoimpl.X.MessageFieldStringOf(md, protoreflect.FieldNumber(x))
-}
-
type isInstructionRequest_Request interface {
isInstructionRequest_Request()
}
@@ -879,7 +544,10 @@ func (*InstructionRequest_Register) isInstructionRequest_Request() {}
// The response for an associated request the SDK had been asked to fulfill.
// Stable
type InstructionResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) A reference provided by the runner which represents a requests
// execution. The InstructionResponse MUST have the matching id when
// responding to the runner.
@@ -891,7 +559,7 @@ type InstructionResponse struct {
// If the instruction did not fail, it is required to return an equivalent
// response type depending on the request this matches.
//
- // Types that are valid to be assigned to Response:
+ // Types that are assignable to Response:
//
// *InstructionResponse_ProcessBundle
// *InstructionResponse_ProcessBundleProgress
@@ -901,16 +569,16 @@ type InstructionResponse struct {
// *InstructionResponse_HarnessMonitoringInfos
// *InstructionResponse_SampleData
// *InstructionResponse_Register
- Response isInstructionResponse_Response `protobuf_oneof:"response"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ Response isInstructionResponse_Response `protobuf_oneof:"response"`
}
func (x *InstructionResponse) Reset() {
*x = InstructionResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *InstructionResponse) String() string {
@@ -921,7 +589,7 @@ func (*InstructionResponse) ProtoMessage() {}
func (x *InstructionResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -931,6 +599,11 @@ func (x *InstructionResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use InstructionResponse.ProtoReflect.Descriptor instead.
+func (*InstructionResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{4}
+}
+
func (x *InstructionResponse) GetInstructionId() string {
if x != nil {
return x.InstructionId
@@ -945,412 +618,95 @@ func (x *InstructionResponse) GetError() string {
return ""
}
-func (x *InstructionResponse) GetResponse() isInstructionResponse_Response {
- if x != nil {
- return x.Response
+func (m *InstructionResponse) GetResponse() isInstructionResponse_Response {
+ if m != nil {
+ return m.Response
}
return nil
}
func (x *InstructionResponse) GetProcessBundle() *ProcessBundleResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_ProcessBundle); ok {
- return x.ProcessBundle
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_ProcessBundle); ok {
+ return x.ProcessBundle
}
return nil
}
func (x *InstructionResponse) GetProcessBundleProgress() *ProcessBundleProgressResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_ProcessBundleProgress); ok {
- return x.ProcessBundleProgress
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_ProcessBundleProgress); ok {
+ return x.ProcessBundleProgress
}
return nil
}
func (x *InstructionResponse) GetProcessBundleSplit() *ProcessBundleSplitResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_ProcessBundleSplit); ok {
- return x.ProcessBundleSplit
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_ProcessBundleSplit); ok {
+ return x.ProcessBundleSplit
}
return nil
}
func (x *InstructionResponse) GetFinalizeBundle() *FinalizeBundleResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_FinalizeBundle); ok {
- return x.FinalizeBundle
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_FinalizeBundle); ok {
+ return x.FinalizeBundle
}
return nil
}
func (x *InstructionResponse) GetMonitoringInfos() *MonitoringInfosMetadataResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_MonitoringInfos); ok {
- return x.MonitoringInfos
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_MonitoringInfos); ok {
+ return x.MonitoringInfos
}
return nil
}
func (x *InstructionResponse) GetHarnessMonitoringInfos() *HarnessMonitoringInfosResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_HarnessMonitoringInfos); ok {
- return x.HarnessMonitoringInfos
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_HarnessMonitoringInfos); ok {
+ return x.HarnessMonitoringInfos
}
return nil
}
func (x *InstructionResponse) GetSampleData() *SampleDataResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_SampleData); ok {
- return x.SampleData
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_SampleData); ok {
+ return x.SampleData
}
return nil
}
func (x *InstructionResponse) GetRegister() *RegisterResponse {
- if x != nil {
- if x, ok := x.Response.(*InstructionResponse_Register); ok {
- return x.Register
- }
+ if x, ok := x.GetResponse().(*InstructionResponse_Register); ok {
+ return x.Register
}
return nil
}
-func (x *InstructionResponse) SetInstructionId(v string) {
- x.InstructionId = v
+type isInstructionResponse_Response interface {
+ isInstructionResponse_Response()
}
-func (x *InstructionResponse) SetError(v string) {
- x.Error = v
+type InstructionResponse_ProcessBundle struct {
+ ProcessBundle *ProcessBundleResponse `protobuf:"bytes,1001,opt,name=process_bundle,json=processBundle,proto3,oneof"`
}
-func (x *InstructionResponse) SetProcessBundle(v *ProcessBundleResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_ProcessBundle{v}
+type InstructionResponse_ProcessBundleProgress struct {
+ ProcessBundleProgress *ProcessBundleProgressResponse `protobuf:"bytes,1002,opt,name=process_bundle_progress,json=processBundleProgress,proto3,oneof"`
}
-func (x *InstructionResponse) SetProcessBundleProgress(v *ProcessBundleProgressResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_ProcessBundleProgress{v}
+type InstructionResponse_ProcessBundleSplit struct {
+ ProcessBundleSplit *ProcessBundleSplitResponse `protobuf:"bytes,1003,opt,name=process_bundle_split,json=processBundleSplit,proto3,oneof"`
}
-func (x *InstructionResponse) SetProcessBundleSplit(v *ProcessBundleSplitResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_ProcessBundleSplit{v}
+type InstructionResponse_FinalizeBundle struct {
+ FinalizeBundle *FinalizeBundleResponse `protobuf:"bytes,1004,opt,name=finalize_bundle,json=finalizeBundle,proto3,oneof"`
}
-func (x *InstructionResponse) SetFinalizeBundle(v *FinalizeBundleResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_FinalizeBundle{v}
+type InstructionResponse_MonitoringInfos struct {
+ MonitoringInfos *MonitoringInfosMetadataResponse `protobuf:"bytes,1005,opt,name=monitoring_infos,json=monitoringInfos,proto3,oneof"`
}
-func (x *InstructionResponse) SetMonitoringInfos(v *MonitoringInfosMetadataResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_MonitoringInfos{v}
-}
-
-func (x *InstructionResponse) SetHarnessMonitoringInfos(v *HarnessMonitoringInfosResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_HarnessMonitoringInfos{v}
-}
-
-func (x *InstructionResponse) SetSampleData(v *SampleDataResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_SampleData{v}
-}
-
-func (x *InstructionResponse) SetRegister(v *RegisterResponse) {
- if v == nil {
- x.Response = nil
- return
- }
- x.Response = &InstructionResponse_Register{v}
-}
-
-func (x *InstructionResponse) HasResponse() bool {
- if x == nil {
- return false
- }
- return x.Response != nil
-}
-
-func (x *InstructionResponse) HasProcessBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_ProcessBundle)
- return ok
-}
-
-func (x *InstructionResponse) HasProcessBundleProgress() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_ProcessBundleProgress)
- return ok
-}
-
-func (x *InstructionResponse) HasProcessBundleSplit() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_ProcessBundleSplit)
- return ok
-}
-
-func (x *InstructionResponse) HasFinalizeBundle() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_FinalizeBundle)
- return ok
-}
-
-func (x *InstructionResponse) HasMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_MonitoringInfos)
- return ok
-}
-
-func (x *InstructionResponse) HasHarnessMonitoringInfos() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_HarnessMonitoringInfos)
- return ok
-}
-
-func (x *InstructionResponse) HasSampleData() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_SampleData)
- return ok
-}
-
-func (x *InstructionResponse) HasRegister() bool {
- if x == nil {
- return false
- }
- _, ok := x.Response.(*InstructionResponse_Register)
- return ok
-}
-
-func (x *InstructionResponse) ClearResponse() {
- x.Response = nil
-}
-
-func (x *InstructionResponse) ClearProcessBundle() {
- if _, ok := x.Response.(*InstructionResponse_ProcessBundle); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearProcessBundleProgress() {
- if _, ok := x.Response.(*InstructionResponse_ProcessBundleProgress); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearProcessBundleSplit() {
- if _, ok := x.Response.(*InstructionResponse_ProcessBundleSplit); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearFinalizeBundle() {
- if _, ok := x.Response.(*InstructionResponse_FinalizeBundle); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearMonitoringInfos() {
- if _, ok := x.Response.(*InstructionResponse_MonitoringInfos); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearHarnessMonitoringInfos() {
- if _, ok := x.Response.(*InstructionResponse_HarnessMonitoringInfos); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearSampleData() {
- if _, ok := x.Response.(*InstructionResponse_SampleData); ok {
- x.Response = nil
- }
-}
-
-func (x *InstructionResponse) ClearRegister() {
- if _, ok := x.Response.(*InstructionResponse_Register); ok {
- x.Response = nil
- }
-}
-
-const InstructionResponse_Response_not_set_case case_InstructionResponse_Response = 0
-const InstructionResponse_ProcessBundle_case case_InstructionResponse_Response = 1001
-const InstructionResponse_ProcessBundleProgress_case case_InstructionResponse_Response = 1002
-const InstructionResponse_ProcessBundleSplit_case case_InstructionResponse_Response = 1003
-const InstructionResponse_FinalizeBundle_case case_InstructionResponse_Response = 1004
-const InstructionResponse_MonitoringInfos_case case_InstructionResponse_Response = 1005
-const InstructionResponse_HarnessMonitoringInfos_case case_InstructionResponse_Response = 1006
-const InstructionResponse_SampleData_case case_InstructionResponse_Response = 1007
-const InstructionResponse_Register_case case_InstructionResponse_Response = 1000
-
-func (x *InstructionResponse) WhichResponse() case_InstructionResponse_Response {
- if x == nil {
- return InstructionResponse_Response_not_set_case
- }
- switch x.Response.(type) {
- case *InstructionResponse_ProcessBundle:
- return InstructionResponse_ProcessBundle_case
- case *InstructionResponse_ProcessBundleProgress:
- return InstructionResponse_ProcessBundleProgress_case
- case *InstructionResponse_ProcessBundleSplit:
- return InstructionResponse_ProcessBundleSplit_case
- case *InstructionResponse_FinalizeBundle:
- return InstructionResponse_FinalizeBundle_case
- case *InstructionResponse_MonitoringInfos:
- return InstructionResponse_MonitoringInfos_case
- case *InstructionResponse_HarnessMonitoringInfos:
- return InstructionResponse_HarnessMonitoringInfos_case
- case *InstructionResponse_SampleData:
- return InstructionResponse_SampleData_case
- case *InstructionResponse_Register:
- return InstructionResponse_Register_case
- default:
- return InstructionResponse_Response_not_set_case
- }
-}
-
-type InstructionResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) A reference provided by the runner which represents a requests
- // execution. The InstructionResponse MUST have the matching id when
- // responding to the runner.
- InstructionId string
- // If this is specified, then this instruction has failed.
- // A human readable string representing the reason as to why processing has
- // failed.
- Error string
- // If the instruction did not fail, it is required to return an equivalent
- // response type depending on the request this matches.
-
- // Fields of oneof Response:
- ProcessBundle *ProcessBundleResponse
- ProcessBundleProgress *ProcessBundleProgressResponse
- ProcessBundleSplit *ProcessBundleSplitResponse
- FinalizeBundle *FinalizeBundleResponse
- MonitoringInfos *MonitoringInfosMetadataResponse
- HarnessMonitoringInfos *HarnessMonitoringInfosResponse
- SampleData *SampleDataResponse
- // DEPRECATED
- Register *RegisterResponse
- // -- end of Response
-}
-
-func (b0 InstructionResponse_builder) Build() *InstructionResponse {
- m0 := &InstructionResponse{}
- b, x := &b0, m0
- _, _ = b, x
- x.InstructionId = b.InstructionId
- x.Error = b.Error
- if b.ProcessBundle != nil {
- x.Response = &InstructionResponse_ProcessBundle{b.ProcessBundle}
- }
- if b.ProcessBundleProgress != nil {
- x.Response = &InstructionResponse_ProcessBundleProgress{b.ProcessBundleProgress}
- }
- if b.ProcessBundleSplit != nil {
- x.Response = &InstructionResponse_ProcessBundleSplit{b.ProcessBundleSplit}
- }
- if b.FinalizeBundle != nil {
- x.Response = &InstructionResponse_FinalizeBundle{b.FinalizeBundle}
- }
- if b.MonitoringInfos != nil {
- x.Response = &InstructionResponse_MonitoringInfos{b.MonitoringInfos}
- }
- if b.HarnessMonitoringInfos != nil {
- x.Response = &InstructionResponse_HarnessMonitoringInfos{b.HarnessMonitoringInfos}
- }
- if b.SampleData != nil {
- x.Response = &InstructionResponse_SampleData{b.SampleData}
- }
- if b.Register != nil {
- x.Response = &InstructionResponse_Register{b.Register}
- }
- return m0
-}
-
-type case_InstructionResponse_Response protoreflect.FieldNumber
-
-func (x case_InstructionResponse_Response) String() string {
- md := file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4].Descriptor()
- if x == 0 {
- return "not set"
- }
- return protoimpl.X.MessageFieldStringOf(md, protoreflect.FieldNumber(x))
-}
-
-type isInstructionResponse_Response interface {
- isInstructionResponse_Response()
-}
-
-type InstructionResponse_ProcessBundle struct {
- ProcessBundle *ProcessBundleResponse `protobuf:"bytes,1001,opt,name=process_bundle,json=processBundle,proto3,oneof"`
-}
-
-type InstructionResponse_ProcessBundleProgress struct {
- ProcessBundleProgress *ProcessBundleProgressResponse `protobuf:"bytes,1002,opt,name=process_bundle_progress,json=processBundleProgress,proto3,oneof"`
-}
-
-type InstructionResponse_ProcessBundleSplit struct {
- ProcessBundleSplit *ProcessBundleSplitResponse `protobuf:"bytes,1003,opt,name=process_bundle_split,json=processBundleSplit,proto3,oneof"`
-}
-
-type InstructionResponse_FinalizeBundle struct {
- FinalizeBundle *FinalizeBundleResponse `protobuf:"bytes,1004,opt,name=finalize_bundle,json=finalizeBundle,proto3,oneof"`
-}
-
-type InstructionResponse_MonitoringInfos struct {
- MonitoringInfos *MonitoringInfosMetadataResponse `protobuf:"bytes,1005,opt,name=monitoring_infos,json=monitoringInfos,proto3,oneof"`
-}
-
-type InstructionResponse_HarnessMonitoringInfos struct {
- HarnessMonitoringInfos *HarnessMonitoringInfosResponse `protobuf:"bytes,1006,opt,name=harness_monitoring_infos,json=harnessMonitoringInfos,proto3,oneof"`
+type InstructionResponse_HarnessMonitoringInfos struct {
+ HarnessMonitoringInfos *HarnessMonitoringInfosResponse `protobuf:"bytes,1006,opt,name=harness_monitoring_infos,json=harnessMonitoringInfos,proto3,oneof"`
}
type InstructionResponse_SampleData struct {
@@ -1383,18 +739,21 @@ func (*InstructionResponse_Register) isInstructionResponse_Response() {}
// "beam:protocol:data_sampling:v1" capability. Samples are taken only from the
// specified PCollection ids. An empty list will return everything.
type SampleDataRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Optional) The PCollection ids to filter for.
PcollectionIds []string `protobuf:"bytes,1,rep,name=pcollection_ids,json=pcollectionIds,proto3" json:"pcollection_ids,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
}
func (x *SampleDataRequest) Reset() {
*x = SampleDataRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *SampleDataRequest) String() string {
@@ -1405,7 +764,7 @@ func (*SampleDataRequest) ProtoMessage() {}
func (x *SampleDataRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1415,6 +774,11 @@ func (x *SampleDataRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use SampleDataRequest.ProtoReflect.Descriptor instead.
+func (*SampleDataRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{5}
+}
+
func (x *SampleDataRequest) GetPcollectionIds() []string {
if x != nil {
return x.PcollectionIds
@@ -1422,29 +786,13 @@ func (x *SampleDataRequest) GetPcollectionIds() []string {
return nil
}
-func (x *SampleDataRequest) SetPcollectionIds(v []string) {
- x.PcollectionIds = v
-}
-
-type SampleDataRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Optional) The PCollection ids to filter for.
- PcollectionIds []string
-}
-
-func (b0 SampleDataRequest_builder) Build() *SampleDataRequest {
- m0 := &SampleDataRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.PcollectionIds = b.PcollectionIds
- return m0
-}
-
// An element sampled when the SDK is processing a bundle. This is a proto
// message to allow for additional per-element metadata.
type SampledElement struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) Sampled raw bytes for an element. This is a
// single encoded element in the nested context.
Element []byte `protobuf:"bytes,1,opt,name=element,proto3" json:"element,omitempty"`
@@ -1452,16 +800,16 @@ type SampledElement struct {
SampleTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=sample_timestamp,json=sampleTimestamp,proto3" json:"sample_timestamp,omitempty"`
// (Optional) This will be set if this element was sampled because of a user
// exception.
- Exception *SampledElement_Exception `protobuf:"bytes,3,opt,name=exception,proto3" json:"exception,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ Exception *SampledElement_Exception `protobuf:"bytes,3,opt,name=exception,proto3" json:"exception,omitempty"`
}
func (x *SampledElement) Reset() {
*x = SampledElement{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *SampledElement) String() string {
@@ -1472,7 +820,7 @@ func (*SampledElement) ProtoMessage() {}
func (x *SampledElement) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1482,6 +830,11 @@ func (x *SampledElement) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use SampledElement.ProtoReflect.Descriptor instead.
+func (*SampledElement) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{6}
+}
+
func (x *SampledElement) GetElement() []byte {
if x != nil {
return x.Element
@@ -1503,81 +856,24 @@ func (x *SampledElement) GetException() *SampledElement_Exception {
return nil
}
-func (x *SampledElement) SetElement(v []byte) {
- if v == nil {
- v = []byte{}
- }
- x.Element = v
-}
-
-func (x *SampledElement) SetSampleTimestamp(v *timestamppb.Timestamp) {
- x.SampleTimestamp = v
-}
-
-func (x *SampledElement) SetException(v *SampledElement_Exception) {
- x.Exception = v
-}
-
-func (x *SampledElement) HasSampleTimestamp() bool {
- if x == nil {
- return false
- }
- return x.SampleTimestamp != nil
-}
-
-func (x *SampledElement) HasException() bool {
- if x == nil {
- return false
- }
- return x.Exception != nil
-}
-
-func (x *SampledElement) ClearSampleTimestamp() {
- x.SampleTimestamp = nil
-}
-
-func (x *SampledElement) ClearException() {
- x.Exception = nil
-}
-
-type SampledElement_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Required) Sampled raw bytes for an element. This is a
- // single encoded element in the nested context.
- Element []byte
- // (Required) Timestamp of when the sample was taken.
- SampleTimestamp *timestamppb.Timestamp
- // (Optional) This will be set if this element was sampled because of a user
- // exception.
- Exception *SampledElement_Exception
-}
-
-func (b0 SampledElement_builder) Build() *SampledElement {
- m0 := &SampledElement{}
- b, x := &b0, m0
- _, _ = b, x
- x.Element = b.Element
- x.SampleTimestamp = b.SampleTimestamp
- x.Exception = b.Exception
- return m0
-}
-
// If supported, the `SampleDataResponse` will contain samples from PCollections
// based upon the filters specified in the request.
type SampleDataResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// Map from PCollection id to sampled elements.
- ElementSamples map[string]*SampleDataResponse_ElementList `protobuf:"bytes,1,rep,name=element_samples,json=elementSamples,proto3" json:"element_samples,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ ElementSamples map[string]*SampleDataResponse_ElementList `protobuf:"bytes,1,rep,name=element_samples,json=elementSamples,proto3" json:"element_samples,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (x *SampleDataResponse) Reset() {
*x = SampleDataResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *SampleDataResponse) String() string {
@@ -1588,7 +884,7 @@ func (*SampleDataResponse) ProtoMessage() {}
func (x *SampleDataResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1598,6 +894,11 @@ func (x *SampleDataResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use SampleDataResponse.ProtoReflect.Descriptor instead.
+func (*SampleDataResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{7}
+}
+
func (x *SampleDataResponse) GetElementSamples() map[string]*SampleDataResponse_ElementList {
if x != nil {
return x.ElementSamples
@@ -1605,25 +906,6 @@ func (x *SampleDataResponse) GetElementSamples() map[string]*SampleDataResponse_
return nil
}
-func (x *SampleDataResponse) SetElementSamples(v map[string]*SampleDataResponse_ElementList) {
- x.ElementSamples = v
-}
-
-type SampleDataResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // Map from PCollection id to sampled elements.
- ElementSamples map[string]*SampleDataResponse_ElementList
-}
-
-func (b0 SampleDataResponse_builder) Build() *SampleDataResponse {
- m0 := &SampleDataResponse{}
- b, x := &b0, m0
- _, _ = b, x
- x.ElementSamples = b.ElementSamples
- return m0
-}
-
// A request to provide full MonitoringInfo associated with the entire SDK
// harness process, not specific to a bundle.
//
@@ -1637,16 +919,18 @@ func (b0 SampleDataResponse_builder) Build() *SampleDataResponse {
// as the MonitoringInfo could be reconstructed fully by overwriting its
// payload field with the bytes specified here.
type HarnessMonitoringInfosRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- unknownFields protoimpl.UnknownFields
+ state protoimpl.MessageState
sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
}
func (x *HarnessMonitoringInfosRequest) Reset() {
*x = HarnessMonitoringInfosRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *HarnessMonitoringInfosRequest) String() string {
@@ -1657,7 +941,7 @@ func (*HarnessMonitoringInfosRequest) ProtoMessage() {}
func (x *HarnessMonitoringInfosRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1667,20 +951,16 @@ func (x *HarnessMonitoringInfosRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-type HarnessMonitoringInfosRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
-}
-
-func (b0 HarnessMonitoringInfosRequest_builder) Build() *HarnessMonitoringInfosRequest {
- m0 := &HarnessMonitoringInfosRequest{}
- b, x := &b0, m0
- _, _ = b, x
- return m0
+// Deprecated: Use HarnessMonitoringInfosRequest.ProtoReflect.Descriptor instead.
+func (*HarnessMonitoringInfosRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{8}
}
type HarnessMonitoringInfosResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// An identifier to MonitoringInfo.payload mapping containing
// Metrics associated with the SDK harness, not a specific bundle.
//
@@ -1693,16 +973,16 @@ type HarnessMonitoringInfosResponse struct {
// for the lifetime of the associated control connection as long
// as the MonitoringInfo could be reconstructed fully by overwriting its
// payload field with the bytes specified here.
- MonitoringData map[string][]byte `protobuf:"bytes,1,rep,name=monitoring_data,json=monitoringData,proto3" json:"monitoring_data,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ MonitoringData map[string][]byte `protobuf:"bytes,1,rep,name=monitoring_data,json=monitoringData,proto3" json:"monitoring_data,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (x *HarnessMonitoringInfosResponse) Reset() {
*x = HarnessMonitoringInfosResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *HarnessMonitoringInfosResponse) String() string {
@@ -1713,7 +993,7 @@ func (*HarnessMonitoringInfosResponse) ProtoMessage() {}
func (x *HarnessMonitoringInfosResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1723,6 +1003,11 @@ func (x *HarnessMonitoringInfosResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use HarnessMonitoringInfosResponse.ProtoReflect.Descriptor instead.
+func (*HarnessMonitoringInfosResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{9}
+}
+
func (x *HarnessMonitoringInfosResponse) GetMonitoringData() map[string][]byte {
if x != nil {
return x.MonitoringData
@@ -1730,52 +1015,25 @@ func (x *HarnessMonitoringInfosResponse) GetMonitoringData() map[string][]byte {
return nil
}
-func (x *HarnessMonitoringInfosResponse) SetMonitoringData(v map[string][]byte) {
- x.MonitoringData = v
-}
-
-type HarnessMonitoringInfosResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // An identifier to MonitoringInfo.payload mapping containing
- // Metrics associated with the SDK harness, not a specific bundle.
- //
- // An SDK can report metrics using an identifier that only contains the
- // associated payload. A runner who wants to receive the full metrics
- // information can request all the monitoring metadata via a
- // MonitoringInfosMetadataRequest providing a list of ids as necessary.
- //
- // The SDK is allowed to reuse the identifiers
- // for the lifetime of the associated control connection as long
- // as the MonitoringInfo could be reconstructed fully by overwriting its
- // payload field with the bytes specified here.
- MonitoringData map[string][]byte
-}
-
-func (b0 HarnessMonitoringInfosResponse_builder) Build() *HarnessMonitoringInfosResponse {
- m0 := &HarnessMonitoringInfosResponse{}
- b, x := &b0, m0
- _, _ = b, x
- x.MonitoringData = b.MonitoringData
- return m0
-}
-
// A list of objects which can be referred to by the runner in
// future requests.
// Stable
type RegisterRequest struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Optional) The set of descriptors used to process bundles.
ProcessBundleDescriptor []*ProcessBundleDescriptor `protobuf:"bytes,1,rep,name=process_bundle_descriptor,json=processBundleDescriptor,proto3" json:"process_bundle_descriptor,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
}
func (x *RegisterRequest) Reset() {
*x = RegisterRequest{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *RegisterRequest) String() string {
@@ -1786,7 +1044,7 @@ func (*RegisterRequest) ProtoMessage() {}
func (x *RegisterRequest) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1796,6 +1054,11 @@ func (x *RegisterRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use RegisterRequest.ProtoReflect.Descriptor instead.
+func (*RegisterRequest) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{10}
+}
+
func (x *RegisterRequest) GetProcessBundleDescriptor() []*ProcessBundleDescriptor {
if x != nil {
return x.ProcessBundleDescriptor
@@ -1803,37 +1066,20 @@ func (x *RegisterRequest) GetProcessBundleDescriptor() []*ProcessBundleDescripto
return nil
}
-func (x *RegisterRequest) SetProcessBundleDescriptor(v []*ProcessBundleDescriptor) {
- x.ProcessBundleDescriptor = v
-}
-
-type RegisterRequest_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
- // (Optional) The set of descriptors used to process bundles.
- ProcessBundleDescriptor []*ProcessBundleDescriptor
-}
-
-func (b0 RegisterRequest_builder) Build() *RegisterRequest {
- m0 := &RegisterRequest{}
- b, x := &b0, m0
- _, _ = b, x
- x.ProcessBundleDescriptor = b.ProcessBundleDescriptor
- return m0
-}
-
// Stable
type RegisterResponse struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
- unknownFields protoimpl.UnknownFields
+ state protoimpl.MessageState
sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
}
func (x *RegisterResponse) Reset() {
*x = RegisterResponse{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *RegisterResponse) String() string {
@@ -1844,7 +1090,7 @@ func (*RegisterResponse) ProtoMessage() {}
func (x *RegisterResponse) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1854,34 +1100,30 @@ func (x *RegisterResponse) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-type RegisterResponse_builder struct {
- _ [0]func() // Prevents comparability and use of unkeyed literals for the builder.
-
-}
-
-func (b0 RegisterResponse_builder) Build() *RegisterResponse {
- m0 := &RegisterResponse{}
- b, x := &b0, m0
- _, _ = b, x
- return m0
+// Deprecated: Use RegisterResponse.ProtoReflect.Descriptor instead.
+func (*RegisterResponse) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{11}
}
// Definitions that should be used to construct the bundle processing graph.
type ProcessBundleDescriptor struct {
- state protoimpl.MessageState `protogen:"hybrid.v1"`
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
// (Required) A pipeline level unique id which can be used as a reference to
// refer to this.
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
// (Required) A map from pipeline-scoped id to PTransform.
- Transforms map[string]*pipeline_v1.PTransform `protobuf:"bytes,2,rep,name=transforms,proto3" json:"transforms,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Transforms map[string]*pipeline_v1.PTransform `protobuf:"bytes,2,rep,name=transforms,proto3" json:"transforms,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to PCollection.
- Pcollections map[string]*pipeline_v1.PCollection `protobuf:"bytes,3,rep,name=pcollections,proto3" json:"pcollections,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Pcollections map[string]*pipeline_v1.PCollection `protobuf:"bytes,3,rep,name=pcollections,proto3" json:"pcollections,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to WindowingStrategy.
- WindowingStrategies map[string]*pipeline_v1.WindowingStrategy `protobuf:"bytes,4,rep,name=windowing_strategies,json=windowingStrategies,proto3" json:"windowing_strategies,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ WindowingStrategies map[string]*pipeline_v1.WindowingStrategy `protobuf:"bytes,4,rep,name=windowing_strategies,json=windowingStrategies,proto3" json:"windowing_strategies,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to Coder.
- Coders map[string]*pipeline_v1.Coder `protobuf:"bytes,5,rep,name=coders,proto3" json:"coders,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Coders map[string]*pipeline_v1.Coder `protobuf:"bytes,5,rep,name=coders,proto3" json:"coders,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// (Required) A map from pipeline-scoped id to Environment.
- Environments map[string]*pipeline_v1.Environment `protobuf:"bytes,6,rep,name=environments,proto3" json:"environments,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+ Environments map[string]*pipeline_v1.Environment `protobuf:"bytes,6,rep,name=environments,proto3" json:"environments,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
// A descriptor describing the end point to use for State API
// calls. Required if the Runner intends to send remote references over the
// data plane or if any of the transforms rely on user state or side inputs.
@@ -1889,15 +1131,15 @@ type ProcessBundleDescriptor struct {
// A descriptor describing the end point to use for Data API for user timers.
// Required if the ProcessBundleDescriptor contains any transforms that have user timers.
TimerApiServiceDescriptor *pipeline_v1.ApiServiceDescriptor `protobuf:"bytes,8,opt,name=timer_api_service_descriptor,json=timerApiServiceDescriptor,proto3" json:"timer_api_service_descriptor,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
}
func (x *ProcessBundleDescriptor) Reset() {
*x = ProcessBundleDescriptor{}
- mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
+ if protoimpl.UnsafeEnabled {
+ mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
}
func (x *ProcessBundleDescriptor) String() string {
@@ -1908,7 +1150,7 @@ func (*ProcessBundleDescriptor) ProtoMessage() {}
func (x *ProcessBundleDescriptor) ProtoReflect() protoreflect.Message {
mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12]
- if x != nil {
+ if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -1918,6 +1160,11 @@ func (x *ProcessBundleDescriptor) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
+// Deprecated: Use ProcessBundleDescriptor.ProtoReflect.Descriptor instead.
+func (*ProcessBundleDescriptor) Descriptor() ([]byte, []int) {
+ return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{12}
+}
+
func (x *ProcessBundleDescriptor) GetId() string {
if x != nil {
return x.Id
@@ -1974,106 +1221,15 @@ func (x *ProcessBundleDescriptor) GetTimerApiServiceDescriptor() *pipeline_v1.Ap
return nil
}
-func (x *ProcessBundleDescriptor) SetId(v string) {
- x.Id = v
-}
-
-func (x *ProcessBundleDescriptor) SetTransforms(v map[string]*pipeline_v1.PTransform) {
- x.Transforms = v
-}
-
... 71248 lines suppressed ...