You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2019/12/03 23:30:14 UTC
[beam] branch revert-9959-job-service-state-history created (now
b143752)
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a change to branch revert-9959-job-service-state-history
in repository https://gitbox.apache.org/repos/asf/beam.git.
at b143752 Revert "[BEAM-8523] JobAPI: Give access to timestamped state change history"
This branch includes the following new commits:
new b143752 Revert "[BEAM-8523] JobAPI: Give access to timestamped state change history"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[beam] 01/01: Revert "[BEAM-8523] JobAPI: Give access to
timestamped state change history"
Posted by lo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch revert-9959-job-service-state-history
in repository https://gitbox.apache.org/repos/asf/beam.git
commit b143752eea0469f0cc26eab15bf863f0d39167c4
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Tue Dec 3 15:30:02 2019 -0800
Revert "[BEAM-8523] JobAPI: Give access to timestamped state change history"
---
.../src/main/proto/beam_job_api.proto | 10 +--
.../jobsubmission/InMemoryJobService.java | 33 ++++----
.../fnexecution/jobsubmission/JobInvocation.java | 33 ++------
.../portability/JobServicePipelineResult.java | 6 +-
.../portability/testing/TestJobService.java | 7 +-
.../beam/model/jobmanagement_v1/beam_job_api.pb.go | 48 +++++------
sdks/python/apache_beam/metrics/cells.py | 13 ++-
.../runners/portability/abstract_job_service.py | 67 ++-------------
.../portability/flink_uber_jar_job_server.py | 50 ++++-------
.../portability/flink_uber_jar_job_server_test.py | 33 +++-----
.../runners/portability/local_job_service.py | 63 +++++++-------
.../runners/portability/local_job_service_test.py | 98 ----------------------
sdks/python/apache_beam/utils/proto_utils.py | 16 ----
13 files changed, 139 insertions(+), 338 deletions(-)
diff --git a/model/job-management/src/main/proto/beam_job_api.proto b/model/job-management/src/main/proto/beam_job_api.proto
index 6a72f5d..d297d3b 100644
--- a/model/job-management/src/main/proto/beam_job_api.proto
+++ b/model/job-management/src/main/proto/beam_job_api.proto
@@ -32,7 +32,6 @@ option java_outer_classname = "JobApi";
import "beam_runner_api.proto";
import "endpoints.proto";
import "google/protobuf/struct.proto";
-import "google/protobuf/timestamp.proto";
import "metrics.proto";
// Job Service for running RunnerAPI pipelines
@@ -48,7 +47,7 @@ service JobService {
rpc GetJobs (GetJobsRequest) returns (GetJobsResponse);
// Get the current state of the job
- rpc GetState (GetJobStateRequest) returns (JobStateEvent);
+ rpc GetState (GetJobStateRequest) returns (GetJobStateResponse);
// Get the job's pipeline
rpc GetPipeline (GetJobPipelineRequest) returns (GetJobPipelineResponse);
@@ -57,7 +56,7 @@ service JobService {
rpc Cancel (CancelJobRequest) returns (CancelJobResponse);
// Subscribe to a stream of state changes of the job, will immediately return the current state of the job as the first response.
- rpc GetStateStream (GetJobStateRequest) returns (stream JobStateEvent);
+ rpc GetStateStream (GetJobStateRequest) returns (stream GetJobStateResponse);
// Subscribe to a stream of state changes and messages from the job
rpc GetMessageStream (JobMessagesRequest) returns (stream JobMessagesResponse);
@@ -152,9 +151,8 @@ message GetJobStateRequest {
}
-message JobStateEvent {
+message GetJobStateResponse {
JobState.Enum state = 1; // (required)
- google.protobuf.Timestamp timestamp = 2; // (required)
}
@@ -198,7 +196,7 @@ message JobMessage {
message JobMessagesResponse {
oneof response {
JobMessage message_response = 1;
- JobStateEvent state_response = 2;
+ GetJobStateResponse state_response = 2;
}
}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/InMemoryJobService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/InMemoryJobService.java
index 59a2647..9ef3f06 100644
--- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/InMemoryJobService.java
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/InMemoryJobService.java
@@ -32,6 +32,7 @@ import org.apache.beam.model.jobmanagement.v1.JobApi.DescribePipelineOptionsResp
import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobPipelineRequest;
import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobPipelineResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateRequest;
+import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobsRequest;
import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobsResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobInfo;
@@ -39,7 +40,6 @@ import org.apache.beam.model.jobmanagement.v1.JobApi.JobMessage;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobMessagesRequest;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobMessagesResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobState;
-import org.apache.beam.model.jobmanagement.v1.JobApi.JobStateEvent;
import org.apache.beam.model.jobmanagement.v1.JobApi.PrepareJobRequest;
import org.apache.beam.model.jobmanagement.v1.JobApi.PrepareJobResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.RunJobRequest;
@@ -190,8 +190,8 @@ public class InMemoryJobService extends JobServiceGrpc.JobServiceImplBase implem
String invocationId = invocation.getId();
invocation.addStateListener(
- event -> {
- if (!JobInvocation.isTerminated(event.getState())) {
+ state -> {
+ if (!JobInvocation.isTerminated(state)) {
return;
}
String stagingSessionToken = stagingSessionTokens.get(preparationId);
@@ -243,12 +243,14 @@ public class InMemoryJobService extends JobServiceGrpc.JobServiceImplBase implem
}
@Override
- public void getState(GetJobStateRequest request, StreamObserver<JobStateEvent> responseObserver) {
+ public void getState(
+ GetJobStateRequest request, StreamObserver<GetJobStateResponse> responseObserver) {
LOG.trace("{} {}", GetJobStateRequest.class.getSimpleName(), request);
String invocationId = request.getJobId();
try {
JobInvocation invocation = getInvocation(invocationId);
- JobStateEvent response = invocation.getStateEvent();
+ JobState.Enum state = invocation.getState();
+ GetJobStateResponse response = GetJobStateResponse.newBuilder().setState(state).build();
responseObserver.onNext(response);
responseObserver.onCompleted();
} catch (StatusRuntimeException | StatusException e) {
@@ -306,16 +308,15 @@ public class InMemoryJobService extends JobServiceGrpc.JobServiceImplBase implem
@Override
public void getStateStream(
- GetJobStateRequest request, StreamObserver<JobStateEvent> responseObserver) {
+ GetJobStateRequest request, StreamObserver<GetJobStateResponse> responseObserver) {
LOG.trace("{} {}", GetJobStateRequest.class.getSimpleName(), request);
String invocationId = request.getJobId();
try {
JobInvocation invocation = getInvocation(invocationId);
-
- Consumer<JobStateEvent> stateListener =
- event -> {
- responseObserver.onNext(event);
- if (JobInvocation.isTerminated(event.getState())) {
+ Consumer<JobState.Enum> stateListener =
+ state -> {
+ responseObserver.onNext(GetJobStateResponse.newBuilder().setState(state).build());
+ if (JobInvocation.isTerminated(state)) {
responseObserver.onCompleted();
}
};
@@ -340,11 +341,13 @@ public class InMemoryJobService extends JobServiceGrpc.JobServiceImplBase implem
// and message listener.
StreamObserver<JobMessagesResponse> syncResponseObserver =
SynchronizedStreamObserver.wrapping(responseObserver);
- Consumer<JobStateEvent> stateListener =
- event -> {
+ Consumer<JobState.Enum> stateListener =
+ state -> {
syncResponseObserver.onNext(
- JobMessagesResponse.newBuilder().setStateResponse(event).build());
- if (JobInvocation.isTerminated(event.getState())) {
+ JobMessagesResponse.newBuilder()
+ .setStateResponse(GetJobStateResponse.newBuilder().setState(state).build())
+ .build());
+ if (JobInvocation.isTerminated(state)) {
responseObserver.onCompleted();
}
};
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/JobInvocation.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/JobInvocation.java
index 2da0592..12acfa1 100644
--- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/JobInvocation.java
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/JobInvocation.java
@@ -31,12 +31,10 @@ import org.apache.beam.model.jobmanagement.v1.JobApi;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobMessage;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobState;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobState.Enum;
-import org.apache.beam.model.jobmanagement.v1.JobApi.JobStateEvent;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.model.pipeline.v1.RunnerApi.Pipeline;
import org.apache.beam.runners.fnexecution.provisioning.JobInfo;
import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.util.Timestamps;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.FutureCallback;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListenableFuture;
@@ -53,12 +51,12 @@ public class JobInvocation {
private final PortablePipelineRunner pipelineRunner;
private final JobInfo jobInfo;
private final ListeningExecutorService executorService;
- private List<Consumer<JobStateEvent>> stateObservers;
+ private List<Consumer<Enum>> stateObservers;
private List<Consumer<JobMessage>> messageObservers;
+ private JobState.Enum jobState;
private JobApi.MetricResults metrics;
private PortablePipelineResult resultHandle;
@Nullable private ListenableFuture<PortablePipelineResult> invocationFuture;
- private List<JobStateEvent> stateHistory;
public JobInvocation(
JobInfo jobInfo,
@@ -72,9 +70,8 @@ public class JobInvocation {
this.stateObservers = new ArrayList<>();
this.messageObservers = new ArrayList<>();
this.invocationFuture = null;
- this.stateHistory = new ArrayList<>();
+ this.jobState = JobState.Enum.STOPPED;
this.metrics = JobApi.MetricResults.newBuilder().build();
- this.setState(JobState.Enum.STOPPED);
}
private PortablePipelineResult runPipeline() throws Exception {
@@ -194,12 +191,7 @@ public class JobInvocation {
/** Retrieve the job's current state. */
public JobState.Enum getState() {
- return getStateEvent().getState();
- }
-
- /** Retrieve the job's current state. */
- public JobStateEvent getStateEvent() {
- return stateHistory.get(stateHistory.size() - 1);
+ return this.jobState;
}
/** Retrieve the job's pipeline. */
@@ -208,10 +200,8 @@ public class JobInvocation {
}
/** Listen for job state changes with a {@link Consumer}. */
- public synchronized void addStateListener(Consumer<JobStateEvent> stateStreamObserver) {
- for (JobStateEvent event : stateHistory) {
- stateStreamObserver.accept(event);
- }
+ public synchronized void addStateListener(Consumer<JobState.Enum> stateStreamObserver) {
+ stateStreamObserver.accept(getState());
stateObservers.add(stateStreamObserver);
}
@@ -231,14 +221,9 @@ public class JobInvocation {
}
private synchronized void setState(JobState.Enum state) {
- JobStateEvent event =
- JobStateEvent.newBuilder()
- .setState(state)
- .setTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))
- .build();
- this.stateHistory.add(event);
- for (Consumer<JobStateEvent> observer : stateObservers) {
- observer.accept(event);
+ this.jobState = state;
+ for (Consumer<JobState.Enum> observer : stateObservers) {
+ observer.accept(state);
}
}
diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/JobServicePipelineResult.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/JobServicePipelineResult.java
index bcfa321..9097712 100644
--- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/JobServicePipelineResult.java
+++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/JobServicePipelineResult.java
@@ -26,7 +26,7 @@ import org.apache.beam.model.jobmanagement.v1.JobApi;
import org.apache.beam.model.jobmanagement.v1.JobApi.CancelJobRequest;
import org.apache.beam.model.jobmanagement.v1.JobApi.CancelJobResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateRequest;
-import org.apache.beam.model.jobmanagement.v1.JobApi.JobStateEvent;
+import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateResponse;
import org.apache.beam.model.jobmanagement.v1.JobServiceGrpc.JobServiceBlockingStub;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.MetricResults;
@@ -61,7 +61,7 @@ class JobServicePipelineResult implements PipelineResult, AutoCloseable {
return terminationState;
}
JobServiceBlockingStub stub = jobService.get();
- JobStateEvent response =
+ GetJobStateResponse response =
stub.getState(GetJobStateRequest.newBuilder().setJobIdBytes(jobId).build());
return getJavaState(response.getState());
}
@@ -103,7 +103,7 @@ class JobServicePipelineResult implements PipelineResult, AutoCloseable {
}
JobServiceBlockingStub stub = jobService.get();
GetJobStateRequest request = GetJobStateRequest.newBuilder().setJobIdBytes(jobId).build();
- JobStateEvent response = stub.getState(request);
+ GetJobStateResponse response = stub.getState(request);
State lastState = getJavaState(response.getState());
while (!lastState.isTerminal()) {
try {
diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestJobService.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestJobService.java
index 76b8974..f8d6899 100644
--- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestJobService.java
+++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestJobService.java
@@ -19,8 +19,8 @@ package org.apache.beam.runners.portability.testing;
import org.apache.beam.model.jobmanagement.v1.JobApi;
import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateRequest;
+import org.apache.beam.model.jobmanagement.v1.JobApi.GetJobStateResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.JobState;
-import org.apache.beam.model.jobmanagement.v1.JobApi.JobStateEvent;
import org.apache.beam.model.jobmanagement.v1.JobApi.PrepareJobRequest;
import org.apache.beam.model.jobmanagement.v1.JobApi.PrepareJobResponse;
import org.apache.beam.model.jobmanagement.v1.JobApi.RunJobRequest;
@@ -75,8 +75,9 @@ public class TestJobService extends JobServiceImplBase {
}
@Override
- public void getState(GetJobStateRequest request, StreamObserver<JobStateEvent> responseObserver) {
- responseObserver.onNext(JobStateEvent.newBuilder().setState(jobState).build());
+ public void getState(
+ GetJobStateRequest request, StreamObserver<GetJobStateResponse> responseObserver) {
+ responseObserver.onNext(GetJobStateResponse.newBuilder().setState(jobState).build());
responseObserver.onCompleted();
}
diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go
index f843ecc..661c75f 100644
--- a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go
+++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go
@@ -625,38 +625,38 @@ func (m *GetJobStateRequest) GetJobId() string {
return ""
}
-type JobStateEvent struct {
+type GetJobStateResponse struct {
State JobState_Enum `protobuf:"varint,1,opt,name=state,proto3,enum=org.apache.beam.model.job_management.v1.JobState_Enum" json:"state,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
-func (m *JobStateEvent) Reset() { *m = JobStateEvent{} }
-func (m *JobStateEvent) String() string { return proto.CompactTextString(m) }
-func (*JobStateEvent) ProtoMessage() {}
-func (*JobStateEvent) Descriptor() ([]byte, []int) {
+func (m *GetJobStateResponse) Reset() { *m = GetJobStateResponse{} }
+func (m *GetJobStateResponse) String() string { return proto.CompactTextString(m) }
+func (*GetJobStateResponse) ProtoMessage() {}
+func (*GetJobStateResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_beam_job_api_cf64c696c499a6a1, []int{10}
}
-func (m *JobStateEvent) XXX_Unmarshal(b []byte) error {
+func (m *GetJobStateResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetJobStateResponse.Unmarshal(m, b)
}
-func (m *JobStateEvent) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+func (m *GetJobStateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GetJobStateResponse.Marshal(b, m, deterministic)
}
-func (dst *JobStateEvent) XXX_Merge(src proto.Message) {
+func (dst *GetJobStateResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_GetJobStateResponse.Merge(dst, src)
}
-func (m *JobStateEvent) XXX_Size() int {
+func (m *GetJobStateResponse) XXX_Size() int {
return xxx_messageInfo_GetJobStateResponse.Size(m)
}
-func (m *JobStateEvent) XXX_DiscardUnknown() {
+func (m *GetJobStateResponse) XXX_DiscardUnknown() {
xxx_messageInfo_GetJobStateResponse.DiscardUnknown(m)
}
var xxx_messageInfo_GetJobStateResponse proto.InternalMessageInfo
-func (m *JobStateEvent) GetState() JobState_Enum {
+func (m *GetJobStateResponse) GetState() JobState_Enum {
if m != nil {
return m.State
}
@@ -888,7 +888,7 @@ type JobMessagesResponse_MessageResponse struct {
MessageResponse *JobMessage `protobuf:"bytes,1,opt,name=message_response,json=messageResponse,proto3,oneof"`
}
type JobMessagesResponse_StateResponse struct {
- StateResponse *JobStateEvent `protobuf:"bytes,2,opt,name=state_response,json=stateResponse,proto3,oneof"`
+ StateResponse *GetJobStateResponse `protobuf:"bytes,2,opt,name=state_response,json=stateResponse,proto3,oneof"`
}
func (*JobMessagesResponse_MessageResponse) isJobMessagesResponse_Response() {}
@@ -908,7 +908,7 @@ func (m *JobMessagesResponse) GetMessageResponse() *JobMessage {
return nil
}
-func (m *JobMessagesResponse) GetStateResponse() *JobStateEvent {
+func (m *JobMessagesResponse) GetStateResponse() *GetJobStateResponse {
if x, ok := m.GetResponse().(*JobMessagesResponse_StateResponse); ok {
return x.StateResponse
}
@@ -959,7 +959,7 @@ func _JobMessagesResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *
if wire != proto.WireBytes {
return true, proto.ErrInternalBadWireType
}
- msg := new(JobStateEvent)
+ msg := new(GetJobStateResponse)
err := b.DecodeMessage(msg)
m.Response = &JobMessagesResponse_StateResponse{msg}
return true, err
@@ -1345,7 +1345,7 @@ func init() {
proto.RegisterType((*GetJobsRequest)(nil), "org.apache.beam.model.job_management.v1.GetJobsRequest")
proto.RegisterType((*GetJobsResponse)(nil), "org.apache.beam.model.job_management.v1.GetJobsResponse")
proto.RegisterType((*GetJobStateRequest)(nil), "org.apache.beam.model.job_management.v1.GetJobStateRequest")
- proto.RegisterType((*JobStateEvent)(nil), "org.apache.beam.model.job_management.v1.JobStateEvent")
+ proto.RegisterType((*GetJobStateResponse)(nil), "org.apache.beam.model.job_management.v1.GetJobStateResponse")
proto.RegisterType((*GetJobPipelineRequest)(nil), "org.apache.beam.model.job_management.v1.GetJobPipelineRequest")
proto.RegisterType((*GetJobPipelineResponse)(nil), "org.apache.beam.model.job_management.v1.GetJobPipelineResponse")
proto.RegisterType((*JobMessagesRequest)(nil), "org.apache.beam.model.job_management.v1.JobMessagesRequest")
@@ -1384,7 +1384,7 @@ type JobServiceClient interface {
// Get a list of all invoked jobs
GetJobs(ctx context.Context, in *GetJobsRequest, opts ...grpc.CallOption) (*GetJobsResponse, error)
// Get the current state of the job
- GetState(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (*JobStateEvent, error)
+ GetState(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (*GetJobStateResponse, error)
// Get the job's pipeline
GetPipeline(ctx context.Context, in *GetJobPipelineRequest, opts ...grpc.CallOption) (*GetJobPipelineResponse, error)
// Cancel the job
@@ -1434,8 +1434,8 @@ func (c *jobServiceClient) GetJobs(ctx context.Context, in *GetJobsRequest, opts
return out, nil
}
-func (c *jobServiceClient) GetState(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (*JobStateEvent, error) {
- out := new(JobStateEvent)
+func (c *jobServiceClient) GetState(ctx context.Context, in *GetJobStateRequest, opts ...grpc.CallOption) (*GetJobStateResponse, error) {
+ out := new(GetJobStateResponse)
err := c.cc.Invoke(ctx, "/org.apache.beam.model.job_management.v1.JobService/GetState", in, out, opts...)
if err != nil {
return nil, err
@@ -1477,7 +1477,7 @@ func (c *jobServiceClient) GetStateStream(ctx context.Context, in *GetJobStateRe
}
type JobService_GetStateStreamClient interface {
- Recv() (*JobStateEvent, error)
+ Recv() (*GetJobStateResponse, error)
grpc.ClientStream
}
@@ -1485,8 +1485,8 @@ type jobServiceGetStateStreamClient struct {
grpc.ClientStream
}
-func (x *jobServiceGetStateStreamClient) Recv() (*JobStateEvent, error) {
- m := new(JobStateEvent)
+func (x *jobServiceGetStateStreamClient) Recv() (*GetJobStateResponse, error) {
+ m := new(GetJobStateResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
@@ -1553,7 +1553,7 @@ type JobServiceServer interface {
// Get a list of all invoked jobs
GetJobs(context.Context, *GetJobsRequest) (*GetJobsResponse, error)
// Get the current state of the job
- GetState(context.Context, *GetJobStateRequest) (*JobStateEvent, error)
+ GetState(context.Context, *GetJobStateRequest) (*GetJobStateResponse, error)
// Get the job's pipeline
GetPipeline(context.Context, *GetJobPipelineRequest) (*GetJobPipelineResponse, error)
// Cancel the job
@@ -1689,7 +1689,7 @@ func _JobService_GetStateStream_Handler(srv interface{}, stream grpc.ServerStrea
}
type JobService_GetStateStreamServer interface {
- Send(*JobStateEvent) error
+ Send(*GetJobStateResponse) error
grpc.ServerStream
}
@@ -1697,7 +1697,7 @@ type jobServiceGetStateStreamServer struct {
grpc.ServerStream
}
-func (x *jobServiceGetStateStreamServer) Send(m *JobStateEvent) error {
+func (x *jobServiceGetStateStreamServer) Send(m *GetJobStateResponse) error {
return x.ServerStream.SendMsg(m)
}
diff --git a/sdks/python/apache_beam/metrics/cells.py b/sdks/python/apache_beam/metrics/cells.py
index 103be44..1df52d9 100644
--- a/sdks/python/apache_beam/metrics/cells.py
+++ b/sdks/python/apache_beam/metrics/cells.py
@@ -28,9 +28,10 @@ import threading
import time
from builtins import object
+from google.protobuf import timestamp_pb2
+
from apache_beam.portability.api import beam_fn_api_pb2
from apache_beam.portability.api import metrics_pb2
-from apache_beam.utils import proto_utils
try:
import cython
@@ -363,13 +364,17 @@ class GaugeData(object):
return GaugeData(value, timestamp=timestamp)
def to_runner_api(self):
+ seconds = int(self.timestamp)
+ nanos = int((self.timestamp - seconds) * 10**9)
+ gauge_timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)
return beam_fn_api_pb2.Metrics.User.GaugeData(
- value=self.value, timestamp=proto_utils.to_Timestamp(self.timestamp))
+ value=self.value, timestamp=gauge_timestamp)
@staticmethod
def from_runner_api(proto):
- return GaugeData(proto.value,
- timestamp=proto_utils.from_Timestamp(proto.timestamp))
+ gauge_timestamp = (proto.timestamp.seconds +
+ float(proto.timestamp.nanos) / 10**9)
+ return GaugeData(proto.value, timestamp=gauge_timestamp)
def to_runner_api_monitoring_info(self):
"""Returns a Metric with this value for use in a MonitoringInfo."""
diff --git a/sdks/python/apache_beam/runners/portability/abstract_job_service.py b/sdks/python/apache_beam/runners/portability/abstract_job_service.py
index b86d0f5..24309fc 100644
--- a/sdks/python/apache_beam/runners/portability/abstract_job_service.py
+++ b/sdks/python/apache_beam/runners/portability/abstract_job_service.py
@@ -16,34 +16,16 @@
#
from __future__ import absolute_import
-import itertools
import logging
import uuid
from builtins import object
-from google.protobuf import timestamp_pb2
-
from apache_beam.portability.api import beam_job_api_pb2
from apache_beam.portability.api import beam_job_api_pb2_grpc
-from apache_beam.utils.timestamp import Timestamp
_LOGGER = logging.getLogger(__name__)
-def make_state_event(state, timestamp):
- if isinstance(timestamp, Timestamp):
- proto_timestamp = timestamp.to_proto()
- elif isinstance(timestamp, timestamp_pb2.Timestamp):
- proto_timestamp = timestamp
- else:
- raise ValueError("Expected apache_beam.utils.timestamp.Timestamp, "
- "or google.protobuf.timestamp_pb2.Timestamp. "
- "Got %s" % type(timestamp))
-
- return beam_job_api_pb2.JobStateEvent(
- state=state, timestamp=proto_timestamp)
-
-
class AbstractJobServiceServicer(beam_job_api_pb2_grpc.JobServiceServicer):
"""Manages one or more pipelines, possibly concurrently.
Experimental: No backward compatibility guaranteed.
@@ -81,10 +63,10 @@ class AbstractJobServiceServicer(beam_job_api_pb2_grpc.JobServiceServicer):
def GetJobs(self, request, context=None, timeout=None):
return beam_job_api_pb2.GetJobsResponse(
- job_info=[job.to_runner_api() for job in self._jobs.values()])
+ [job.to_runner_api() for job in self._jobs.values()])
def GetState(self, request, context=None):
- return beam_job_api_pb2.JobStateEvent(
+ return beam_job_api_pb2.GetJobStateResponse(
state=self._jobs[request.job_id].get_state())
def GetPipeline(self, request, context=None, timeout=None):
@@ -93,7 +75,7 @@ class AbstractJobServiceServicer(beam_job_api_pb2_grpc.JobServiceServicer):
def Cancel(self, request, context=None, timeout=None):
self._jobs[request.job_id].cancel()
- return beam_job_api_pb2.CancelJobResponse(
+ return beam_job_api_pb2.CancelJobRequest(
state=self._jobs[request.job_id].get_state())
def GetStateStream(self, request, context=None, timeout=None):
@@ -103,8 +85,8 @@ class AbstractJobServiceServicer(beam_job_api_pb2_grpc.JobServiceServicer):
raise LookupError("Job {} does not exist".format(request.job_id))
job = self._jobs[request.job_id]
- for state, timestamp in job.get_state_stream():
- yield make_state_event(state, timestamp)
+ for state in job.get_state_stream():
+ yield beam_job_api_pb2.GetJobStateResponse(state=state)
def GetMessageStream(self, request, context=None, timeout=None):
"""Yields messages since the stream started.
@@ -114,9 +96,9 @@ class AbstractJobServiceServicer(beam_job_api_pb2_grpc.JobServiceServicer):
job = self._jobs[request.job_id]
for msg in job.get_message_stream():
- if isinstance(msg, tuple):
+ if isinstance(msg, int):
resp = beam_job_api_pb2.JobMessagesResponse(
- state_response=make_state_event(*msg))
+ state_response=beam_job_api_pb2.GetJobStateResponse(state=msg))
else:
resp = beam_job_api_pb2.JobMessagesResponse(message_response=msg)
yield resp
@@ -133,44 +115,13 @@ class AbstractBeamJob(object):
self._job_name = job_name
self._pipeline_proto = pipeline
self._pipeline_options = options
- self._state_history = [(beam_job_api_pb2.JobState.STOPPED,
- Timestamp.now())]
def _to_implement(self):
raise NotImplementedError(self)
prepare = run = cancel = _to_implement
artifact_staging_endpoint = _to_implement
- get_state_stream = get_message_stream = _to_implement
-
- @property
- def state(self):
- """Get the latest state enum."""
- return self.get_state()[0]
-
- def get_state(self):
- """Get a tuple of the latest state and its timestamp."""
- # this is safe: initial state is set in __init__
- return self._state_history[-1]
-
- def set_state(self, new_state):
- """Set the latest state as an int enum and update the state history.
-
- :param new_state: int
- latest state enum
- :return: Timestamp or None
- the new timestamp if the state has not changed, else None
- """
- if new_state != self._state_history[-1][0]:
- timestamp = Timestamp.now()
- self._state_history.append((new_state, timestamp))
- return timestamp
- else:
- return None
-
- def with_state_history(self, state_stream):
- """Utility to prepend recorded state history to an active state stream"""
- return itertools.chain(self._state_history[:], state_stream)
+ get_state = get_state_stream = get_message_stream = _to_implement
def get_pipeline(self):
return self._pipeline_proto
@@ -185,4 +136,4 @@ class AbstractBeamJob(object):
job_id=self._job_id,
job_name=self._job_name,
pipeline_options=self._pipeline_options,
- state=self.state)
+ state=self.get_state())
diff --git a/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py b/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py
index c6348cf..c67edc4 100644
--- a/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py
+++ b/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py
@@ -207,12 +207,7 @@ class FlinkBeamJob(abstract_job_service.AbstractBeamJob):
_LOGGER.info(
'Error deleting jar %s' % self._flink_jar_id, exc_info=True)
- def _get_state(self):
- """Query flink to get the current state.
-
- :return: tuple of int and Timestamp or None
- timestamp will be None if the state has not changed since the last query.
- """
+ def get_state(self):
# For just getting the status, execution-result seems cheaper.
flink_status = self.get(
'v1/jobs/%s/execution-result' % self._flink_job_id)['status']['id']
@@ -223,7 +218,7 @@ class FlinkBeamJob(abstract_job_service.AbstractBeamJob):
'RUNNING': beam_job_api_pb2.JobState.RUNNING,
'FAILING': beam_job_api_pb2.JobState.RUNNING,
'FAILED': beam_job_api_pb2.JobState.FAILED,
- 'CANCELLING': beam_job_api_pb2.JobState.CANCELLING,
+ 'CANCELLING': beam_job_api_pb2.JobState.RUNNING,
'CANCELED': beam_job_api_pb2.JobState.CANCELLED,
'FINISHED': beam_job_api_pb2.JobState.DONE,
'RESTARTING': beam_job_api_pb2.JobState.RUNNING,
@@ -234,35 +229,21 @@ class FlinkBeamJob(abstract_job_service.AbstractBeamJob):
}.get(flink_status, beam_job_api_pb2.JobState.UNSPECIFIED)
if self.is_terminal_state(beam_state):
self.delete_jar()
- # update the state history if it has changed
- return beam_state, self.set_state(beam_state)
-
- def get_state(self):
- state, timestamp = self._get_state()
- if timestamp is None:
- # state has not changed since it was last checked: use previous timestamp
- return super(FlinkBeamJob, self).get_state()
- else:
- return state, timestamp
+ return beam_state
def get_state_stream(self):
- def _state_iter():
- sleep_secs = 1.0
- while True:
- current_state, timestamp = self._get_state()
- if timestamp is not None:
- # non-None indicates that the state has changed
- yield current_state, timestamp
- sleep_secs = min(60, sleep_secs * 1.2)
- time.sleep(sleep_secs)
-
- for state, timestamp in self.with_state_history(_state_iter()):
- yield state, timestamp
- if self.is_terminal_state(state):
- break
+ sleep_secs = 1.0
+ current_state = self.get_state()
+ yield current_state
+ while not self.is_terminal_state(current_state):
+ sleep_secs = min(60, sleep_secs * 1.2)
+ time.sleep(sleep_secs)
+ previous_state, current_state = current_state, self.get_state()
+ if previous_state != current_state:
+ yield current_state
def get_message_stream(self):
- for state, timestamp in self.get_state_stream():
+ for state in self.get_state_stream():
if self.is_terminal_state(state):
response = self.get('v1/jobs/%s/exceptions' % self._flink_job_id)
for ix, exc in enumerate(response['all-exceptions']):
@@ -272,7 +253,4 @@ class FlinkBeamJob(abstract_job_service.AbstractBeamJob):
importance=
beam_job_api_pb2.JobMessage.MessageImportance.JOB_MESSAGE_ERROR,
message_text=exc['exception'])
- yield state, timestamp
- break
- else:
- yield state, timestamp
+ yield state
diff --git a/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server_test.py b/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server_test.py
index 2c24cbd..dba328e 100644
--- a/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server_test.py
+++ b/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server_test.py
@@ -110,12 +110,9 @@ class FlinkUberJarJobServerTest(unittest.TestCase):
state_stream = job_server.GetStateStream(
beam_job_api_pb2.GetJobStateRequest(
job_id=prepare_response.preparation_id))
-
self.assertEqual(
[s.state for s in state_stream],
- [beam_job_api_pb2.JobState.STOPPED,
- beam_job_api_pb2.JobState.RUNNING,
- beam_job_api_pb2.JobState.DONE])
+ [beam_job_api_pb2.JobState.RUNNING, beam_job_api_pb2.JobState.DONE])
http_mock.get(
'http://flink/v1/jobs/some_job_id/exceptions',
@@ -123,25 +120,19 @@ class FlinkUberJarJobServerTest(unittest.TestCase):
message_stream = job_server.GetMessageStream(
beam_job_api_pb2.JobMessagesRequest(
job_id=prepare_response.preparation_id))
-
- def get_item(x):
- if x.HasField('message_response'):
- return x.message_response
- else:
- return x.state_response.state
-
self.assertEqual(
- [get_item(m) for m in message_stream],
+ [m for m in message_stream],
[
- beam_job_api_pb2.JobState.STOPPED,
- beam_job_api_pb2.JobState.RUNNING,
- beam_job_api_pb2.JobMessage(
- message_id='message0',
- time='0',
- importance=beam_job_api_pb2.JobMessage.MessageImportance
- .JOB_MESSAGE_ERROR,
- message_text='exc_text'),
- beam_job_api_pb2.JobState.DONE,
+ beam_job_api_pb2.JobMessagesResponse(
+ message_response=beam_job_api_pb2.JobMessage(
+ message_id='message0',
+ time='0',
+ importance=beam_job_api_pb2.JobMessage.MessageImportance
+ .JOB_MESSAGE_ERROR,
+ message_text='exc_text')),
+ beam_job_api_pb2.JobMessagesResponse(
+ state_response=beam_job_api_pb2.GetJobStateResponse(
+ state=beam_job_api_pb2.JobState.DONE)),
])
diff --git a/sdks/python/apache_beam/runners/portability/local_job_service.py b/sdks/python/apache_beam/runners/portability/local_job_service.py
index b97f683..c6d5619 100644
--- a/sdks/python/apache_beam/runners/portability/local_job_service.py
+++ b/sdks/python/apache_beam/runners/portability/local_job_service.py
@@ -46,11 +46,6 @@ from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor
_LOGGER = logging.getLogger(__name__)
-def _iter_queue(q):
- while True:
- yield q.get(block=True)
-
-
class LocalJobServicer(abstract_job_service.AbstractJobServiceServicer):
"""Manages one or more pipelines, possibly concurrently.
Experimental: No backward compatibility guaranteed.
@@ -199,18 +194,26 @@ class BeamJob(abstract_job_service.AbstractBeamJob):
job_id, provision_info.provision_info.job_name, pipeline, options)
self._provision_info = provision_info
self._artifact_staging_endpoint = artifact_staging_endpoint
+ self._state = None
self._state_queues = []
self._log_queues = []
+ self.state = beam_job_api_pb2.JobState.STOPPED
self.daemon = True
self.result = None
- def set_state(self, new_state):
- """Set the latest state as an int enum and notify consumers"""
- timestamp = super(BeamJob, self).set_state(new_state)
- if timestamp is not None:
- # Inform consumers of the new state.
- for queue in self._state_queues:
- queue.put((new_state, timestamp))
+ @property
+ def state(self):
+ return self._state
+
+ @state.setter
+ def state(self, new_state):
+ # Inform consumers of the new state.
+ for queue in self._state_queues:
+ queue.put(new_state)
+ self._state = new_state
+
+ def get_state(self):
+ return self.state
def prepare(self):
pass
@@ -219,40 +222,42 @@ class BeamJob(abstract_job_service.AbstractBeamJob):
return self._artifact_staging_endpoint
def run(self):
- self.set_state(beam_job_api_pb2.JobState.STARTING)
+ self.state = beam_job_api_pb2.JobState.STARTING
self._run_thread = threading.Thread(target=self._run_job)
self._run_thread.start()
def _run_job(self):
- self.set_state(beam_job_api_pb2.JobState.RUNNING)
+ self.state = beam_job_api_pb2.JobState.RUNNING
with JobLogHandler(self._log_queues):
try:
result = fn_api_runner.FnApiRunner(
provision_info=self._provision_info).run_via_runner_api(
self._pipeline_proto)
_LOGGER.info('Successfully completed job.')
- self.set_state(beam_job_api_pb2.JobState.DONE)
+ self.state = beam_job_api_pb2.JobState.DONE
self.result = result
except: # pylint: disable=bare-except
_LOGGER.exception('Error running pipeline.')
_LOGGER.exception(traceback)
- self.set_state(beam_job_api_pb2.JobState.FAILED)
+ self.state = beam_job_api_pb2.JobState.FAILED
raise
def cancel(self):
if not self.is_terminal_state(self.state):
- self.set_state(beam_job_api_pb2.JobState.CANCELLING)
+ self.state = beam_job_api_pb2.JobState.CANCELLING
# TODO(robertwb): Actually cancel...
- self.set_state(beam_job_api_pb2.JobState.CANCELLED)
+ self.state = beam_job_api_pb2.JobState.CANCELLED
def get_state_stream(self):
# Register for any new state changes.
state_queue = queue.Queue()
self._state_queues.append(state_queue)
- for state, timestamp in self.with_state_history(_iter_queue(state_queue)):
- yield state, timestamp
- if self.is_terminal_state(state):
+ yield self.state
+ while True:
+ current_state = state_queue.get(block=True)
+ yield current_state
+ if self.is_terminal_state(current_state):
break
def get_message_stream(self):
@@ -261,15 +266,13 @@ class BeamJob(abstract_job_service.AbstractBeamJob):
self._log_queues.append(log_queue)
self._state_queues.append(log_queue)
- for msg in self.with_state_history(_iter_queue(log_queue)):
- if isinstance(msg, tuple):
- assert len(msg) == 2 and isinstance(msg[0], int)
- current_state = msg[0]
- yield msg
- if self.is_terminal_state(current_state):
- break
- else:
- yield msg
+ current_state = self.state
+ yield current_state
+ while not self.is_terminal_state(current_state):
+ msg = log_queue.get(block=True)
+ yield msg
+ if isinstance(msg, int):
+ current_state = msg
class BeamFnLoggingServicer(beam_fn_api_pb2_grpc.BeamFnLoggingServicer):
diff --git a/sdks/python/apache_beam/runners/portability/local_job_service_test.py b/sdks/python/apache_beam/runners/portability/local_job_service_test.py
deleted file mode 100644
index 475d2fe..0000000
--- a/sdks/python/apache_beam/runners/portability/local_job_service_test.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-from __future__ import absolute_import
-from __future__ import print_function
-
-import itertools
-import logging
-import unittest
-
-import grpc
-
-from apache_beam.portability.api import beam_artifact_api_pb2
-from apache_beam.portability.api import beam_artifact_api_pb2_grpc
-from apache_beam.portability.api import beam_job_api_pb2
-from apache_beam.portability.api import beam_runner_api_pb2
-from apache_beam.runners.portability import local_job_service
-
-
-def increment_iter(iter):
- return itertools.chain([next(iter)], iter)
-
-
-class LocalJobServerTest(unittest.TestCase):
-
- def test_end_to_end(self):
-
- job_service = local_job_service.LocalJobServicer()
- job_service.start_grpc_server()
-
- # this logic is taken roughly from PortableRunner.run_pipeline()
-
- # Prepare the job.
- prepare_response = job_service.Prepare(
- beam_job_api_pb2.PrepareJobRequest(
- job_name='job',
- pipeline=beam_runner_api_pb2.Pipeline()))
- channel = grpc.insecure_channel(
- prepare_response.artifact_staging_endpoint.url)
- retrieval_token = beam_artifact_api_pb2_grpc.ArtifactStagingServiceStub(
- channel).CommitManifest(
- beam_artifact_api_pb2.CommitManifestRequest(
- staging_session_token=prepare_response.staging_session_token,
- manifest=beam_artifact_api_pb2.Manifest())
- ).retrieval_token
- channel.close()
-
- state_stream = job_service.GetStateStream(
- beam_job_api_pb2.GetJobStateRequest(
- job_id=prepare_response.preparation_id))
- # If there's an error, we don't always get it until we try to read.
- # Fortunately, there's always an immediate current state published.
- # state_results.append(next(state_stream))
- state_stream = increment_iter(state_stream)
-
- message_stream = job_service.GetMessageStream(
- beam_job_api_pb2.JobMessagesRequest(
- job_id=prepare_response.preparation_id))
-
- job_service.Run(
- beam_job_api_pb2.RunJobRequest(
- preparation_id=prepare_response.preparation_id,
- retrieval_token=retrieval_token))
-
- state_results = list(state_stream)
- message_results = list(message_stream)
-
- expected_states = [
- beam_job_api_pb2.JobState.STOPPED,
- beam_job_api_pb2.JobState.STARTING,
- beam_job_api_pb2.JobState.RUNNING,
- beam_job_api_pb2.JobState.DONE,
- ]
- self.assertEqual(
- [s.state for s in state_results],
- expected_states)
-
- self.assertEqual(
- [s.state_response.state for s in message_results],
- expected_states)
-
-
-if __name__ == '__main__':
- logging.getLogger().setLevel(logging.INFO)
- unittest.main()
diff --git a/sdks/python/apache_beam/utils/proto_utils.py b/sdks/python/apache_beam/utils/proto_utils.py
index 79464c2..9a76448 100644
--- a/sdks/python/apache_beam/utils/proto_utils.py
+++ b/sdks/python/apache_beam/utils/proto_utils.py
@@ -18,11 +18,9 @@
"""For internal use only; no backwards-compatibility guarantees."""
from __future__ import absolute_import
-from __future__ import division
from google.protobuf import any_pb2
from google.protobuf import struct_pb2
-from google.protobuf import timestamp_pb2
def pack_Any(msg):
@@ -74,17 +72,3 @@ def from_micros(cls, micros):
result = cls()
result.FromMicroseconds(micros)
return result
-
-
-def to_Timestamp(time):
- """Convert a float returned by time.time() to a Timestamp.
- """
- seconds = int(time)
- nanos = int((time - seconds) * 10**9)
- return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)
-
-
-def from_Timestamp(timestamp):
- """Convert a Timestamp to a float expressed as seconds since the epoch.
- """
- return timestamp.seconds + float(timestamp.nanos) / 10**9