You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2019/05/17 22:40:53 UTC
[beam] branch master updated: Updates python SDK GCP client API in
sync with the current Google internal definition. (#8554)
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a1cb61a Updates python SDK GCP client API in sync with the current Google internal definition. (#8554)
a1cb61a is described below
commit a1cb61ab63acff2c3d93f166d59f4789d45ecc11
Author: slavachernyak <ch...@google.com>
AuthorDate: Fri May 17 15:40:41 2019 -0700
Updates python SDK GCP client API in sync with the current Google internal definition. (#8554)
---
.../clients/dataflow/dataflow_v1b3_client.py | 218 +++++++++++++++++++++
.../clients/dataflow/dataflow_v1b3_messages.py | 182 ++++++++++++++++-
2 files changed, 391 insertions(+), 9 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
index 9a2d4d5..061a60b 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
@@ -64,8 +64,11 @@ class DataflowV1b3(base_api.BaseApiClient):
self.projects_locations_jobs_messages = self.ProjectsLocationsJobsMessagesService(self)
self.projects_locations_jobs_workItems = self.ProjectsLocationsJobsWorkItemsService(self)
self.projects_locations_jobs = self.ProjectsLocationsJobsService(self)
+ self.projects_locations_snapshots = self.ProjectsLocationsSnapshotsService(self)
+ self.projects_locations_sql = self.ProjectsLocationsSqlService(self)
self.projects_locations_templates = self.ProjectsLocationsTemplatesService(self)
self.projects_locations = self.ProjectsLocationsService(self)
+ self.projects_snapshots = self.ProjectsSnapshotsService(self)
self.projects_templates = self.ProjectsTemplatesService(self)
self.projects = self.ProjectsService(self)
@@ -821,6 +824,133 @@ of jobs that are running in `us-central1`.
supports_download=False,
)
+ class ProjectsLocationsSnapshotsService(base_api.BaseApiService):
+ """Service class for the projects_locations_snapshots resource."""
+
+ _NAME = u'projects_locations_snapshots'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsSnapshotsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Delete(self, request, global_params=None):
+ r"""Deletes a snapshot.
+
+ Args:
+ request: (DataflowProjectsLocationsSnapshotsDeleteRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (DeleteSnapshotResponse) The response message.
+ """
+ config = self.GetMethodConfig('Delete')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Delete.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'DELETE',
+ method_id=u'dataflow.projects.locations.snapshots.delete',
+ ordered_params=[u'projectId', u'location', u'snapshotId'],
+ path_params=[u'location', u'projectId', u'snapshotId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/snapshots/{snapshotId}',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsSnapshotsDeleteRequest',
+ response_type_name=u'DeleteSnapshotResponse',
+ supports_download=False,
+ )
+
+ def Get(self, request, global_params=None):
+ r"""Gets information about a snapshot.
+
+ Args:
+ request: (DataflowProjectsLocationsSnapshotsGetRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Snapshot) The response message.
+ """
+ config = self.GetMethodConfig('Get')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Get.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.snapshots.get',
+ ordered_params=[u'projectId', u'location', u'snapshotId'],
+ path_params=[u'location', u'projectId', u'snapshotId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/snapshots/{snapshotId}',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsSnapshotsGetRequest',
+ response_type_name=u'Snapshot',
+ supports_download=False,
+ )
+
+ def List(self, request, global_params=None):
+ r"""Lists snapshots.
+
+ Args:
+ request: (DataflowProjectsLocationsSnapshotsListRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ListSnapshotsResponse) The response message.
+ """
+ config = self.GetMethodConfig('List')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ List.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.snapshots.list',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/snapshots',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsSnapshotsListRequest',
+ response_type_name=u'ListSnapshotsResponse',
+ supports_download=False,
+ )
+
+ class ProjectsLocationsSqlService(base_api.BaseApiService):
+ """Service class for the projects_locations_sql resource."""
+
+ _NAME = u'projects_locations_sql'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsSqlService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Validate(self, request, global_params=None):
+ r"""Validates a GoogleSQL query for Cloud Dataflow syntax. Will always.
+confirm the given query parses correctly, and if able to look up
+schema information from DataCatalog, will validate that the query
+analyzes properly as well.
+
+ Args:
+ request: (DataflowProjectsLocationsSqlValidateRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ValidateResponse) The response message.
+ """
+ config = self.GetMethodConfig('Validate')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Validate.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.sql.validate',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[u'query'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/sql:validate',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsSqlValidateRequest',
+ response_type_name=u'ValidateResponse',
+ supports_download=False,
+ )
+
class ProjectsLocationsTemplatesService(base_api.BaseApiService):
"""Service class for the projects_locations_templates resource."""
@@ -945,6 +1075,68 @@ of jobs that are running in `us-central1`.
supports_download=False,
)
+ class ProjectsSnapshotsService(base_api.BaseApiService):
+ """Service class for the projects_snapshots resource."""
+
+ _NAME = u'projects_snapshots'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsSnapshotsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Get(self, request, global_params=None):
+ r"""Gets information about a snapshot.
+
+ Args:
+ request: (DataflowProjectsSnapshotsGetRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Snapshot) The response message.
+ """
+ config = self.GetMethodConfig('Get')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Get.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.snapshots.get',
+ ordered_params=[u'projectId', u'snapshotId'],
+ path_params=[u'projectId', u'snapshotId'],
+ query_params=[u'location'],
+ relative_path=u'v1b3/projects/{projectId}/snapshots/{snapshotId}',
+ request_field='',
+ request_type_name=u'DataflowProjectsSnapshotsGetRequest',
+ response_type_name=u'Snapshot',
+ supports_download=False,
+ )
+
+ def List(self, request, global_params=None):
+ r"""Lists snapshots.
+
+ Args:
+ request: (DataflowProjectsSnapshotsListRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ListSnapshotsResponse) The response message.
+ """
+ config = self.GetMethodConfig('List')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ List.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.snapshots.list',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[u'location'],
+ relative_path=u'v1b3/projects/{projectId}/snapshots',
+ request_field='',
+ request_type_name=u'DataflowProjectsSnapshotsListRequest',
+ response_type_name=u'ListSnapshotsResponse',
+ supports_download=False,
+ )
+
class ProjectsTemplatesService(base_api.BaseApiService):
"""Service class for the projects_templates resource."""
@@ -1043,6 +1235,32 @@ of jobs that are running in `us-central1`.
self._upload_configs = {
}
+ def DeleteSnapshots(self, request, global_params=None):
+ r"""Deletes a snapshot.
+
+ Args:
+ request: (DataflowProjectsDeleteSnapshotsRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (DeleteSnapshotResponse) The response message.
+ """
+ config = self.GetMethodConfig('DeleteSnapshots')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ DeleteSnapshots.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'DELETE',
+ method_id=u'dataflow.projects.deleteSnapshots',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[u'location', u'snapshotId'],
+ relative_path=u'v1b3/projects/{projectId}/snapshots',
+ request_field='',
+ request_type_name=u'DataflowProjectsDeleteSnapshotsRequest',
+ response_type_name=u'DeleteSnapshotResponse',
+ supports_download=False,
+ )
+
def WorkerMessages(self, request, global_params=None):
r"""Send a worker_message to the service.
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
index 5d8fdae..021f394 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
@@ -575,6 +575,21 @@ class DataDiskAssignment(_messages.Message):
vmInstance = _messages.StringField(2)
+class DataflowProjectsDeleteSnapshotsRequest(_messages.Message):
+ r"""A DataflowProjectsDeleteSnapshotsRequest object.
+
+ Fields:
+ location: The location that contains this snapshot.
+ projectId: The ID of the Cloud Platform project that the snapshot belongs
+ to.
+ snapshotId: The ID of the snapshot.
+ """
+
+ location = _messages.StringField(1)
+ projectId = _messages.StringField(2, required=True)
+ snapshotId = _messages.StringField(3)
+
+
class DataflowProjectsJobsAggregatedRequest(_messages.Message):
r"""A DataflowProjectsJobsAggregatedRequest object.
@@ -1251,6 +1266,65 @@ class DataflowProjectsLocationsJobsWorkItemsReportStatusRequest(_messages.Messag
reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 4)
+class DataflowProjectsLocationsSnapshotsDeleteRequest(_messages.Message):
+ r"""A DataflowProjectsLocationsSnapshotsDeleteRequest object.
+
+ Fields:
+ location: The location that contains this snapshot.
+ projectId: The ID of the Cloud Platform project that the snapshot belongs
+ to.
+ snapshotId: The ID of the snapshot.
+ """
+
+ location = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ snapshotId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsLocationsSnapshotsGetRequest(_messages.Message):
+ r"""A DataflowProjectsLocationsSnapshotsGetRequest object.
+
+ Fields:
+ location: The location that contains this snapshot.
+ projectId: The ID of the Cloud Platform project that the snapshot belongs
+ to.
+ snapshotId: The ID of the snapshot.
+ """
+
+ location = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ snapshotId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsLocationsSnapshotsListRequest(_messages.Message):
+ r"""A DataflowProjectsLocationsSnapshotsListRequest object.
+
+ Fields:
+ location: The location to list snapshots in.
+ projectId: The project ID to list snapshots for.
+ """
+
+ location = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+
+
+class DataflowProjectsLocationsSqlValidateRequest(_messages.Message):
+ r"""A DataflowProjectsLocationsSqlValidateRequest object.
+
+ Fields:
+ location: The [regional endpoint]
+ (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
+ which to direct the request.
+ projectId: Required. The ID of the Cloud Platform project that the job
+ belongs to.
+ query: The sql query to validate.
+ """
+
+ location = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ query = _messages.StringField(3)
+
+
class DataflowProjectsLocationsTemplatesCreateRequest(_messages.Message):
r"""A DataflowProjectsLocationsTemplatesCreateRequest object.
@@ -1347,6 +1421,33 @@ class DataflowProjectsLocationsWorkerMessagesRequest(_messages.Message):
sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 3)
+class DataflowProjectsSnapshotsGetRequest(_messages.Message):
+ r"""A DataflowProjectsSnapshotsGetRequest object.
+
+ Fields:
+ location: The location that contains this snapshot.
+ projectId: The ID of the Cloud Platform project that the snapshot belongs
+ to.
+ snapshotId: The ID of the snapshot.
+ """
+
+ location = _messages.StringField(1)
+ projectId = _messages.StringField(2, required=True)
+ snapshotId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsSnapshotsListRequest(_messages.Message):
+ r"""A DataflowProjectsSnapshotsListRequest object.
+
+ Fields:
+ location: The location to list snapshots in.
+ projectId: The project ID to list snapshots for.
+ """
+
+ location = _messages.StringField(1)
+ projectId = _messages.StringField(2, required=True)
+
+
class DataflowProjectsTemplatesCreateRequest(_messages.Message):
r"""A DataflowProjectsTemplatesCreateRequest object.
@@ -1447,6 +1548,10 @@ class DatastoreIODetails(_messages.Message):
projectId = _messages.StringField(2)
+class DeleteSnapshotResponse(_messages.Message):
+ r"""Response from deleting a snapshot."""
+
+
class DerivedSource(_messages.Message):
r"""Specification of one of the bundles produced as a result of splitting a
Source (e.g. when executing a SourceSplitRequest, or when splitting an
@@ -2849,6 +2954,16 @@ class ListJobsResponse(_messages.Message):
nextPageToken = _messages.StringField(3)
+class ListSnapshotsResponse(_messages.Message):
+ r"""List of snapshots.
+
+ Fields:
+ snapshots: Returned snapshots.
+ """
+
+ snapshots = _messages.MessageField('Snapshot', 1, repeated=True)
+
+
class MapTask(_messages.Message):
r"""MapTask consists of an ordered set of instructions, each of which
describes one particular low-level operation for the worker to perform in
@@ -3507,11 +3622,17 @@ class RuntimeEnvironment(_messages.Message):
r"""The environment values to set at runtime.
Messages:
- AdditionalUserLabelsValue: Additional user labels attached to the job.
+ AdditionalUserLabelsValue: Additional user labels to be specified for the
+ job. Keys and values should follow the restrictions specified in the
+ [labeling restrictions](https://cloud.google.com/compute/docs/labeling-
+ resources#restrictions) page.
Fields:
additionalExperiments: Additional experiment flags for the job.
- additionalUserLabels: Additional user labels attached to the job.
+ additionalUserLabels: Additional user labels to be specified for the job.
+ Keys and values should follow the restrictions specified in the
+ [labeling restrictions](https://cloud.google.com/compute/docs/labeling-
+ resources#restrictions) page.
bypassTempDirValidation: Whether to bypass the safety checks for the job's
temporary directory. Use with caution.
machineType: The machine type to use for the job. Defaults to the value
@@ -3535,7 +3656,10 @@ class RuntimeEnvironment(_messages.Message):
@encoding.MapUnrecognizedFields('additionalProperties')
class AdditionalUserLabelsValue(_messages.Message):
- r"""Additional user labels attached to the job.
+ r"""Additional user labels to be specified for the job. Keys and values
+ should follow the restrictions specified in the [labeling
+ restrictions](https://cloud.google.com/compute/docs/labeling-
+ resources#restrictions) page.
Messages:
AdditionalProperty: An additional property for a
@@ -3841,19 +3965,43 @@ class Sink(_messages.Message):
class Snapshot(_messages.Message):
r"""Represents a snapshot of a job.
+ Enums:
+ StateValueValuesEnum: State of the snapshot.
+
Fields:
creationTime: The time this snapshot was created.
id: The unique ID of this snapshot.
projectId: The project this snapshot belongs to.
sourceJobId: The job this snapshot was created from.
+ state: State of the snapshot.
ttl: The time after which this snapshot will be automatically deleted.
"""
+ class StateValueValuesEnum(_messages.Enum):
+ r"""State of the snapshot.
+
+ Values:
+ UNKNOWN_SNAPSHOT_STATE: Unknown state.
+ PENDING: Snapshot intent to create has been persisted, snapshotting of
+ state has not yet started.
+ RUNNING: Snapshotting is being performed.
+ READY: Snapshot has been created and is ready to be used.
+ FAILED: Snapshot failed to be created.
+ DELETED: Snapshot has been deleted.
+ """
+ UNKNOWN_SNAPSHOT_STATE = 0
+ PENDING = 1
+ RUNNING = 2
+ READY = 3
+ FAILED = 4
+ DELETED = 5
+
creationTime = _messages.StringField(1)
id = _messages.StringField(2)
projectId = _messages.StringField(3)
sourceJobId = _messages.StringField(4)
- ttl = _messages.StringField(5)
+ state = _messages.EnumField('StateValueValuesEnum', 5)
+ ttl = _messages.StringField(6)
class SnapshotJobRequest(_messages.Message):
@@ -4888,6 +5036,16 @@ class TransformSummary(_messages.Message):
outputCollectionName = _messages.StringField(6, repeated=True)
+class ValidateResponse(_messages.Message):
+ r"""Response to the validation request.
+
+ Fields:
+ errorMessage: Will be empty if validation succeeds.
+ """
+
+ errorMessage = _messages.StringField(1)
+
+
class WorkItem(_messages.Message):
r"""WorkItem represents basic information about a WorkItem to be executed in
the cloud.
@@ -5079,6 +5237,7 @@ class WorkerHealthReport(_messages.Message):
PodsValueListEntry: A PodsValueListEntry object.
Fields:
+ msg: A message describing any unusual health reports.
pods: The pods running on the worker. See: http://kubernetes.io/v1.1/docs
/api-reference/v1/definitions.html#_v1_pod This field is used by the
worker to send the status of the indvidual containers running on each
@@ -5086,7 +5245,10 @@ class WorkerHealthReport(_messages.Message):
reportInterval: The interval at which the worker is sending health
reports. The default value of 0 should be interpreted as the field is
not being explicitly set by the worker.
- vmIsHealthy: Whether the VM is healthy.
+ vmIsBroken: Whether the VM is in a permanently broken state. Broken VMs
+ should be abandoned or deleted ASAP to avoid assigning or completing any
+ work.
+ vmIsHealthy: Whether the VM is currently healthy.
vmStartupTime: The time the VM was booted.
"""
@@ -5115,10 +5277,12 @@ class WorkerHealthReport(_messages.Message):
additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
- pods = _messages.MessageField('PodsValueListEntry', 1, repeated=True)
- reportInterval = _messages.StringField(2)
- vmIsHealthy = _messages.BooleanField(3)
- vmStartupTime = _messages.StringField(4)
+ msg = _messages.StringField(1)
+ pods = _messages.MessageField('PodsValueListEntry', 2, repeated=True)
+ reportInterval = _messages.StringField(3)
+ vmIsBroken = _messages.BooleanField(4)
+ vmIsHealthy = _messages.BooleanField(5)
+ vmStartupTime = _messages.StringField(6)
class WorkerHealthReportResponse(_messages.Message):