You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2016/06/14 23:12:53 UTC
[18/50] [abbrv] incubator-beam git commit: Move all files to
apache_beam folder
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b14dfadd/sdks/python/google/cloud/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
----------------------------------------------------------------------
diff --git a/sdks/python/google/cloud/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/google/cloud/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
deleted file mode 100644
index 2e0273f..0000000
--- a/sdks/python/google/cloud/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
+++ /dev/null
@@ -1,3056 +0,0 @@
-"""Generated message classes for dataflow version v1b3.
-
-Develops and executes data processing patterns like ETL, batch computation,
-and continuous computation.
-"""
-# NOTE: This file is autogenerated and should not be edited by hand.
-
-from apitools.base.protorpclite import messages as _messages
-from apitools.base.py import encoding
-
-
-package = 'dataflow'
-
-
-class ApproximateProgress(_messages.Message):
- """Obsolete in favor of ApproximateReportedProgress and
- ApproximateSplitRequest.
-
- Fields:
- percentComplete: Obsolete.
- position: Obsolete.
- remainingTime: Obsolete.
- """
-
- percentComplete = _messages.FloatField(1, variant=_messages.Variant.FLOAT)
- position = _messages.MessageField('Position', 2)
- remainingTime = _messages.StringField(3)
-
-
-class ApproximateReportedProgress(_messages.Message):
- """A progress measurement of a WorkItem by a worker.
-
- Fields:
- consumedParallelism: Total amount of parallelism in the portion of input
- of this work item that has already been consumed. In the first two
- examples above (see remaining_parallelism), the value should be 30 or 3
- respectively. The sum of remaining_parallelism and consumed_parallelism
- should equal the total amount of parallelism in this work item. If
- specified, must be finite.
- fractionConsumed: Completion as fraction of the input consumed, from 0.0
- (beginning, nothing consumed), to 1.0 (end of the input, entire input
- consumed).
- position: A Position within the work to represent a progress.
- remainingParallelism: Total amount of parallelism in the input of this
- WorkItem that has not been consumed yet (i.e. can be delegated to a new
- WorkItem via dynamic splitting). "Amount of parallelism" refers to how
- many non-empty parts of the input can be read in parallel. This does not
- necessarily equal number of records. An input that can be read in
- parallel down to the individual records is called "perfectly
- splittable". An example of non-perfectly parallelizable input is a
- block-compressed file format where a block of records has to be read as
- a whole, but different blocks can be read in parallel. Examples: * If we
- have read 30 records out of 50 in a perfectly splittable 50-record
- input, this value should be 20. * If we are reading through block 3 in a
- block-compressed file consisting of 5 blocks, this value should be 2
- (since blocks 4 and 5 can be processed in parallel by new work items via
- dynamic splitting). * If we are reading through the last block in a
- block-compressed file, or reading or processing the last record in a
- perfectly splittable input, this value should be 0, because the
- remainder of the work item cannot be further split.
- """
-
- consumedParallelism = _messages.MessageField('ReportedParallelism', 1)
- fractionConsumed = _messages.FloatField(2)
- position = _messages.MessageField('Position', 3)
- remainingParallelism = _messages.MessageField('ReportedParallelism', 4)
-
-
-class ApproximateSplitRequest(_messages.Message):
- """A suggestion by the service to the worker to dynamically split the
- WorkItem.
-
- Fields:
- fractionConsumed: A fraction at which to split the work item, from 0.0
- (beginning of the input) to 1.0 (end of the input).
- position: A Position at which to split the work item.
- """
-
- fractionConsumed = _messages.FloatField(1)
- position = _messages.MessageField('Position', 2)
-
-
-class AutoscalingSettings(_messages.Message):
- """Settings for WorkerPool autoscaling.
-
- Enums:
- AlgorithmValueValuesEnum: The algorithm to use for autoscaling.
-
- Fields:
- algorithm: The algorithm to use for autoscaling.
- maxNumWorkers: The maximum number of workers to cap scaling at.
- """
-
- class AlgorithmValueValuesEnum(_messages.Enum):
- """The algorithm to use for autoscaling.
-
- Values:
- AUTOSCALING_ALGORITHM_UNKNOWN: <no description>
- AUTOSCALING_ALGORITHM_NONE: <no description>
- AUTOSCALING_ALGORITHM_BASIC: <no description>
- """
- AUTOSCALING_ALGORITHM_UNKNOWN = 0
- AUTOSCALING_ALGORITHM_NONE = 1
- AUTOSCALING_ALGORITHM_BASIC = 2
-
- algorithm = _messages.EnumField('AlgorithmValueValuesEnum', 1)
- maxNumWorkers = _messages.IntegerField(2, variant=_messages.Variant.INT32)
-
-
-class ComputationTopology(_messages.Message):
- """All configuration data for a particular Computation.
-
- Fields:
- computationId: The ID of the computation.
- inputs: The inputs to the computation.
- keyRanges: The key ranges processed by the computation.
- outputs: The outputs from the computation.
- stateFamilies: The state family values.
- systemStageName: The system stage name.
- userStageName: The user stage name.
- """
-
- computationId = _messages.StringField(1)
- inputs = _messages.MessageField('StreamLocation', 2, repeated=True)
- keyRanges = _messages.MessageField('KeyRangeLocation', 3, repeated=True)
- outputs = _messages.MessageField('StreamLocation', 4, repeated=True)
- stateFamilies = _messages.MessageField('StateFamilyConfig', 5, repeated=True)
- systemStageName = _messages.StringField(6)
- userStageName = _messages.StringField(7)
-
-
-class ConcatPosition(_messages.Message):
- """A position that encapsulates an inner position and an index for the inner
- position. A ConcatPosition can be used by a reader of a source that
- encapsulates a set of other sources.
-
- Fields:
- index: Index of the inner source.
- position: Position within the inner source.
- """
-
- index = _messages.IntegerField(1, variant=_messages.Variant.INT32)
- position = _messages.MessageField('Position', 2)
-
-
-class CustomSourceLocation(_messages.Message):
- """Identifies the location of a custom souce.
-
- Fields:
- stateful: Whether this source is stateful.
- """
-
- stateful = _messages.BooleanField(1)
-
-
-class DataDiskAssignment(_messages.Message):
- """Data disk assignment for a given VM instance.
-
- Fields:
- dataDisks: Mounted data disks. The order is important a data disk's
- 0-based index in this list defines which persistent directory the disk
- is mounted to, for example the list of {
- "myproject-1014-104817-4c2-harness-0-disk-0" }, {
- "myproject-1014-104817-4c2-harness-0-disk-1" }.
- vmInstance: VM instance name the data disks mounted to, for example
- "myproject-1014-104817-4c2-harness-0".
- """
-
- dataDisks = _messages.StringField(1, repeated=True)
- vmInstance = _messages.StringField(2)
-
-
-class DataflowProjectsJobsCreateRequest(_messages.Message):
- """A DataflowProjectsJobsCreateRequest object.
-
- Enums:
- ViewValueValuesEnum: Level of information requested in response.
-
- Fields:
- job: A Job resource to be passed as the request body.
- projectId: The project which owns the job.
- replaceJobId: DEPRECATED. This field is now on the Job message.
- view: Level of information requested in response.
- """
-
- class ViewValueValuesEnum(_messages.Enum):
- """Level of information requested in response.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- job = _messages.MessageField('Job', 1)
- projectId = _messages.StringField(2, required=True)
- replaceJobId = _messages.StringField(3)
- view = _messages.EnumField('ViewValueValuesEnum', 4)
-
-
-class DataflowProjectsJobsGetMetricsRequest(_messages.Message):
- """A DataflowProjectsJobsGetMetricsRequest object.
-
- Fields:
- jobId: The job to get messages for.
- projectId: A project id.
- startTime: Return only metric data that has changed since this time.
- Default is to return all information about all metrics for the job.
- """
-
- jobId = _messages.StringField(1, required=True)
- projectId = _messages.StringField(2, required=True)
- startTime = _messages.StringField(3)
-
-
-class DataflowProjectsJobsGetRequest(_messages.Message):
- """A DataflowProjectsJobsGetRequest object.
-
- Enums:
- ViewValueValuesEnum: Level of information requested in response.
-
- Fields:
- jobId: Identifies a single job.
- projectId: The project which owns the job.
- view: Level of information requested in response.
- """
-
- class ViewValueValuesEnum(_messages.Enum):
- """Level of information requested in response.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- jobId = _messages.StringField(1, required=True)
- projectId = _messages.StringField(2, required=True)
- view = _messages.EnumField('ViewValueValuesEnum', 3)
-
-
-class DataflowProjectsJobsListRequest(_messages.Message):
- """A DataflowProjectsJobsListRequest object.
-
- Enums:
- FilterValueValuesEnum: The kind of filter to use.
- ViewValueValuesEnum: Level of information requested in response. Default
- is SUMMARY.
-
- Fields:
- filter: The kind of filter to use.
- pageSize: If there are many jobs, limit response to at most this many. The
- actual number of jobs returned will be the lesser of max_responses and
- an unspecified server-defined limit.
- pageToken: Set this to the 'next_page_token' field of a previous response
- to request additional results in a long list.
- projectId: The project which owns the jobs.
- view: Level of information requested in response. Default is SUMMARY.
- """
-
- class FilterValueValuesEnum(_messages.Enum):
- """The kind of filter to use.
-
- Values:
- ALL: <no description>
- TERMINATED: <no description>
- ACTIVE: <no description>
- """
- ALL = 0
- TERMINATED = 1
- ACTIVE = 2
-
- class ViewValueValuesEnum(_messages.Enum):
- """Level of information requested in response. Default is SUMMARY.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- filter = _messages.EnumField('FilterValueValuesEnum', 1)
- pageSize = _messages.IntegerField(2, variant=_messages.Variant.INT32)
- pageToken = _messages.StringField(3)
- projectId = _messages.StringField(4, required=True)
- view = _messages.EnumField('ViewValueValuesEnum', 5)
-
-
-class DataflowProjectsJobsMessagesListRequest(_messages.Message):
- """A DataflowProjectsJobsMessagesListRequest object.
-
- Enums:
- MinimumImportanceValueValuesEnum: Filter to only get messages with
- importance >= level
-
- Fields:
- endTime: Return only messages with timestamps < end_time. The default is
- now (i.e. return up to the latest messages available).
- jobId: The job to get messages about.
- minimumImportance: Filter to only get messages with importance >= level
- pageSize: If specified, determines the maximum number of messages to
- return. If unspecified, the service may choose an appropriate default,
- or may return an arbitrarily large number of results.
- pageToken: If supplied, this should be the value of next_page_token
- returned by an earlier call. This will cause the next page of results to
- be returned.
- projectId: A project id.
- startTime: If specified, return only messages with timestamps >=
- start_time. The default is the job creation time (i.e. beginning of
- messages).
- """
-
- class MinimumImportanceValueValuesEnum(_messages.Enum):
- """Filter to only get messages with importance >= level
-
- Values:
- JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
- JOB_MESSAGE_DEBUG: <no description>
- JOB_MESSAGE_DETAILED: <no description>
- JOB_MESSAGE_BASIC: <no description>
- JOB_MESSAGE_WARNING: <no description>
- JOB_MESSAGE_ERROR: <no description>
- """
- JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
- JOB_MESSAGE_DEBUG = 1
- JOB_MESSAGE_DETAILED = 2
- JOB_MESSAGE_BASIC = 3
- JOB_MESSAGE_WARNING = 4
- JOB_MESSAGE_ERROR = 5
-
- endTime = _messages.StringField(1)
- jobId = _messages.StringField(2, required=True)
- minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 3)
- pageSize = _messages.IntegerField(4, variant=_messages.Variant.INT32)
- pageToken = _messages.StringField(5)
- projectId = _messages.StringField(6, required=True)
- startTime = _messages.StringField(7)
-
-
-class DataflowProjectsJobsUpdateRequest(_messages.Message):
- """A DataflowProjectsJobsUpdateRequest object.
-
- Fields:
- job: A Job resource to be passed as the request body.
- jobId: Identifies a single job.
- projectId: The project which owns the job.
- """
-
- job = _messages.MessageField('Job', 1)
- jobId = _messages.StringField(2, required=True)
- projectId = _messages.StringField(3, required=True)
-
-
-class DataflowProjectsJobsWorkItemsLeaseRequest(_messages.Message):
- """A DataflowProjectsJobsWorkItemsLeaseRequest object.
-
- Fields:
- jobId: Identifies the workflow job this worker belongs to.
- leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the
- request body.
- projectId: Identifies the project this worker belongs to.
- """
-
- jobId = _messages.StringField(1, required=True)
- leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2)
- projectId = _messages.StringField(3, required=True)
-
-
-class DataflowProjectsJobsWorkItemsReportStatusRequest(_messages.Message):
- """A DataflowProjectsJobsWorkItemsReportStatusRequest object.
-
- Fields:
- jobId: The job which the WorkItem is part of.
- projectId: The project which owns the WorkItem's job.
- reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be
- passed as the request body.
- """
-
- jobId = _messages.StringField(1, required=True)
- projectId = _messages.StringField(2, required=True)
- reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 3)
-
-
-class DataflowProjectsWorkerMessagesRequest(_messages.Message):
- """A DataflowProjectsWorkerMessagesRequest object.
-
- Fields:
- projectId: The project to send the WorkerMessages to.
- sendWorkerMessagesRequest: A SendWorkerMessagesRequest resource to be
- passed as the request body.
- """
-
- projectId = _messages.StringField(1, required=True)
- sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 2)
-
-
-class DerivedSource(_messages.Message):
- """Specification of one of the bundles produced as a result of splitting a
- Source (e.g. when executing a SourceSplitRequest, or when splitting an
- active task using WorkItemStatus.dynamic_source_split), relative to the
- source being split.
-
- Enums:
- DerivationModeValueValuesEnum: What source to base the produced source on
- (if any).
-
- Fields:
- derivationMode: What source to base the produced source on (if any).
- source: Specification of the source.
- """
-
- class DerivationModeValueValuesEnum(_messages.Enum):
- """What source to base the produced source on (if any).
-
- Values:
- SOURCE_DERIVATION_MODE_UNKNOWN: <no description>
- SOURCE_DERIVATION_MODE_INDEPENDENT: <no description>
- SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: <no description>
- SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: <no description>
- """
- SOURCE_DERIVATION_MODE_UNKNOWN = 0
- SOURCE_DERIVATION_MODE_INDEPENDENT = 1
- SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2
- SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3
-
- derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1)
- source = _messages.MessageField('Source', 2)
-
-
-class Disk(_messages.Message):
- """Describes the data disk used by a workflow job.
-
- Fields:
- diskType: Disk storage type, as defined by Google Compute Engine. This
- must be a disk type appropriate to the project and zone in which the
- workers will run. If unknown or unspecified, the service will attempt to
- choose a reasonable default. For example, the standard persistent disk
- type is a resource name typically ending in "pd-standard". If SSD
- persistent disks are available, the resource name typically ends with
- "pd-ssd". The actual valid values are defined the Google Compute Engine
- API, not by the Dataflow API; consult the Google Compute Engine
- documentation for more information about determining the set of
- available disk types for a particular project and zone. Google Compute
- Engine Disk types are local to a particular project in a particular
- zone, and so the resource name will typically look something like this:
- compute.googleapis.com/projects/ /zones//diskTypes/pd-standard
- mountPoint: Directory in a VM where disk is mounted.
- sizeGb: Size of disk in GB. If zero or unspecified, the service will
- attempt to choose a reasonable default.
- """
-
- diskType = _messages.StringField(1)
- mountPoint = _messages.StringField(2)
- sizeGb = _messages.IntegerField(3, variant=_messages.Variant.INT32)
-
-
-class DynamicSourceSplit(_messages.Message):
- """When a task splits using WorkItemStatus.dynamic_source_split, this
- message describes the two parts of the split relative to the description of
- the current task's input.
-
- Fields:
- primary: Primary part (continued to be processed by worker). Specified
- relative to the previously-current source. Becomes current.
- residual: Residual part (returned to the pool of work). Specified relative
- to the previously-current source.
- """
-
- primary = _messages.MessageField('DerivedSource', 1)
- residual = _messages.MessageField('DerivedSource', 2)
-
-
-class Environment(_messages.Message):
- """Describes the environment in which a Dataflow Job runs.
-
- Messages:
- InternalExperimentsValue: Experimental settings.
- SdkPipelineOptionsValue: The Dataflow SDK pipeline options specified by
- the user. These options are passed through the service and are used to
- recreate the SDK pipeline options on the worker in a language agnostic
- and platform independent way.
- UserAgentValue: A description of the process that generated the request.
- VersionValue: A structure describing which components and their versions
- of the service are required in order to run the job.
-
- Fields:
- clusterManagerApiService: The type of cluster manager API to use. If
- unknown or unspecified, the service will attempt to choose a reasonable
- default. This should be in the form of the API service name, e.g.
- "compute.googleapis.com".
- dataset: The dataset for the current project where various workflow
- related tables are stored. The supported resource type is: Google
- BigQuery: bigquery.googleapis.com/{dataset}
- experiments: The list of experiments to enable.
- internalExperiments: Experimental settings.
- sdkPipelineOptions: The Dataflow SDK pipeline options specified by the
- user. These options are passed through the service and are used to
- recreate the SDK pipeline options on the worker in a language agnostic
- and platform independent way.
- tempStoragePrefix: The prefix of the resources the system should use for
- temporary storage. The system will append the suffix "/temp-{JOBNAME} to
- this resource prefix, where {JOBNAME} is the value of the job_name
- field. The resulting bucket and object prefix is used as the prefix of
- the resources used to store temporary data needed during the job
- execution. NOTE: This will override the value in taskrunner_settings.
- The supported resource type is: Google Cloud Storage:
- storage.googleapis.com/{bucket}/{object}
- bucket.storage.googleapis.com/{object}
- userAgent: A description of the process that generated the request.
- version: A structure describing which components and their versions of the
- service are required in order to run the job.
- workerPools: Worker pools. At least one "harness" worker pool must be
- specified in order for the job to have workers.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class InternalExperimentsValue(_messages.Message):
- """Experimental settings.
-
- Messages:
- AdditionalProperty: An additional property for a
- InternalExperimentsValue object.
-
- Fields:
- additionalProperties: Properties of the object. Contains field @ype with
- type URL.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a InternalExperimentsValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class SdkPipelineOptionsValue(_messages.Message):
- """The Dataflow SDK pipeline options specified by the user. These options
- are passed through the service and are used to recreate the SDK pipeline
- options on the worker in a language agnostic and platform independent way.
-
- Messages:
- AdditionalProperty: An additional property for a SdkPipelineOptionsValue
- object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a SdkPipelineOptionsValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class UserAgentValue(_messages.Message):
- """A description of the process that generated the request.
-
- Messages:
- AdditionalProperty: An additional property for a UserAgentValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a UserAgentValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class VersionValue(_messages.Message):
- """A structure describing which components and their versions of the
- service are required in order to run the job.
-
- Messages:
- AdditionalProperty: An additional property for a VersionValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a VersionValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- clusterManagerApiService = _messages.StringField(1)
- dataset = _messages.StringField(2)
- experiments = _messages.StringField(3, repeated=True)
- internalExperiments = _messages.MessageField('InternalExperimentsValue', 4)
- sdkPipelineOptions = _messages.MessageField('SdkPipelineOptionsValue', 5)
- tempStoragePrefix = _messages.StringField(6)
- userAgent = _messages.MessageField('UserAgentValue', 7)
- version = _messages.MessageField('VersionValue', 8)
- workerPools = _messages.MessageField('WorkerPool', 9, repeated=True)
-
-
-class FlattenInstruction(_messages.Message):
- """An instruction that copies its inputs (zero or more) to its (single)
- output.
-
- Fields:
- inputs: Describes the inputs to the flatten instruction.
- """
-
- inputs = _messages.MessageField('InstructionInput', 1, repeated=True)
-
-
-class InstructionInput(_messages.Message):
- """An input of an instruction, as a reference to an output of a producer
- instruction.
-
- Fields:
- outputNum: The output index (origin zero) within the producer.
- producerInstructionIndex: The index (origin zero) of the parallel
- instruction that produces the output to be consumed by this input. This
- index is relative to the list of instructions in this input's
- instruction's containing MapTask.
- """
-
- outputNum = _messages.IntegerField(1, variant=_messages.Variant.INT32)
- producerInstructionIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32)
-
-
-class InstructionOutput(_messages.Message):
- """An output of an instruction.
-
- Messages:
- CodecValue: The codec to use to encode data being written via this output.
-
- Fields:
- codec: The codec to use to encode data being written via this output.
- name: The user-provided name of this output.
- systemName: System-defined name of this output. Unique across the
- workflow.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class CodecValue(_messages.Message):
- """The codec to use to encode data being written via this output.
-
- Messages:
- AdditionalProperty: An additional property for a CodecValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a CodecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- codec = _messages.MessageField('CodecValue', 1)
- name = _messages.StringField(2)
- systemName = _messages.StringField(3)
-
-
-class Job(_messages.Message):
- """Defines a job to be run by the Dataflow service.
-
- Enums:
- CurrentStateValueValuesEnum: The current state of the job. Jobs are
- created in the JOB_STATE_STOPPED state unless otherwise specified. A job
- in the JOB_STATE_RUNNING state may asynchronously enter a terminal
- state. Once a job has reached a terminal state, no further state updates
- may be made. This field may be mutated by the Dataflow service; callers
- cannot mutate it.
- RequestedStateValueValuesEnum: The job's requested state. UpdateJob may be
- used to switch between the JOB_STATE_STOPPED and JOB_STATE_RUNNING
- states, by setting requested_state. UpdateJob may also be used to
- directly set a job's requested state to JOB_STATE_CANCELLED or
- JOB_STATE_DONE, irrevocably terminating the job if it has not already
- reached a terminal state.
- TypeValueValuesEnum: The type of dataflow job.
-
- Messages:
- TransformNameMappingValue: Map of transform name prefixes of the job to be
- replaced to the corresponding name prefixes of the new job.
-
- Fields:
- clientRequestId: Client's unique identifier of the job, re-used by SDK
- across retried attempts. If this field is set, the service will ensure
- its uniqueness. That is, the request to create a job will fail if the
- service has knowledge of a previously submitted job with the same
- client's id and job name. The caller may, for example, use this field to
- ensure idempotence of job creation across retried attempts to create a
- job. By default, the field is empty and, in that case, the service
- ignores it.
- createTime: Timestamp when job was initially created. Immutable, set by
- the Dataflow service.
- currentState: The current state of the job. Jobs are created in the
- JOB_STATE_STOPPED state unless otherwise specified. A job in the
- JOB_STATE_RUNNING state may asynchronously enter a terminal state. Once
- a job has reached a terminal state, no further state updates may be
- made. This field may be mutated by the Dataflow service; callers cannot
- mutate it.
- currentStateTime: The timestamp associated with the current state.
- environment: Environment for the job.
- executionInfo: Information about how the Dataflow service will actually
- run the job.
- id: The unique ID of this job. This field is set by the Dataflow service
- when the Job is created, and is immutable for the life of the Job.
- name: The user-specified Dataflow job name. Only one Job with a given name
- may exist in a project at any given time. If a caller attempts to create
- a Job with the same name as an already-existing Job, the attempt will
- return the existing Job. The name must match the regular expression
- [a-z]([-a-z0-9]{0,38}[a-z0-9])?
- projectId: The project which owns the job.
- replaceJobId: If this job is an update of an existing job, this field will
- be the ID of the job it replaced. When sending a CreateJobRequest, you
- can update a job by specifying it here. The job named here will be
- stopped, and its intermediate state transferred to this job.
- replacedByJobId: If another job is an update of this job (and thus, this
- job is in JOB_STATE_UPDATED), this field will contain the ID of that
- job.
- requestedState: The job's requested state. UpdateJob may be used to switch
- between the JOB_STATE_STOPPED and JOB_STATE_RUNNING states, by setting
- requested_state. UpdateJob may also be used to directly set a job's
- requested state to JOB_STATE_CANCELLED or JOB_STATE_DONE, irrevocably
- terminating the job if it has not already reached a terminal state.
- steps: The top-level steps that constitute the entire job.
- tempFiles: A set of files the system should be aware of that are used for
- temporary storage. These temporary files will be removed on job
- completion. No duplicates are allowed. No file patterns are supported.
- The supported files are: Google Cloud Storage:
- storage.googleapis.com/{bucket}/{object}
- bucket.storage.googleapis.com/{object}
- transformNameMapping: Map of transform name prefixes of the job to be
- replaced to the corresponding name prefixes of the new job.
- type: The type of dataflow job.
- """
-
- class CurrentStateValueValuesEnum(_messages.Enum):
- """The current state of the job. Jobs are created in the JOB_STATE_STOPPED
- state unless otherwise specified. A job in the JOB_STATE_RUNNING state may
- asynchronously enter a terminal state. Once a job has reached a terminal
- state, no further state updates may be made. This field may be mutated by
- the Dataflow service; callers cannot mutate it.
-
- Values:
- JOB_STATE_UNKNOWN: <no description>
- JOB_STATE_STOPPED: <no description>
- JOB_STATE_RUNNING: <no description>
- JOB_STATE_DONE: <no description>
- JOB_STATE_FAILED: <no description>
- JOB_STATE_CANCELLED: <no description>
- JOB_STATE_UPDATED: <no description>
- JOB_STATE_DRAINING: <no description>
- JOB_STATE_DRAINED: <no description>
- """
- JOB_STATE_UNKNOWN = 0
- JOB_STATE_STOPPED = 1
- JOB_STATE_RUNNING = 2
- JOB_STATE_DONE = 3
- JOB_STATE_FAILED = 4
- JOB_STATE_CANCELLED = 5
- JOB_STATE_UPDATED = 6
- JOB_STATE_DRAINING = 7
- JOB_STATE_DRAINED = 8
-
- class RequestedStateValueValuesEnum(_messages.Enum):
- """The job's requested state. UpdateJob may be used to switch between the
- JOB_STATE_STOPPED and JOB_STATE_RUNNING states, by setting
- requested_state. UpdateJob may also be used to directly set a job's
- requested state to JOB_STATE_CANCELLED or JOB_STATE_DONE, irrevocably
- terminating the job if it has not already reached a terminal state.
-
- Values:
- JOB_STATE_UNKNOWN: <no description>
- JOB_STATE_STOPPED: <no description>
- JOB_STATE_RUNNING: <no description>
- JOB_STATE_DONE: <no description>
- JOB_STATE_FAILED: <no description>
- JOB_STATE_CANCELLED: <no description>
- JOB_STATE_UPDATED: <no description>
- JOB_STATE_DRAINING: <no description>
- JOB_STATE_DRAINED: <no description>
- """
- JOB_STATE_UNKNOWN = 0
- JOB_STATE_STOPPED = 1
- JOB_STATE_RUNNING = 2
- JOB_STATE_DONE = 3
- JOB_STATE_FAILED = 4
- JOB_STATE_CANCELLED = 5
- JOB_STATE_UPDATED = 6
- JOB_STATE_DRAINING = 7
- JOB_STATE_DRAINED = 8
-
- class TypeValueValuesEnum(_messages.Enum):
- """The type of dataflow job.
-
- Values:
- JOB_TYPE_UNKNOWN: <no description>
- JOB_TYPE_BATCH: <no description>
- JOB_TYPE_STREAMING: <no description>
- """
- JOB_TYPE_UNKNOWN = 0
- JOB_TYPE_BATCH = 1
- JOB_TYPE_STREAMING = 2
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class TransformNameMappingValue(_messages.Message):
- """Map of transform name prefixes of the job to be replaced to the
- corresponding name prefixes of the new job.
-
- Messages:
- AdditionalProperty: An additional property for a
- TransformNameMappingValue object.
-
- Fields:
- additionalProperties: Additional properties of type
- TransformNameMappingValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a TransformNameMappingValue object.
-
- Fields:
- key: Name of the additional property.
- value: A string attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.StringField(2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- clientRequestId = _messages.StringField(1)
- createTime = _messages.StringField(2)
- currentState = _messages.EnumField('CurrentStateValueValuesEnum', 3)
- currentStateTime = _messages.StringField(4)
- environment = _messages.MessageField('Environment', 5)
- executionInfo = _messages.MessageField('JobExecutionInfo', 6)
- id = _messages.StringField(7)
- name = _messages.StringField(8)
- projectId = _messages.StringField(9)
- replaceJobId = _messages.StringField(10)
- replacedByJobId = _messages.StringField(11)
- requestedState = _messages.EnumField('RequestedStateValueValuesEnum', 12)
- steps = _messages.MessageField('Step', 13, repeated=True)
- tempFiles = _messages.StringField(14, repeated=True)
- transformNameMapping = _messages.MessageField('TransformNameMappingValue', 15)
- type = _messages.EnumField('TypeValueValuesEnum', 16)
-
-
-class JobExecutionInfo(_messages.Message):
- """Additional information about how a Dataflow job will be executed which
- isn\u2019t contained in the submitted job.
-
- Messages:
- StagesValue: A mapping from each stage to the information about that
- stage.
-
- Fields:
- stages: A mapping from each stage to the information about that stage.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class StagesValue(_messages.Message):
- """A mapping from each stage to the information about that stage.
-
- Messages:
- AdditionalProperty: An additional property for a StagesValue object.
-
- Fields:
- additionalProperties: Additional properties of type StagesValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a StagesValue object.
-
- Fields:
- key: Name of the additional property.
- value: A JobExecutionStageInfo attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('JobExecutionStageInfo', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- stages = _messages.MessageField('StagesValue', 1)
-
-
-class JobExecutionStageInfo(_messages.Message):
- """Contains information about how a particular google.dataflow.v1beta3.Step
- will be executed.
-
- Fields:
- stepName: The steps associated with the execution stage. Note that stages
- may have several steps, and that a given step might be run by more than
- one stage.
- """
-
- stepName = _messages.StringField(1, repeated=True)
-
-
-class JobMessage(_messages.Message):
- """A particular message pertaining to a Dataflow job.
-
- Enums:
- MessageImportanceValueValuesEnum: Importance level of the message.
-
- Fields:
- id: Identifies the message. This is automatically generated by the
- service; the caller should treat it as an opaque string.
- messageImportance: Importance level of the message.
- messageText: The text of the message.
- time: The timestamp of the message.
- """
-
- class MessageImportanceValueValuesEnum(_messages.Enum):
- """Importance level of the message.
-
- Values:
- JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
- JOB_MESSAGE_DEBUG: <no description>
- JOB_MESSAGE_DETAILED: <no description>
- JOB_MESSAGE_BASIC: <no description>
- JOB_MESSAGE_WARNING: <no description>
- JOB_MESSAGE_ERROR: <no description>
- """
- JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
- JOB_MESSAGE_DEBUG = 1
- JOB_MESSAGE_DETAILED = 2
- JOB_MESSAGE_BASIC = 3
- JOB_MESSAGE_WARNING = 4
- JOB_MESSAGE_ERROR = 5
-
- id = _messages.StringField(1)
- messageImportance = _messages.EnumField('MessageImportanceValueValuesEnum', 2)
- messageText = _messages.StringField(3)
- time = _messages.StringField(4)
-
-
-class JobMetrics(_messages.Message):
- """JobMetrics contains a collection of metrics descibing the detailed
- progress of a Dataflow job. Metrics correspond to user-defined and system-
- defined metrics in the job. This resource captures only the most recent
- values of each metric; time-series data can be queried for them (under the
- same metric names) from Cloud Monitoring.
-
- Fields:
- metricTime: Timestamp as of which metric values are current.
- metrics: All metrics for this job.
- """
-
- metricTime = _messages.StringField(1)
- metrics = _messages.MessageField('MetricUpdate', 2, repeated=True)
-
-
-class KeyRangeDataDiskAssignment(_messages.Message):
- """Data disk assignment information for a specific key-range of a sharded
- computation. Currently we only support UTF-8 character splits to simplify
- encoding into JSON.
-
- Fields:
- dataDisk: The name of the data disk where data for this range is stored.
- This name is local to the Google Cloud Platform project and uniquely
- identifies the disk within that project, for example
- "myproject-1014-104817-4c2-harness-0-disk-1".
- end: The end (exclusive) of the key range.
- start: The start (inclusive) of the key range.
- """
-
- dataDisk = _messages.StringField(1)
- end = _messages.StringField(2)
- start = _messages.StringField(3)
-
-
-class KeyRangeLocation(_messages.Message):
- """Location information for a specific key-range of a sharded computation.
- Currently we only support UTF-8 character splits to simplify encoding into
- JSON.
-
- Fields:
- dataDisk: The name of the data disk where data for this range is stored.
- This name is local to the Google Cloud Platform project and uniquely
- identifies the disk within that project, for example
- "myproject-1014-104817-4c2-harness-0-disk-1".
- deliveryEndpoint: The physical location of this range assignment to be
- used for streaming computation cross-worker message delivery.
- end: The end (exclusive) of the key range.
- persistentDirectory: The location of the persistent state for this range,
- as a persistent directory in the worker local filesystem.
- start: The start (inclusive) of the key range.
- """
-
- dataDisk = _messages.StringField(1)
- deliveryEndpoint = _messages.StringField(2)
- end = _messages.StringField(3)
- persistentDirectory = _messages.StringField(4)
- start = _messages.StringField(5)
-
-
-class LeaseWorkItemRequest(_messages.Message):
- """Request to lease WorkItems.
-
- Fields:
- currentWorkerTime: The current timestamp at the worker.
- requestedLeaseDuration: The initial lease period.
- workItemTypes: Filter for WorkItem type.
- workerCapabilities: Worker capabilities. WorkItems might be limited to
- workers with specific capabilities.
- workerId: Identifies the worker leasing work -- typically the ID of the
- virtual machine running the worker.
- """
-
- currentWorkerTime = _messages.StringField(1)
- requestedLeaseDuration = _messages.StringField(2)
- workItemTypes = _messages.StringField(3, repeated=True)
- workerCapabilities = _messages.StringField(4, repeated=True)
- workerId = _messages.StringField(5)
-
-
-class LeaseWorkItemResponse(_messages.Message):
- """Response to a request to lease WorkItems.
-
- Fields:
- workItems: A list of the leased WorkItems.
- """
-
- workItems = _messages.MessageField('WorkItem', 1, repeated=True)
-
-
-class ListJobMessagesResponse(_messages.Message):
- """Response to a request to list job messages.
-
- Fields:
- jobMessages: Messages in ascending timestamp order.
- nextPageToken: The token to obtain the next page of results if there are
- more.
- """
-
- jobMessages = _messages.MessageField('JobMessage', 1, repeated=True)
- nextPageToken = _messages.StringField(2)
-
-
-class ListJobsResponse(_messages.Message):
- """Response to a request to list Dataflow jobs. This may be a partial
- response, depending on the page size in the ListJobsRequest.
-
- Fields:
- jobs: A subset of the requested job information.
- nextPageToken: Set if there may be more results than fit in this response.
- """
-
- jobs = _messages.MessageField('Job', 1, repeated=True)
- nextPageToken = _messages.StringField(2)
-
-
-class MapTask(_messages.Message):
- """MapTask consists of an ordered set of instructions, each of which
- describes one particular low-level operation for the worker to perform in
- order to accomplish the MapTask's WorkItem. Each instruction must appear in
- the list before any instructions which depends on its output.
-
- Fields:
- instructions: The instructions in the MapTask.
- stageName: System-defined name of the stage containing this MapTask.
- Unique across the workflow.
- systemName: System-defined name of this MapTask. Unique across the
- workflow.
- """
-
- instructions = _messages.MessageField('ParallelInstruction', 1, repeated=True)
- stageName = _messages.StringField(2)
- systemName = _messages.StringField(3)
-
-
-class MetricStructuredName(_messages.Message):
- """Identifies a metric, by describing the source which generated the metric.
-
- Messages:
- ContextValue: Zero or more labeled fields which identify the part of the
- job this metric is associated with, such as the name of a step or
- collection. For example, built-in counters associated with steps will
- have context['step'] = . Counters associated with PCollections in the
- SDK will have context['pcollection'] = .
-
- Fields:
- context: Zero or more labeled fields which identify the part of the job
- this metric is associated with, such as the name of a step or
- collection. For example, built-in counters associated with steps will
- have context['step'] = . Counters associated with PCollections in the
- SDK will have context['pcollection'] = .
- name: Worker-defined metric name.
- origin: Origin (namespace) of metric name. May be blank for user-define
- metrics; will be "dataflow" for metrics defined by the Dataflow service
- or SDK.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class ContextValue(_messages.Message):
- """Zero or more labeled fields which identify the part of the job this
- metric is associated with, such as the name of a step or collection. For
- example, built-in counters associated with steps will have context['step']
- = . Counters associated with PCollections in the SDK will have
- context['pcollection'] = .
-
- Messages:
- AdditionalProperty: An additional property for a ContextValue object.
-
- Fields:
- additionalProperties: Additional properties of type ContextValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a ContextValue object.
-
- Fields:
- key: Name of the additional property.
- value: A string attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.StringField(2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- context = _messages.MessageField('ContextValue', 1)
- name = _messages.StringField(2)
- origin = _messages.StringField(3)
-
-
-class MetricUpdate(_messages.Message):
- """Describes the state of a metric.
-
- Fields:
- cumulative: True if this metric is reported as the total cumulative
- aggregate value accumulated since the worker started working on this
- WorkItem. By default this is false, indicating that this metric is
- reported as a delta that is not associated with any WorkItem.
- internal: Worker-computed aggregate value for internal use by the Dataflow
- service.
- kind: Metric aggregation kind. The possible metric aggregation kinds are
- "Sum", "Max", "Min", "Mean", "Set", "And", and "Or". The specified
- aggregation kind is case-insensitive. If omitted, this is not an
- aggregated value but instead a single metric sample value.
- meanCount: Worker-computed aggregate value for the "Mean" aggregation
- kind. This holds the count of the aggregated values and is used in
- combination with mean_sum above to obtain the actual mean aggregate
- value. The only possible value type is Long.
- meanSum: Worker-computed aggregate value for the "Mean" aggregation kind.
- This holds the sum of the aggregated values and is used in combination
- with mean_count below to obtain the actual mean aggregate value. The
- only possible value types are Long and Double.
- name: Name of the metric.
- scalar: Worker-computed aggregate value for aggregation kinds "Sum",
- "Max", "Min", "And", and "Or". The possible value types are Long,
- Double, and Boolean.
- set: Worker-computed aggregate value for the "Set" aggregation kind. The
- only possible value type is a list of Values whose type can be Long,
- Double, or String, according to the metric's type. All Values in the
- list must be of the same type.
- updateTime: Timestamp associated with the metric value. Optional when
- workers are reporting work progress; it will be filled in responses from
- the metrics API.
- """
-
- cumulative = _messages.BooleanField(1)
- internal = _messages.MessageField('extra_types.JsonValue', 2)
- kind = _messages.StringField(3)
- meanCount = _messages.MessageField('extra_types.JsonValue', 4)
- meanSum = _messages.MessageField('extra_types.JsonValue', 5)
- name = _messages.MessageField('MetricStructuredName', 6)
- scalar = _messages.MessageField('extra_types.JsonValue', 7)
- set = _messages.MessageField('extra_types.JsonValue', 8)
- updateTime = _messages.StringField(9)
-
-
-class MountedDataDisk(_messages.Message):
- """Describes mounted data disk.
-
- Fields:
- dataDisk: The name of the data disk. This name is local to the Google
- Cloud Platform project and uniquely identifies the disk within that
- project, for example "myproject-1014-104817-4c2-harness-0-disk-1".
- """
-
- dataDisk = _messages.StringField(1)
-
-
-class MultiOutputInfo(_messages.Message):
- """Information about an output of a multi-output DoFn.
-
- Fields:
- tag: The id of the tag the user code will emit to this output by; this
- should correspond to the tag of some SideInputInfo.
- """
-
- tag = _messages.StringField(1)
-
-
-class Package(_messages.Message):
- """Packages that need to be installed in order for a worker to run the steps
- of the Dataflow job which will be assigned to its worker pool. This is the
- mechanism by which the SDK causes code to be loaded onto the workers. For
- example, the Dataflow Java SDK might use this to install jars containing the
- user's code and all of the various dependencies (libraries, data files, etc)
- required in order for that code to run.
-
- Fields:
- location: The resource to read the package from. The supported resource
- type is: Google Cloud Storage: storage.googleapis.com/{bucket}
- bucket.storage.googleapis.com/
- name: The name of the package.
- """
-
- location = _messages.StringField(1)
- name = _messages.StringField(2)
-
-
-class ParDoInstruction(_messages.Message):
- """An instruction that does a ParDo operation. Takes one main input and zero
- or more side inputs, and produces zero or more outputs. Runs user code.
-
- Messages:
- UserFnValue: The user function to invoke.
-
- Fields:
- input: The input.
- multiOutputInfos: Information about each of the outputs, if user_fn is a
- MultiDoFn.
- numOutputs: The number of outputs.
- sideInputs: Zero or more side inputs.
- userFn: The user function to invoke.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class UserFnValue(_messages.Message):
- """The user function to invoke.
-
- Messages:
- AdditionalProperty: An additional property for a UserFnValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a UserFnValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- input = _messages.MessageField('InstructionInput', 1)
- multiOutputInfos = _messages.MessageField('MultiOutputInfo', 2, repeated=True)
- numOutputs = _messages.IntegerField(3, variant=_messages.Variant.INT32)
- sideInputs = _messages.MessageField('SideInputInfo', 4, repeated=True)
- userFn = _messages.MessageField('UserFnValue', 5)
-
-
-class ParallelInstruction(_messages.Message):
- """Describes a particular operation comprising a MapTask.
-
- Fields:
- flatten: Additional information for Flatten instructions.
- name: User-provided name of this operation.
- outputs: Describes the outputs of the instruction.
- parDo: Additional information for ParDo instructions.
- partialGroupByKey: Additional information for PartialGroupByKey
- instructions.
- read: Additional information for Read instructions.
- systemName: System-defined name of this operation. Unique across the
- workflow.
- write: Additional information for Write instructions.
- """
-
- flatten = _messages.MessageField('FlattenInstruction', 1)
- name = _messages.StringField(2)
- outputs = _messages.MessageField('InstructionOutput', 3, repeated=True)
- parDo = _messages.MessageField('ParDoInstruction', 4)
- partialGroupByKey = _messages.MessageField('PartialGroupByKeyInstruction', 5)
- read = _messages.MessageField('ReadInstruction', 6)
- systemName = _messages.StringField(7)
- write = _messages.MessageField('WriteInstruction', 8)
-
-
-class PartialGroupByKeyInstruction(_messages.Message):
- """An instruction that does a partial group-by-key. One input and one
- output.
-
- Messages:
- InputElementCodecValue: The codec to use for interpreting an element in
- the input PTable.
- ValueCombiningFnValue: The value combining function to invoke.
-
- Fields:
- input: Describes the input to the partial group-by-key instruction.
- inputElementCodec: The codec to use for interpreting an element in the
- input PTable.
- sideInputs: Zero or more side inputs.
- valueCombiningFn: The value combining function to invoke.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class InputElementCodecValue(_messages.Message):
- """The codec to use for interpreting an element in the input PTable.
-
- Messages:
- AdditionalProperty: An additional property for a InputElementCodecValue
- object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a InputElementCodecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class ValueCombiningFnValue(_messages.Message):
- """The value combining function to invoke.
-
- Messages:
- AdditionalProperty: An additional property for a ValueCombiningFnValue
- object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a ValueCombiningFnValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- input = _messages.MessageField('InstructionInput', 1)
- inputElementCodec = _messages.MessageField('InputElementCodecValue', 2)
- sideInputs = _messages.MessageField('SideInputInfo', 3, repeated=True)
- valueCombiningFn = _messages.MessageField('ValueCombiningFnValue', 4)
-
-
-class Position(_messages.Message):
- """Position defines a position within a collection of data. The value can be
- either the end position, a key (used with ordered collections), a byte
- offset, or a record index.
-
- Fields:
- byteOffset: Position is a byte offset.
- concatPosition: CloudPosition is a concat position.
- end: Position is past all other positions. Also useful for the end
- position of an unbounded range.
- key: Position is a string key, ordered lexicographically.
- recordIndex: Position is a record index.
- shufflePosition: CloudPosition is a base64 encoded BatchShufflePosition
- (with FIXED sharding).
- """
-
- byteOffset = _messages.IntegerField(1)
- concatPosition = _messages.MessageField('ConcatPosition', 2)
- end = _messages.BooleanField(3)
- key = _messages.StringField(4)
- recordIndex = _messages.IntegerField(5)
- shufflePosition = _messages.StringField(6)
-
-
-class PubsubLocation(_messages.Message):
- """Identifies a pubsub location to use for transferring data into or out of
- a streaming Dataflow job.
-
- Fields:
- dropLateData: Indicates whether the pipeline allows late-arriving data.
- idLabel: If set, contains a pubsub label from which to extract record ids.
- If left empty, record deduplication will be strictly best effort.
- subscription: A pubsub subscription, in the form of
- "pubsub.googleapis.com/subscriptions/ /"
- timestampLabel: If set, contains a pubsub label from which to extract
- record timestamps. If left empty, record timestamps will be generated
- upon arrival.
- topic: A pubsub topic, in the form of "pubsub.googleapis.com/topics/ /"
- trackingSubscription: If set, specifies the pubsub subscription that will
- be used for tracking custom time timestamps for watermark estimation.
- """
-
- dropLateData = _messages.BooleanField(1)
- idLabel = _messages.StringField(2)
- subscription = _messages.StringField(3)
- timestampLabel = _messages.StringField(4)
- topic = _messages.StringField(5)
- trackingSubscription = _messages.StringField(6)
-
-
-class ReadInstruction(_messages.Message):
- """An instruction that reads records. Takes no inputs, produces one output.
-
- Fields:
- source: The source to read from.
- """
-
- source = _messages.MessageField('Source', 1)
-
-
-class ReportWorkItemStatusRequest(_messages.Message):
- """Request to report the status of WorkItems.
-
- Fields:
- currentWorkerTime: The current timestamp at the worker.
- workItemStatuses: The order is unimportant, except that the order of the
- WorkItemServiceState messages in the ReportWorkItemStatusResponse
- corresponds to the order of WorkItemStatus messages here.
- workerId: The ID of the worker reporting the WorkItem status. If this does
- not match the ID of the worker which the Dataflow service believes
- currently has the lease on the WorkItem, the report will be dropped
- (with an error response).
- """
-
- currentWorkerTime = _messages.StringField(1)
- workItemStatuses = _messages.MessageField('WorkItemStatus', 2, repeated=True)
- workerId = _messages.StringField(3)
-
-
-class ReportWorkItemStatusResponse(_messages.Message):
- """Response from a request to report the status of WorkItems.
-
- Fields:
- workItemServiceStates: A set of messages indicating the service-side state
- for each WorkItem whose status was reported, in the same order as the
- WorkItemStatus messages in the ReportWorkItemStatusRequest which
- resulting in this response.
- """
-
- workItemServiceStates = _messages.MessageField('WorkItemServiceState', 1, repeated=True)
-
-
-class ReportedParallelism(_messages.Message):
- """Represents the level of parallelism in a WorkItem's input, reported by
- the worker.
-
- Fields:
- isInfinite: Specifies whether the parallelism is infinite. If true,
- "value" is ignored. Infinite parallelism means the service will assume
- that the work item can always be split into more non-empty work items by
- dynamic splitting. This is a work-around for lack of support for
- infinity by the current JSON-based Java RPC stack.
- value: Specifies the level of parallelism in case it is finite.
- """
-
- isInfinite = _messages.BooleanField(1)
- value = _messages.FloatField(2)
-
-
-class SendWorkerMessagesRequest(_messages.Message):
- """A request for sending worker messages to the service.
-
- Fields:
- workerMessages: The WorkerMessages to send.
- """
-
- workerMessages = _messages.MessageField('WorkerMessage', 1, repeated=True)
-
-
-class SendWorkerMessagesResponse(_messages.Message):
- """The response to the worker messages.
-
- Fields:
- workerMessageResponses: The servers response to the worker messages.
- """
-
- workerMessageResponses = _messages.MessageField('WorkerMessageResponse', 1, repeated=True)
-
-
-class SeqMapTask(_messages.Message):
- """Describes a particular function to invoke.
-
- Messages:
- UserFnValue: The user function to invoke.
-
- Fields:
- inputs: Information about each of the inputs.
- name: The user-provided name of the SeqDo operation.
- outputInfos: Information about each of the outputs.
- stageName: System-defined name of the stage containing the SeqDo
- operation. Unique across the workflow.
- systemName: System-defined name of the SeqDo operation. Unique across the
- workflow.
- userFn: The user function to invoke.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class UserFnValue(_messages.Message):
- """The user function to invoke.
-
- Messages:
- AdditionalProperty: An additional property for a UserFnValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a UserFnValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- inputs = _messages.MessageField('SideInputInfo', 1, repeated=True)
- name = _messages.StringField(2)
- outputInfos = _messages.MessageField('SeqMapTaskOutputInfo', 3, repeated=True)
- stageName = _messages.StringField(4)
- systemName = _messages.StringField(5)
- userFn = _messages.MessageField('UserFnValue', 6)
-
-
-class SeqMapTaskOutputInfo(_messages.Message):
- """Information about an output of a SeqMapTask.
-
- Fields:
- sink: The sink to write the output value to.
- tag: The id of the TupleTag the user code will tag the output value by.
- """
-
- sink = _messages.MessageField('Sink', 1)
- tag = _messages.StringField(2)
-
-
-class ShellTask(_messages.Message):
- """A task which consists of a shell command for the worker to execute.
-
- Fields:
- command: The shell command to run.
- exitCode: Exit code for the task.
- """
-
- command = _messages.StringField(1)
- exitCode = _messages.IntegerField(2, variant=_messages.Variant.INT32)
-
-
-class SideInputInfo(_messages.Message):
- """Information about a side input of a DoFn or an input of a SeqDoFn.
-
- Messages:
- KindValue: How to interpret the source element(s) as a side input value.
-
- Fields:
- kind: How to interpret the source element(s) as a side input value.
- sources: The source(s) to read element(s) from to get the value of this
- side input. If more than one source, then the elements are taken from
- the sources, in the specified order if order matters. At least one
- source is required.
- tag: The id of the tag the user code will access this side input by; this
- should correspond to the tag of some MultiOutputInfo.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class KindValue(_messages.Message):
- """How to interpret the source element(s) as a side input value.
-
- Messages:
- AdditionalProperty: An additional property for a KindValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a KindValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- kind = _messages.MessageField('KindValue', 1)
- sources = _messages.MessageField('Source', 2, repeated=True)
- tag = _messages.StringField(3)
-
-
-class Sink(_messages.Message):
- """A sink that records can be encoded and written to.
-
- Messages:
- CodecValue: The codec to use to encode data written to the sink.
- SpecValue: The sink to write to, plus its parameters.
-
- Fields:
- codec: The codec to use to encode data written to the sink.
- spec: The sink to write to, plus its parameters.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class CodecValue(_messages.Message):
- """The codec to use to encode data written to the sink.
-
- Messages:
- AdditionalProperty: An additional property for a CodecValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a CodecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class SpecValue(_messages.Message):
- """The sink to write to, plus its parameters.
-
- Messages:
- AdditionalProperty: An additional property for a SpecValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a SpecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- codec = _messages.MessageField('CodecValue', 1)
- spec = _messages.MessageField('SpecValue', 2)
-
-
-class Source(_messages.Message):
- """A source that records can be read and decoded from.
-
- Messages:
- BaseSpecsValueListEntry: A BaseSpecsValueListEntry object.
- CodecValue: The codec to use to decode data read from the source.
- SpecValue: The source to read from, plus its parameters.
-
- Fields:
- baseSpecs: While splitting, sources may specify the produced bundles as
- differences against another source, in order to save backend-side memory
- and allow bigger jobs. For details, see SourceSplitRequest. To support
- this use case, the full set of parameters of the source is logically
- obtained by taking the latest explicitly specified value of each
- parameter in the order: base_specs (later items win), spec (overrides
- anything in base_specs).
- codec: The codec to use to decode data read from the source.
- doesNotNeedSplitting: Setting this value to true hints to the framework
- that the source doesn't need splitting, and using SourceSplitRequest on
- it would yield SOURCE_SPLIT_OUTCOME_USE_CURRENT. E.g. a file splitter
- may set this to true when splitting a single file into a set of byte
- ranges of appropriate size, and set this to false when splitting a
- filepattern into individual files. However, for efficiency, a file
- splitter may decide to produce file subranges directly from the
- filepattern to avoid a splitting round-trip. See SourceSplitRequest for
- an overview of the splitting process. This field is meaningful only in
- the Source objects populated by the user (e.g. when filling in a
- DerivedSource). Source objects supplied by the framework to the user
- don't have this field populated.
- metadata: Optionally, metadata for this source can be supplied right away,
- avoiding a SourceGetMetadataOperation roundtrip (see
- SourceOperationRequest). This field is meaningful only in the Source
- objects populated by the user (e.g. when filling in a DerivedSource).
- Source objects supplied by the framework to the user don't have this
- field populated.
- spec: The source to read from, plus its parameters.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class BaseSpecsValueListEntry(_messages.Message):
- """A BaseSpecsValueListEntry object.
-
- Messages:
- AdditionalProperty: An additional property for a BaseSpecsValueListEntry
- object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a BaseSpecsValueListEntry object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class CodecValue(_messages.Message):
- """The codec to use to decode data read from the source.
-
- Messages:
- AdditionalProperty: An additional property for a CodecValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a CodecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class SpecValue(_messages.Message):
- """The source to read from, plus its parameters.
-
- Messages:
- AdditionalProperty: An additional property for a SpecValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a SpecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- baseSpecs = _messages.MessageField('BaseSpecsValueListEntry', 1, repeated=True)
- codec = _messages.MessageField('CodecValue', 2)
- doesNotNeedSplitting = _messages.BooleanField(3)
- metadata = _messages.MessageField('SourceMetadata', 4)
- spec = _messages.MessageField('SpecValue', 5)
-
-
-class SourceFork(_messages.Message):
- """DEPRECATED in favor of DynamicSourceSplit.
-
- Fields:
- primary: DEPRECATED
- primarySource: DEPRECATED
- residual: DEPRECATED
- residualSource: DEPRECATED
- """
-
- primary = _messages.MessageField('SourceSplitShard', 1)
- primarySource = _messages.MessageField('DerivedSource', 2)
- residual = _messages.MessageField('SourceSplitShard', 3)
- residualSource = _messages.MessageField('DerivedSource', 4)
-
-
-class SourceGetMetadataRequest(_messages.Message):
- """A request to compute the SourceMetadata of a Source.
-
- Fields:
- source: Specification of the source whose metadata should be computed.
- """
-
- source = _messages.MessageField('Source', 1)
-
-
-class SourceGetMetadataResponse(_messages.Message):
- """The result of a SourceGetMetadataOperation.
-
- Fields:
- metadata: The computed metadata.
- """
-
- metadata = _messages.MessageField('SourceMetadata', 1)
-
-
-class SourceMetadata(_messages.Message):
- """Metadata about a Source useful for automatically optimizing and tuning
- the pipeline, etc.
-
- Fields:
- estimatedSizeBytes: An estimate of the total size (in bytes) of the data
- that would be read from this source. This estimate is in terms of
- external storage size, before any decompression or other processing done
- by the reader.
- infinite: Specifies that the size of this source is known to be infinite
- (this is a streaming source).
- producesSortedKeys: Whether this source is known to produce key/value
- pairs with the (encoded) keys in lexicographically sorted order.
- """
-
- estimatedSizeBytes = _messages.IntegerField(1)
- infinite = _messages.BooleanField(2)
- producesSortedKeys = _messages.BooleanField(3)
-
-
-class SourceOperationRequest(_messages.Message):
- """A work item that represents the different operations that can be
- performed on a user-defined Source specification.
-
- Fields:
- getMetadata: Information about a request to get metadata about a source.
- split: Information about a request to split a source.
- """
-
- getMetadata = _messages.MessageField('SourceGetMetadataRequest', 1)
- split = _messages.MessageField('SourceSplitRequest', 2)
-
-
-class SourceOperationResponse(_messages.Message):
- """The result of a SourceOperationRequest, specified in
- ReportWorkItemStatusRequest.source_operation when the work item is
- completed.
-
- Fields:
- getMetadata: A response to a request to get metadata about a source.
- split: A response to a request to split a source.
- """
-
- getMetadata = _messages.MessageField('SourceGetMetadataResponse', 1)
- split = _messages.MessageField('SourceSplitResponse', 2)
-
-
-class SourceSplitOptions(_messages.Message):
- """Hints for splitting a Source into bundles (parts for parallel processing)
- using SourceSplitRequest.
-
- Fields:
- desiredBundleSizeBytes: The source should be split into a set of bundles
- where the estimated size of each is approximately this many bytes.
- desiredShardSizeBytes: DEPRECATED in favor of desired_bundle_size_bytes.
- """
-
- desiredBundleSizeBytes = _messages.IntegerField(1)
- desiredShardSizeBytes = _messages.IntegerField(2)
-
-
-class SourceSplitRequest(_messages.Message):
- """Represents the operation to split a high-level Source specification into
- bundles (parts for parallel processing). At a high level, splitting of a
- source into bundles happens as follows: SourceSplitRequest is applied to the
- source. If it returns SOURCE_SPLIT_OUTCOME_USE_CURRENT, no further splitting
- happens and the source is used "as is". Otherwise, splitting is applied
- recursively to each produced DerivedSource. As an optimization, for any
- Source, if its does_not_need_splitting is true, the framework assumes that
- splitting this source would return SOURCE_SPLIT_OUTCOME_USE_CURRENT, and
- doesn't initiate a SourceSplitRequest. This applies both to the initial
- source being split and to bundles produced from it.
-
- Fields:
- options: Hints for tuning the splitting process.
- source: Specification of the source to be split.
- """
-
- options = _messages.MessageField('SourceSplitOptions', 1)
- source = _messages.MessageField('Source', 2)
-
-
-class SourceSplitResponse(_messages.Message):
- """The response to a SourceSplitRequest.
-
- Enums:
- OutcomeValueValuesEnum: Indicates whether splitting happened and produced
- a list of bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current
- source should be processed "as is" without splitting. "bundles" is
- ignored in this case. If this is SPLITTING_HAPPENED, then "bundles"
- contains a list of bundles into which the source was split.
-
- Fields:
- bundles: If outcome is SPLITTING_HAPPENED, then this is a list of bundles
- into which the source was split. Otherwise this field is ignored. This
- list can be empty, which means the source represents an empty input.
- outcome: Indicates whether splitting happened and produced a list of
- bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current source should
- be processed "as is" without splitting. "bundles" is ignored in this
- case. If this is SPLITTING_HAPPENED, then "bundles" contains a list of
- bundles into which the source was split.
- shards: DEPRECATED in favor of bundles.
- """
-
- class OutcomeValueValuesEnum(_messages.Enum):
- """Indicates whether splitting happened and produced a list of bundles. If
- this is USE_CURRENT_SOURCE_AS_IS, the current source should be processed
- "as is" without splitting. "bundles" is ignored in this case. If this is
- SPLITTING_HAPPENED, then "bundles" contains a list of bundles into which
- the source was split.
-
- Values:
- SOURCE_SPLIT_OUTCOME_UNKNOWN: <no description>
- SOURCE_SPLIT_OUTCOME_USE_CURRENT: <no description>
- SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED: <no description>
- """
- SOURCE_SPLIT_OUTCOME_UNKNOWN = 0
- SOURCE_SPLIT_OUTCOME_USE_CURRENT = 1
- SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED = 2
-
- bundles = _messages.MessageField('DerivedSource', 1, repeated=True)
- outcome = _messages.EnumField('OutcomeValueValuesEnum', 2)
- shards = _messages.MessageField('SourceSplitShard', 3, repeated=True)
-
-
-class SourceSplitShard(_messages.Message):
- """DEPRECATED in favor of DerivedSource.
-
- Enums:
- DerivationModeValueValuesEnum: DEPRECATED
-
- Fields:
- derivationMode: DEPRECATED
- source: DEPRECATED
- """
-
- class DerivationModeValueValuesEnum(_messages.Enum):
- """DEPRECATED
-
- Values:
- SOURCE_DERIVATION_MODE_UNKNOWN: <no description>
- SOURCE_DERIVATION_MODE_INDEPENDENT: <no description>
- SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: <no description>
- SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: <no description>
- """
- SOURCE_DERIVATION_MODE_UNKNOWN = 0
- SOURCE_DERIVATION_MODE_INDEPENDENT = 1
- SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2
- SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3
-
- derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1)
- source = _messages.MessageField('Source', 2)
-
-
-class StandardQueryParameters(_messages.Message):
- """Query parameters accepted by all methods.
-
- Fields:
- f__xgafv: V1 error format.
- access_token: OAuth access token.
- alt: Data format for response.
- bearer_token: OAuth bearer token.
- callback: JSONP
- fields: Selector specifying which fields to include in a partial response.
- key: API key. Your API key identifies your project and provides you with
- API access, quota, and reports. Required unless you provide an OAuth 2.0
- token.
- oauth_token: OAuth 2.0 token for the current user.
- pp: Pretty-print response.
- prettyPrint: Returns response with indentations and line breaks.
- quotaUser: Available to use for quota purposes for server-side
- applications. Can be any arbitrary string assigned to a user, but should
- not exceed 40 characters.
- trace: A tracing token of the form "token:<tokenid>" to include in api
- requests.
- uploadType: Legacy upload protocol for media (e.g. "media", "multipart").
- upload_protocol: Upload protocol for media (e.g. "raw", "multipart").
- """
-
- f__xgafv = _messages.StringField(1)
- access_token = _messages.StringField(2)
- alt = _messages.StringField(3, default=u'json')
- bearer_token = _messages.StringField(4)
- callback = _messages.StringField(5)
- fields = _messages.StringField(6)
- key = _messages.StringField(7)
- oauth_token = _messages.StringField(8)
- pp = _messages.BooleanField(9, default=True)
- prettyPrint = _messages.BooleanField(10, default=True)
- quotaUser = _messages.StringField(11)
- trace = _messages.StringField(12)
- uploadType = _messages.StringField(13)
- upload_protocol = _messages.StringField(14)
-
-
-class StateFamilyConfig(_messages.Message):
- """State family configuration.
-
- Fields:
- isRead: If true, this family corresponds to a read operation.
- stateFamily: The state family value.
- """
-
- isRead = _messages.BooleanField(1)
- stateFamily = _messages.StringField(2)
-
-
-class Status(_messages.Message):
- """The `Status` type defines a logical error model that is suitable for
- different programming environments, including REST APIs and RPC APIs. It is
- used by [gRPC](https://github.com/grpc). The error model is designed to be:
- - Simple to use and understand for most users - Flexible enough to meet
- unexpected needs # Overview The `Status` message contains three pieces of
- data: error code, error message, and error details. The error code should be
- an enum value of google.rpc.Code, but it may accept additional error codes
- if needed. The error message should be a developer-facing English message
- that helps developers *understand* and *resolve* the error. If a localized
- user-facing error message is needed, put the localized message in the error
- details or localize it in the client. The optional error details may contain
- arbitrary information about the error. There is a predefined set of error
- detail types in the package `google.rpc` which can be used for common error
- conditions. # Language mapping The `Status` message is the logical
- representation of the error model, but it is not necessarily the actual wire
- format. When the `Status` message is exposed in different client libraries
- and different wire protocols, it can be mapped differently. For example, it
- will likely be mapped to some exceptions in Java, but more likely mapped to
- some error codes in C. # Other uses The error model and the `Status` message
- can be used in a variety of environments, either with or without APIs, to
- provide a consistent developer experience across different environments.
- Example uses of this error model include: - Partial errors. If a service
- needs to return partial errors to the client, it may embed the `Status` in
- the normal response to indicate the partial errors. - Workflow errors. A
- typical workflow has multiple steps. Each step may have a `Status` message
- for error reporting purpose. - Batch operations. If a client uses batch
- request and batch response, the `Status` message should be used directly
- inside batch response, one for each error sub-response. - Asynchronous
- operations. If an API call embeds asynchronous operation results in its
- response, the status of those operations should be represented directly
- using the `Status` message. - Logging. If some API errors are stored in
- logs, the message `Status` could be used directly after any stripping needed
- for security/privacy reasons.
-
- Messages:
- DetailsValueListEntry: A DetailsValueListEntry object.
-
- Fields:
- code: The status code, which should be an enum value of google.rpc.Code.
- details: A list of messages that carry the error details. There will be a
- common set of message types for APIs to use.
- message: A developer-facing error message, which should be in English. Any
- user-facing error message should be localized and sent in the
- google.rpc.Status.details field, or localized by the client.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class DetailsValueListEntry(_messages.Message):
- """A DetailsValueListEntry object.
-
- Messages:
- AdditionalProperty: An additional property for a DetailsValueListEntry
- object.
-
- Fields:
- additionalProperties: Properties of the object. Contains field @ype with
- type URL.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a DetailsValueListEntry object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- code = _messages.IntegerField(1, variant=_messages.Variant.INT32)
- details = _messages.MessageField('DetailsValueListEntry', 2, repeated=True)
- message = _messages.StringField(3)
-
-
-class Step(_messages.Message):
- """Defines a particular step within a Dataflow job. A job consists of
- multiple steps, each of which performs some specific operation as part of
- the overall job. Data is typically passed from one step to another as part
- of the job. Here's an example of a sequence of steps which together
- implement a Map-Reduce job: * Read a collection of data from some source,
- parsing the collection's elements. * Validate the elements. * Apply a user-
- defined function to map each element to some value and extract an element-
- specific key value. * Group elements with the same key into a single element
- with that key, transforming a multiply-keyed collection into a uniquely-
- keyed collection. * Write the elements out to some data sink. (Note that the
- Dataflow service may be used to run many different types of jobs, not just
- Map-Reduce).
-
- Messages:
- PropertiesValue: Named properties associated with the step. Each kind of
- predefined step has its own required set of properties.
-
- Fields:
- kind: The kind of step in the dataflow Job.
- name: Name identifying the step. This must be unique for each step with
- respect to all other steps in the dataflow Job.
- properties: Named properties associated with the step. Each kind of
- predefined step has its own required set of properties.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class PropertiesValue(_messages.Message):
- """Named properties associated with the step. Each kind of predefined step
- has its own required set of properties.
-
- Messages:
- AdditionalProperty: An additional property for a PropertiesValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a PropertiesValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- kind = _messages.StringField(1)
- name = _messages.StringField(2)
- properties = _messages.MessageField('PropertiesValue', 3)
-
-
-class StreamLocation(_messages.Message):
- """Describes a stream of data, either as input to be processed or as output
- of a streaming Dataflow job.
-
- Fields:
- customSourceLocation: The stream is a custom source.
- pubsubLocation: The stream is a pubsub stream.
- sideInputLocation: The stream is a streaming side input.
- streamingStageLocation: The stream is part of another computation within
- the current streaming Dataflow job.
- """
-
- customSourceLocation = _messages.MessageField('CustomSourceLocation', 1)
- pubsubLocation = _messages.MessageField('PubsubLocation', 2)
- sideInputLocation = _messages.MessageField('StreamingSideInputLocation', 3)
- streamingStageLocation = _messages.MessageField('StreamingStageLocation', 4)
-
-
-class StreamingComputationRanges(_messages.Message):
- """Describes full or partial data disk assignment information of the
- computation ranges.
-
- Fields:
- computationId: The ID of the computation.
- rangeAssignments: Data disk assignments for ranges from this computation.
- """
-
- computationId = _messages.StringField(1)
- rangeAssignments = _messages.MessageField('KeyRangeDataDiskAssignment', 2, repeated=True)
-
-
-class StreamingComputationTask(_messages.Message):
- """A task which describes what action should be performed for the specified
- streaming computation ranges.
-
- Enums:
- TaskTypeValueValuesEnum: A type of streaming computation task.
-
- Fields:
- computationRanges: Contains ranges of a streaming computation this task
- should apply to.
- dataDisks: Describes the set of data disks this task should apply to.
- taskType: A type of streaming computation task.
- """
-
- class TaskTypeValueValuesEnum(_messages.Enum):
- """A type of streaming computation task.
-
- Values:
- STREAMING_COMPUTATION_TASK_UNKNOWN: <no description>
- STREAMING_COMPUTATION_TASK_STOP: <no description>
- STREAMING_COMPUTATION_TASK_START: <no description>
- """
- STREAMING_COMPUTATION_TASK_UNKNOWN = 0
- STREAMING_COMPUTATION_TASK_STOP = 1
- STREAMING_COMPUTATION_TASK_START = 2
-
- computationRanges = _messages.MessageField('StreamingComputationRanges', 1, repeated=True)
- dataDisks = _messages.MessageField('MountedDataDisk', 2, repeated=True)
- taskType = _messages.EnumField('TaskTypeValueValuesEnum', 3)
-
-
-class StreamingSetupTask(_messages.Message):
- """A task which initializes part of a streaming Dataflow job.
-
- Fields:
- drain: The user has requested drain.
- receiveWorkPort: The TCP port on which the worker should listen for
- messages from other streaming computation workers.
- streamingComputationTopology: T
<TRUNCATED>