You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2016/06/14 23:13:17 UTC
[42/50] [abbrv] incubator-beam git commit: Move all files to
apache_beam folder
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b14dfadd/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py
new file mode 100644
index 0000000..2e0273f
--- /dev/null
+++ b/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py
@@ -0,0 +1,3056 @@
+"""Generated message classes for dataflow version v1b3.
+
+Develops and executes data processing patterns like ETL, batch computation,
+and continuous computation.
+"""
+# NOTE: This file is autogenerated and should not be edited by hand.
+
+from apitools.base.protorpclite import messages as _messages
+from apitools.base.py import encoding
+
+
+package = 'dataflow'
+
+
+class ApproximateProgress(_messages.Message):
+ """Obsolete in favor of ApproximateReportedProgress and
+ ApproximateSplitRequest.
+
+ Fields:
+ percentComplete: Obsolete.
+ position: Obsolete.
+ remainingTime: Obsolete.
+ """
+
+ percentComplete = _messages.FloatField(1, variant=_messages.Variant.FLOAT)
+ position = _messages.MessageField('Position', 2)
+ remainingTime = _messages.StringField(3)
+
+
+class ApproximateReportedProgress(_messages.Message):
+ """A progress measurement of a WorkItem by a worker.
+
+ Fields:
+ consumedParallelism: Total amount of parallelism in the portion of input
+ of this work item that has already been consumed. In the first two
+ examples above (see remaining_parallelism), the value should be 30 or 3
+ respectively. The sum of remaining_parallelism and consumed_parallelism
+ should equal the total amount of parallelism in this work item. If
+ specified, must be finite.
+ fractionConsumed: Completion as fraction of the input consumed, from 0.0
+ (beginning, nothing consumed), to 1.0 (end of the input, entire input
+ consumed).
+ position: A Position within the work to represent a progress.
+ remainingParallelism: Total amount of parallelism in the input of this
+ WorkItem that has not been consumed yet (i.e. can be delegated to a new
+ WorkItem via dynamic splitting). "Amount of parallelism" refers to how
+ many non-empty parts of the input can be read in parallel. This does not
+ necessarily equal number of records. An input that can be read in
+ parallel down to the individual records is called "perfectly
+ splittable". An example of non-perfectly parallelizable input is a
+ block-compressed file format where a block of records has to be read as
+ a whole, but different blocks can be read in parallel. Examples: * If we
+ have read 30 records out of 50 in a perfectly splittable 50-record
+ input, this value should be 20. * If we are reading through block 3 in a
+ block-compressed file consisting of 5 blocks, this value should be 2
+ (since blocks 4 and 5 can be processed in parallel by new work items via
+ dynamic splitting). * If we are reading through the last block in a
+ block-compressed file, or reading or processing the last record in a
+ perfectly splittable input, this value should be 0, because the
+ remainder of the work item cannot be further split.
+ """
+
+ consumedParallelism = _messages.MessageField('ReportedParallelism', 1)
+ fractionConsumed = _messages.FloatField(2)
+ position = _messages.MessageField('Position', 3)
+ remainingParallelism = _messages.MessageField('ReportedParallelism', 4)
+
+
+class ApproximateSplitRequest(_messages.Message):
+ """A suggestion by the service to the worker to dynamically split the
+ WorkItem.
+
+ Fields:
+ fractionConsumed: A fraction at which to split the work item, from 0.0
+ (beginning of the input) to 1.0 (end of the input).
+ position: A Position at which to split the work item.
+ """
+
+ fractionConsumed = _messages.FloatField(1)
+ position = _messages.MessageField('Position', 2)
+
+
+class AutoscalingSettings(_messages.Message):
+ """Settings for WorkerPool autoscaling.
+
+ Enums:
+ AlgorithmValueValuesEnum: The algorithm to use for autoscaling.
+
+ Fields:
+ algorithm: The algorithm to use for autoscaling.
+ maxNumWorkers: The maximum number of workers to cap scaling at.
+ """
+
+ class AlgorithmValueValuesEnum(_messages.Enum):
+ """The algorithm to use for autoscaling.
+
+ Values:
+ AUTOSCALING_ALGORITHM_UNKNOWN: <no description>
+ AUTOSCALING_ALGORITHM_NONE: <no description>
+ AUTOSCALING_ALGORITHM_BASIC: <no description>
+ """
+ AUTOSCALING_ALGORITHM_UNKNOWN = 0
+ AUTOSCALING_ALGORITHM_NONE = 1
+ AUTOSCALING_ALGORITHM_BASIC = 2
+
+ algorithm = _messages.EnumField('AlgorithmValueValuesEnum', 1)
+ maxNumWorkers = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+
+
+class ComputationTopology(_messages.Message):
+ """All configuration data for a particular Computation.
+
+ Fields:
+ computationId: The ID of the computation.
+ inputs: The inputs to the computation.
+ keyRanges: The key ranges processed by the computation.
+ outputs: The outputs from the computation.
+ stateFamilies: The state family values.
+ systemStageName: The system stage name.
+ userStageName: The user stage name.
+ """
+
+ computationId = _messages.StringField(1)
+ inputs = _messages.MessageField('StreamLocation', 2, repeated=True)
+ keyRanges = _messages.MessageField('KeyRangeLocation', 3, repeated=True)
+ outputs = _messages.MessageField('StreamLocation', 4, repeated=True)
+ stateFamilies = _messages.MessageField('StateFamilyConfig', 5, repeated=True)
+ systemStageName = _messages.StringField(6)
+ userStageName = _messages.StringField(7)
+
+
+class ConcatPosition(_messages.Message):
+ """A position that encapsulates an inner position and an index for the inner
+ position. A ConcatPosition can be used by a reader of a source that
+ encapsulates a set of other sources.
+
+ Fields:
+ index: Index of the inner source.
+ position: Position within the inner source.
+ """
+
+ index = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+ position = _messages.MessageField('Position', 2)
+
+
+class CustomSourceLocation(_messages.Message):
+ """Identifies the location of a custom souce.
+
+ Fields:
+ stateful: Whether this source is stateful.
+ """
+
+ stateful = _messages.BooleanField(1)
+
+
+class DataDiskAssignment(_messages.Message):
+ """Data disk assignment for a given VM instance.
+
+ Fields:
+ dataDisks: Mounted data disks. The order is important a data disk's
+ 0-based index in this list defines which persistent directory the disk
+ is mounted to, for example the list of {
+ "myproject-1014-104817-4c2-harness-0-disk-0" }, {
+ "myproject-1014-104817-4c2-harness-0-disk-1" }.
+ vmInstance: VM instance name the data disks mounted to, for example
+ "myproject-1014-104817-4c2-harness-0".
+ """
+
+ dataDisks = _messages.StringField(1, repeated=True)
+ vmInstance = _messages.StringField(2)
+
+
+class DataflowProjectsJobsCreateRequest(_messages.Message):
+ """A DataflowProjectsJobsCreateRequest object.
+
+ Enums:
+ ViewValueValuesEnum: Level of information requested in response.
+
+ Fields:
+ job: A Job resource to be passed as the request body.
+ projectId: The project which owns the job.
+ replaceJobId: DEPRECATED. This field is now on the Job message.
+ view: Level of information requested in response.
+ """
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """Level of information requested in response.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ job = _messages.MessageField('Job', 1)
+ projectId = _messages.StringField(2, required=True)
+ replaceJobId = _messages.StringField(3)
+ view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class DataflowProjectsJobsGetMetricsRequest(_messages.Message):
+ """A DataflowProjectsJobsGetMetricsRequest object.
+
+ Fields:
+ jobId: The job to get messages for.
+ projectId: A project id.
+ startTime: Return only metric data that has changed since this time.
+ Default is to return all information about all metrics for the job.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ startTime = _messages.StringField(3)
+
+
+class DataflowProjectsJobsGetRequest(_messages.Message):
+ """A DataflowProjectsJobsGetRequest object.
+
+ Enums:
+ ViewValueValuesEnum: Level of information requested in response.
+
+ Fields:
+ jobId: Identifies a single job.
+ projectId: The project which owns the job.
+ view: Level of information requested in response.
+ """
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """Level of information requested in response.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ jobId = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ view = _messages.EnumField('ViewValueValuesEnum', 3)
+
+
+class DataflowProjectsJobsListRequest(_messages.Message):
+ """A DataflowProjectsJobsListRequest object.
+
+ Enums:
+ FilterValueValuesEnum: The kind of filter to use.
+ ViewValueValuesEnum: Level of information requested in response. Default
+ is SUMMARY.
+
+ Fields:
+ filter: The kind of filter to use.
+ pageSize: If there are many jobs, limit response to at most this many. The
+ actual number of jobs returned will be the lesser of max_responses and
+ an unspecified server-defined limit.
+ pageToken: Set this to the 'next_page_token' field of a previous response
+ to request additional results in a long list.
+ projectId: The project which owns the jobs.
+ view: Level of information requested in response. Default is SUMMARY.
+ """
+
+ class FilterValueValuesEnum(_messages.Enum):
+ """The kind of filter to use.
+
+ Values:
+ ALL: <no description>
+ TERMINATED: <no description>
+ ACTIVE: <no description>
+ """
+ ALL = 0
+ TERMINATED = 1
+ ACTIVE = 2
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """Level of information requested in response. Default is SUMMARY.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ filter = _messages.EnumField('FilterValueValuesEnum', 1)
+ pageSize = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+ pageToken = _messages.StringField(3)
+ projectId = _messages.StringField(4, required=True)
+ view = _messages.EnumField('ViewValueValuesEnum', 5)
+
+
+class DataflowProjectsJobsMessagesListRequest(_messages.Message):
+ """A DataflowProjectsJobsMessagesListRequest object.
+
+ Enums:
+ MinimumImportanceValueValuesEnum: Filter to only get messages with
+ importance >= level
+
+ Fields:
+ endTime: Return only messages with timestamps < end_time. The default is
+ now (i.e. return up to the latest messages available).
+ jobId: The job to get messages about.
+ minimumImportance: Filter to only get messages with importance >= level
+ pageSize: If specified, determines the maximum number of messages to
+ return. If unspecified, the service may choose an appropriate default,
+ or may return an arbitrarily large number of results.
+ pageToken: If supplied, this should be the value of next_page_token
+ returned by an earlier call. This will cause the next page of results to
+ be returned.
+ projectId: A project id.
+ startTime: If specified, return only messages with timestamps >=
+ start_time. The default is the job creation time (i.e. beginning of
+ messages).
+ """
+
+ class MinimumImportanceValueValuesEnum(_messages.Enum):
+ """Filter to only get messages with importance >= level
+
+ Values:
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
+ JOB_MESSAGE_DEBUG: <no description>
+ JOB_MESSAGE_DETAILED: <no description>
+ JOB_MESSAGE_BASIC: <no description>
+ JOB_MESSAGE_WARNING: <no description>
+ JOB_MESSAGE_ERROR: <no description>
+ """
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
+ JOB_MESSAGE_DEBUG = 1
+ JOB_MESSAGE_DETAILED = 2
+ JOB_MESSAGE_BASIC = 3
+ JOB_MESSAGE_WARNING = 4
+ JOB_MESSAGE_ERROR = 5
+
+ endTime = _messages.StringField(1)
+ jobId = _messages.StringField(2, required=True)
+ minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 3)
+ pageSize = _messages.IntegerField(4, variant=_messages.Variant.INT32)
+ pageToken = _messages.StringField(5)
+ projectId = _messages.StringField(6, required=True)
+ startTime = _messages.StringField(7)
+
+
+class DataflowProjectsJobsUpdateRequest(_messages.Message):
+ """A DataflowProjectsJobsUpdateRequest object.
+
+ Fields:
+ job: A Job resource to be passed as the request body.
+ jobId: Identifies a single job.
+ projectId: The project which owns the job.
+ """
+
+ job = _messages.MessageField('Job', 1)
+ jobId = _messages.StringField(2, required=True)
+ projectId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsJobsWorkItemsLeaseRequest(_messages.Message):
+ """A DataflowProjectsJobsWorkItemsLeaseRequest object.
+
+ Fields:
+ jobId: Identifies the workflow job this worker belongs to.
+ leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the
+ request body.
+ projectId: Identifies the project this worker belongs to.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2)
+ projectId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsJobsWorkItemsReportStatusRequest(_messages.Message):
+ """A DataflowProjectsJobsWorkItemsReportStatusRequest object.
+
+ Fields:
+ jobId: The job which the WorkItem is part of.
+ projectId: The project which owns the WorkItem's job.
+ reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be
+ passed as the request body.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 3)
+
+
+class DataflowProjectsWorkerMessagesRequest(_messages.Message):
+ """A DataflowProjectsWorkerMessagesRequest object.
+
+ Fields:
+ projectId: The project to send the WorkerMessages to.
+ sendWorkerMessagesRequest: A SendWorkerMessagesRequest resource to be
+ passed as the request body.
+ """
+
+ projectId = _messages.StringField(1, required=True)
+ sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 2)
+
+
+class DerivedSource(_messages.Message):
+ """Specification of one of the bundles produced as a result of splitting a
+ Source (e.g. when executing a SourceSplitRequest, or when splitting an
+ active task using WorkItemStatus.dynamic_source_split), relative to the
+ source being split.
+
+ Enums:
+ DerivationModeValueValuesEnum: What source to base the produced source on
+ (if any).
+
+ Fields:
+ derivationMode: What source to base the produced source on (if any).
+ source: Specification of the source.
+ """
+
+ class DerivationModeValueValuesEnum(_messages.Enum):
+ """What source to base the produced source on (if any).
+
+ Values:
+ SOURCE_DERIVATION_MODE_UNKNOWN: <no description>
+ SOURCE_DERIVATION_MODE_INDEPENDENT: <no description>
+ SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: <no description>
+ SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: <no description>
+ """
+ SOURCE_DERIVATION_MODE_UNKNOWN = 0
+ SOURCE_DERIVATION_MODE_INDEPENDENT = 1
+ SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2
+ SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3
+
+ derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1)
+ source = _messages.MessageField('Source', 2)
+
+
+class Disk(_messages.Message):
+ """Describes the data disk used by a workflow job.
+
+ Fields:
+ diskType: Disk storage type, as defined by Google Compute Engine. This
+ must be a disk type appropriate to the project and zone in which the
+ workers will run. If unknown or unspecified, the service will attempt to
+ choose a reasonable default. For example, the standard persistent disk
+ type is a resource name typically ending in "pd-standard". If SSD
+ persistent disks are available, the resource name typically ends with
+ "pd-ssd". The actual valid values are defined the Google Compute Engine
+ API, not by the Dataflow API; consult the Google Compute Engine
+ documentation for more information about determining the set of
+ available disk types for a particular project and zone. Google Compute
+ Engine Disk types are local to a particular project in a particular
+ zone, and so the resource name will typically look something like this:
+ compute.googleapis.com/projects/ /zones//diskTypes/pd-standard
+ mountPoint: Directory in a VM where disk is mounted.
+ sizeGb: Size of disk in GB. If zero or unspecified, the service will
+ attempt to choose a reasonable default.
+ """
+
+ diskType = _messages.StringField(1)
+ mountPoint = _messages.StringField(2)
+ sizeGb = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+
+
+class DynamicSourceSplit(_messages.Message):
+ """When a task splits using WorkItemStatus.dynamic_source_split, this
+ message describes the two parts of the split relative to the description of
+ the current task's input.
+
+ Fields:
+ primary: Primary part (continued to be processed by worker). Specified
+ relative to the previously-current source. Becomes current.
+ residual: Residual part (returned to the pool of work). Specified relative
+ to the previously-current source.
+ """
+
+ primary = _messages.MessageField('DerivedSource', 1)
+ residual = _messages.MessageField('DerivedSource', 2)
+
+
+class Environment(_messages.Message):
+ """Describes the environment in which a Dataflow Job runs.
+
+ Messages:
+ InternalExperimentsValue: Experimental settings.
+ SdkPipelineOptionsValue: The Dataflow SDK pipeline options specified by
+ the user. These options are passed through the service and are used to
+ recreate the SDK pipeline options on the worker in a language agnostic
+ and platform independent way.
+ UserAgentValue: A description of the process that generated the request.
+ VersionValue: A structure describing which components and their versions
+ of the service are required in order to run the job.
+
+ Fields:
+ clusterManagerApiService: The type of cluster manager API to use. If
+ unknown or unspecified, the service will attempt to choose a reasonable
+ default. This should be in the form of the API service name, e.g.
+ "compute.googleapis.com".
+ dataset: The dataset for the current project where various workflow
+ related tables are stored. The supported resource type is: Google
+ BigQuery: bigquery.googleapis.com/{dataset}
+ experiments: The list of experiments to enable.
+ internalExperiments: Experimental settings.
+ sdkPipelineOptions: The Dataflow SDK pipeline options specified by the
+ user. These options are passed through the service and are used to
+ recreate the SDK pipeline options on the worker in a language agnostic
+ and platform independent way.
+ tempStoragePrefix: The prefix of the resources the system should use for
+ temporary storage. The system will append the suffix "/temp-{JOBNAME} to
+ this resource prefix, where {JOBNAME} is the value of the job_name
+ field. The resulting bucket and object prefix is used as the prefix of
+ the resources used to store temporary data needed during the job
+ execution. NOTE: This will override the value in taskrunner_settings.
+ The supported resource type is: Google Cloud Storage:
+ storage.googleapis.com/{bucket}/{object}
+ bucket.storage.googleapis.com/{object}
+ userAgent: A description of the process that generated the request.
+ version: A structure describing which components and their versions of the
+ service are required in order to run the job.
+ workerPools: Worker pools. At least one "harness" worker pool must be
+ specified in order for the job to have workers.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class InternalExperimentsValue(_messages.Message):
+ """Experimental settings.
+
+ Messages:
+ AdditionalProperty: An additional property for a
+ InternalExperimentsValue object.
+
+ Fields:
+ additionalProperties: Properties of the object. Contains field @ype with
+ type URL.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a InternalExperimentsValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class SdkPipelineOptionsValue(_messages.Message):
+ """The Dataflow SDK pipeline options specified by the user. These options
+ are passed through the service and are used to recreate the SDK pipeline
+ options on the worker in a language agnostic and platform independent way.
+
+ Messages:
+ AdditionalProperty: An additional property for a SdkPipelineOptionsValue
+ object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a SdkPipelineOptionsValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class UserAgentValue(_messages.Message):
+ """A description of the process that generated the request.
+
+ Messages:
+ AdditionalProperty: An additional property for a UserAgentValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a UserAgentValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class VersionValue(_messages.Message):
+ """A structure describing which components and their versions of the
+ service are required in order to run the job.
+
+ Messages:
+ AdditionalProperty: An additional property for a VersionValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a VersionValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ clusterManagerApiService = _messages.StringField(1)
+ dataset = _messages.StringField(2)
+ experiments = _messages.StringField(3, repeated=True)
+ internalExperiments = _messages.MessageField('InternalExperimentsValue', 4)
+ sdkPipelineOptions = _messages.MessageField('SdkPipelineOptionsValue', 5)
+ tempStoragePrefix = _messages.StringField(6)
+ userAgent = _messages.MessageField('UserAgentValue', 7)
+ version = _messages.MessageField('VersionValue', 8)
+ workerPools = _messages.MessageField('WorkerPool', 9, repeated=True)
+
+
+class FlattenInstruction(_messages.Message):
+ """An instruction that copies its inputs (zero or more) to its (single)
+ output.
+
+ Fields:
+ inputs: Describes the inputs to the flatten instruction.
+ """
+
+ inputs = _messages.MessageField('InstructionInput', 1, repeated=True)
+
+
+class InstructionInput(_messages.Message):
+ """An input of an instruction, as a reference to an output of a producer
+ instruction.
+
+ Fields:
+ outputNum: The output index (origin zero) within the producer.
+ producerInstructionIndex: The index (origin zero) of the parallel
+ instruction that produces the output to be consumed by this input. This
+ index is relative to the list of instructions in this input's
+ instruction's containing MapTask.
+ """
+
+ outputNum = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+ producerInstructionIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+
+
+class InstructionOutput(_messages.Message):
+ """An output of an instruction.
+
+ Messages:
+ CodecValue: The codec to use to encode data being written via this output.
+
+ Fields:
+ codec: The codec to use to encode data being written via this output.
+ name: The user-provided name of this output.
+ systemName: System-defined name of this output. Unique across the
+ workflow.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class CodecValue(_messages.Message):
+ """The codec to use to encode data being written via this output.
+
+ Messages:
+ AdditionalProperty: An additional property for a CodecValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a CodecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ codec = _messages.MessageField('CodecValue', 1)
+ name = _messages.StringField(2)
+ systemName = _messages.StringField(3)
+
+
+class Job(_messages.Message):
+ """Defines a job to be run by the Dataflow service.
+
+ Enums:
+ CurrentStateValueValuesEnum: The current state of the job. Jobs are
+ created in the JOB_STATE_STOPPED state unless otherwise specified. A job
+ in the JOB_STATE_RUNNING state may asynchronously enter a terminal
+ state. Once a job has reached a terminal state, no further state updates
+ may be made. This field may be mutated by the Dataflow service; callers
+ cannot mutate it.
+ RequestedStateValueValuesEnum: The job's requested state. UpdateJob may be
+ used to switch between the JOB_STATE_STOPPED and JOB_STATE_RUNNING
+ states, by setting requested_state. UpdateJob may also be used to
+ directly set a job's requested state to JOB_STATE_CANCELLED or
+ JOB_STATE_DONE, irrevocably terminating the job if it has not already
+ reached a terminal state.
+ TypeValueValuesEnum: The type of dataflow job.
+
+ Messages:
+ TransformNameMappingValue: Map of transform name prefixes of the job to be
+ replaced to the corresponding name prefixes of the new job.
+
+ Fields:
+ clientRequestId: Client's unique identifier of the job, re-used by SDK
+ across retried attempts. If this field is set, the service will ensure
+ its uniqueness. That is, the request to create a job will fail if the
+ service has knowledge of a previously submitted job with the same
+ client's id and job name. The caller may, for example, use this field to
+ ensure idempotence of job creation across retried attempts to create a
+ job. By default, the field is empty and, in that case, the service
+ ignores it.
+ createTime: Timestamp when job was initially created. Immutable, set by
+ the Dataflow service.
+ currentState: The current state of the job. Jobs are created in the
+ JOB_STATE_STOPPED state unless otherwise specified. A job in the
+ JOB_STATE_RUNNING state may asynchronously enter a terminal state. Once
+ a job has reached a terminal state, no further state updates may be
+ made. This field may be mutated by the Dataflow service; callers cannot
+ mutate it.
+ currentStateTime: The timestamp associated with the current state.
+ environment: Environment for the job.
+ executionInfo: Information about how the Dataflow service will actually
+ run the job.
+ id: The unique ID of this job. This field is set by the Dataflow service
+ when the Job is created, and is immutable for the life of the Job.
+ name: The user-specified Dataflow job name. Only one Job with a given name
+ may exist in a project at any given time. If a caller attempts to create
+ a Job with the same name as an already-existing Job, the attempt will
+ return the existing Job. The name must match the regular expression
+ [a-z]([-a-z0-9]{0,38}[a-z0-9])?
+ projectId: The project which owns the job.
+ replaceJobId: If this job is an update of an existing job, this field will
+ be the ID of the job it replaced. When sending a CreateJobRequest, you
+ can update a job by specifying it here. The job named here will be
+ stopped, and its intermediate state transferred to this job.
+ replacedByJobId: If another job is an update of this job (and thus, this
+ job is in JOB_STATE_UPDATED), this field will contain the ID of that
+ job.
+ requestedState: The job's requested state. UpdateJob may be used to switch
+ between the JOB_STATE_STOPPED and JOB_STATE_RUNNING states, by setting
+ requested_state. UpdateJob may also be used to directly set a job's
+ requested state to JOB_STATE_CANCELLED or JOB_STATE_DONE, irrevocably
+ terminating the job if it has not already reached a terminal state.
+ steps: The top-level steps that constitute the entire job.
+ tempFiles: A set of files the system should be aware of that are used for
+ temporary storage. These temporary files will be removed on job
+ completion. No duplicates are allowed. No file patterns are supported.
+ The supported files are: Google Cloud Storage:
+ storage.googleapis.com/{bucket}/{object}
+ bucket.storage.googleapis.com/{object}
+ transformNameMapping: Map of transform name prefixes of the job to be
+ replaced to the corresponding name prefixes of the new job.
+ type: The type of dataflow job.
+ """
+
+ class CurrentStateValueValuesEnum(_messages.Enum):
+ """The current state of the job. Jobs are created in the JOB_STATE_STOPPED
+ state unless otherwise specified. A job in the JOB_STATE_RUNNING state may
+ asynchronously enter a terminal state. Once a job has reached a terminal
+ state, no further state updates may be made. This field may be mutated by
+ the Dataflow service; callers cannot mutate it.
+
+ Values:
+ JOB_STATE_UNKNOWN: <no description>
+ JOB_STATE_STOPPED: <no description>
+ JOB_STATE_RUNNING: <no description>
+ JOB_STATE_DONE: <no description>
+ JOB_STATE_FAILED: <no description>
+ JOB_STATE_CANCELLED: <no description>
+ JOB_STATE_UPDATED: <no description>
+ JOB_STATE_DRAINING: <no description>
+ JOB_STATE_DRAINED: <no description>
+ """
+ JOB_STATE_UNKNOWN = 0
+ JOB_STATE_STOPPED = 1
+ JOB_STATE_RUNNING = 2
+ JOB_STATE_DONE = 3
+ JOB_STATE_FAILED = 4
+ JOB_STATE_CANCELLED = 5
+ JOB_STATE_UPDATED = 6
+ JOB_STATE_DRAINING = 7
+ JOB_STATE_DRAINED = 8
+
+ class RequestedStateValueValuesEnum(_messages.Enum):
+ """The job's requested state. UpdateJob may be used to switch between the
+ JOB_STATE_STOPPED and JOB_STATE_RUNNING states, by setting
+ requested_state. UpdateJob may also be used to directly set a job's
+ requested state to JOB_STATE_CANCELLED or JOB_STATE_DONE, irrevocably
+ terminating the job if it has not already reached a terminal state.
+
+ Values:
+ JOB_STATE_UNKNOWN: <no description>
+ JOB_STATE_STOPPED: <no description>
+ JOB_STATE_RUNNING: <no description>
+ JOB_STATE_DONE: <no description>
+ JOB_STATE_FAILED: <no description>
+ JOB_STATE_CANCELLED: <no description>
+ JOB_STATE_UPDATED: <no description>
+ JOB_STATE_DRAINING: <no description>
+ JOB_STATE_DRAINED: <no description>
+ """
+ JOB_STATE_UNKNOWN = 0
+ JOB_STATE_STOPPED = 1
+ JOB_STATE_RUNNING = 2
+ JOB_STATE_DONE = 3
+ JOB_STATE_FAILED = 4
+ JOB_STATE_CANCELLED = 5
+ JOB_STATE_UPDATED = 6
+ JOB_STATE_DRAINING = 7
+ JOB_STATE_DRAINED = 8
+
+ class TypeValueValuesEnum(_messages.Enum):
+ """The type of dataflow job.
+
+ Values:
+ JOB_TYPE_UNKNOWN: <no description>
+ JOB_TYPE_BATCH: <no description>
+ JOB_TYPE_STREAMING: <no description>
+ """
+ JOB_TYPE_UNKNOWN = 0
+ JOB_TYPE_BATCH = 1
+ JOB_TYPE_STREAMING = 2
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class TransformNameMappingValue(_messages.Message):
+ """Map of transform name prefixes of the job to be replaced to the
+ corresponding name prefixes of the new job.
+
+ Messages:
+ AdditionalProperty: An additional property for a
+ TransformNameMappingValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type
+ TransformNameMappingValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a TransformNameMappingValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A string attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.StringField(2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ clientRequestId = _messages.StringField(1)
+ createTime = _messages.StringField(2)
+ currentState = _messages.EnumField('CurrentStateValueValuesEnum', 3)
+ currentStateTime = _messages.StringField(4)
+ environment = _messages.MessageField('Environment', 5)
+ executionInfo = _messages.MessageField('JobExecutionInfo', 6)
+ id = _messages.StringField(7)
+ name = _messages.StringField(8)
+ projectId = _messages.StringField(9)
+ replaceJobId = _messages.StringField(10)
+ replacedByJobId = _messages.StringField(11)
+ requestedState = _messages.EnumField('RequestedStateValueValuesEnum', 12)
+ steps = _messages.MessageField('Step', 13, repeated=True)
+ tempFiles = _messages.StringField(14, repeated=True)
+ transformNameMapping = _messages.MessageField('TransformNameMappingValue', 15)
+ type = _messages.EnumField('TypeValueValuesEnum', 16)
+
+
+class JobExecutionInfo(_messages.Message):
+ """Additional information about how a Dataflow job will be executed which
+ isn\u2019t contained in the submitted job.
+
+ Messages:
+ StagesValue: A mapping from each stage to the information about that
+ stage.
+
+ Fields:
+ stages: A mapping from each stage to the information about that stage.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class StagesValue(_messages.Message):
+ """A mapping from each stage to the information about that stage.
+
+ Messages:
+ AdditionalProperty: An additional property for a StagesValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type StagesValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a StagesValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A JobExecutionStageInfo attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('JobExecutionStageInfo', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ stages = _messages.MessageField('StagesValue', 1)
+
+
+class JobExecutionStageInfo(_messages.Message):
+ """Contains information about how a particular google.dataflow.v1beta3.Step
+ will be executed.
+
+ Fields:
+ stepName: The steps associated with the execution stage. Note that stages
+ may have several steps, and that a given step might be run by more than
+ one stage.
+ """
+
+ stepName = _messages.StringField(1, repeated=True)
+
+
+class JobMessage(_messages.Message):
+ """A particular message pertaining to a Dataflow job.
+
+ Enums:
+ MessageImportanceValueValuesEnum: Importance level of the message.
+
+ Fields:
+ id: Identifies the message. This is automatically generated by the
+ service; the caller should treat it as an opaque string.
+ messageImportance: Importance level of the message.
+ messageText: The text of the message.
+ time: The timestamp of the message.
+ """
+
+ class MessageImportanceValueValuesEnum(_messages.Enum):
+ """Importance level of the message.
+
+ Values:
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
+ JOB_MESSAGE_DEBUG: <no description>
+ JOB_MESSAGE_DETAILED: <no description>
+ JOB_MESSAGE_BASIC: <no description>
+ JOB_MESSAGE_WARNING: <no description>
+ JOB_MESSAGE_ERROR: <no description>
+ """
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
+ JOB_MESSAGE_DEBUG = 1
+ JOB_MESSAGE_DETAILED = 2
+ JOB_MESSAGE_BASIC = 3
+ JOB_MESSAGE_WARNING = 4
+ JOB_MESSAGE_ERROR = 5
+
+ id = _messages.StringField(1)
+ messageImportance = _messages.EnumField('MessageImportanceValueValuesEnum', 2)
+ messageText = _messages.StringField(3)
+ time = _messages.StringField(4)
+
+
+class JobMetrics(_messages.Message):
+ """JobMetrics contains a collection of metrics descibing the detailed
+ progress of a Dataflow job. Metrics correspond to user-defined and system-
+ defined metrics in the job. This resource captures only the most recent
+ values of each metric; time-series data can be queried for them (under the
+ same metric names) from Cloud Monitoring.
+
+ Fields:
+ metricTime: Timestamp as of which metric values are current.
+ metrics: All metrics for this job.
+ """
+
+ metricTime = _messages.StringField(1)
+ metrics = _messages.MessageField('MetricUpdate', 2, repeated=True)
+
+
+class KeyRangeDataDiskAssignment(_messages.Message):
+ """Data disk assignment information for a specific key-range of a sharded
+ computation. Currently we only support UTF-8 character splits to simplify
+ encoding into JSON.
+
+ Fields:
+ dataDisk: The name of the data disk where data for this range is stored.
+ This name is local to the Google Cloud Platform project and uniquely
+ identifies the disk within that project, for example
+ "myproject-1014-104817-4c2-harness-0-disk-1".
+ end: The end (exclusive) of the key range.
+ start: The start (inclusive) of the key range.
+ """
+
+ dataDisk = _messages.StringField(1)
+ end = _messages.StringField(2)
+ start = _messages.StringField(3)
+
+
+class KeyRangeLocation(_messages.Message):
+ """Location information for a specific key-range of a sharded computation.
+ Currently we only support UTF-8 character splits to simplify encoding into
+ JSON.
+
+ Fields:
+ dataDisk: The name of the data disk where data for this range is stored.
+ This name is local to the Google Cloud Platform project and uniquely
+ identifies the disk within that project, for example
+ "myproject-1014-104817-4c2-harness-0-disk-1".
+ deliveryEndpoint: The physical location of this range assignment to be
+ used for streaming computation cross-worker message delivery.
+ end: The end (exclusive) of the key range.
+ persistentDirectory: The location of the persistent state for this range,
+ as a persistent directory in the worker local filesystem.
+ start: The start (inclusive) of the key range.
+ """
+
+ dataDisk = _messages.StringField(1)
+ deliveryEndpoint = _messages.StringField(2)
+ end = _messages.StringField(3)
+ persistentDirectory = _messages.StringField(4)
+ start = _messages.StringField(5)
+
+
+class LeaseWorkItemRequest(_messages.Message):
+ """Request to lease WorkItems.
+
+ Fields:
+ currentWorkerTime: The current timestamp at the worker.
+ requestedLeaseDuration: The initial lease period.
+ workItemTypes: Filter for WorkItem type.
+ workerCapabilities: Worker capabilities. WorkItems might be limited to
+ workers with specific capabilities.
+ workerId: Identifies the worker leasing work -- typically the ID of the
+ virtual machine running the worker.
+ """
+
+ currentWorkerTime = _messages.StringField(1)
+ requestedLeaseDuration = _messages.StringField(2)
+ workItemTypes = _messages.StringField(3, repeated=True)
+ workerCapabilities = _messages.StringField(4, repeated=True)
+ workerId = _messages.StringField(5)
+
+
+class LeaseWorkItemResponse(_messages.Message):
+ """Response to a request to lease WorkItems.
+
+ Fields:
+ workItems: A list of the leased WorkItems.
+ """
+
+ workItems = _messages.MessageField('WorkItem', 1, repeated=True)
+
+
+class ListJobMessagesResponse(_messages.Message):
+ """Response to a request to list job messages.
+
+ Fields:
+ jobMessages: Messages in ascending timestamp order.
+ nextPageToken: The token to obtain the next page of results if there are
+ more.
+ """
+
+ jobMessages = _messages.MessageField('JobMessage', 1, repeated=True)
+ nextPageToken = _messages.StringField(2)
+
+
+class ListJobsResponse(_messages.Message):
+ """Response to a request to list Dataflow jobs. This may be a partial
+ response, depending on the page size in the ListJobsRequest.
+
+ Fields:
+ jobs: A subset of the requested job information.
+ nextPageToken: Set if there may be more results than fit in this response.
+ """
+
+ jobs = _messages.MessageField('Job', 1, repeated=True)
+ nextPageToken = _messages.StringField(2)
+
+
+class MapTask(_messages.Message):
+ """MapTask consists of an ordered set of instructions, each of which
+ describes one particular low-level operation for the worker to perform in
+ order to accomplish the MapTask's WorkItem. Each instruction must appear in
+ the list before any instructions which depends on its output.
+
+ Fields:
+ instructions: The instructions in the MapTask.
+ stageName: System-defined name of the stage containing this MapTask.
+ Unique across the workflow.
+ systemName: System-defined name of this MapTask. Unique across the
+ workflow.
+ """
+
+ instructions = _messages.MessageField('ParallelInstruction', 1, repeated=True)
+ stageName = _messages.StringField(2)
+ systemName = _messages.StringField(3)
+
+
+class MetricStructuredName(_messages.Message):
+ """Identifies a metric, by describing the source which generated the metric.
+
+ Messages:
+ ContextValue: Zero or more labeled fields which identify the part of the
+ job this metric is associated with, such as the name of a step or
+ collection. For example, built-in counters associated with steps will
+ have context['step'] = . Counters associated with PCollections in the
+ SDK will have context['pcollection'] = .
+
+ Fields:
+ context: Zero or more labeled fields which identify the part of the job
+ this metric is associated with, such as the name of a step or
+ collection. For example, built-in counters associated with steps will
+ have context['step'] = . Counters associated with PCollections in the
+ SDK will have context['pcollection'] = .
+ name: Worker-defined metric name.
+ origin: Origin (namespace) of metric name. May be blank for user-define
+ metrics; will be "dataflow" for metrics defined by the Dataflow service
+ or SDK.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class ContextValue(_messages.Message):
+ """Zero or more labeled fields which identify the part of the job this
+ metric is associated with, such as the name of a step or collection. For
+ example, built-in counters associated with steps will have context['step']
+ = . Counters associated with PCollections in the SDK will have
+ context['pcollection'] = .
+
+ Messages:
+ AdditionalProperty: An additional property for a ContextValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type ContextValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a ContextValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A string attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.StringField(2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ context = _messages.MessageField('ContextValue', 1)
+ name = _messages.StringField(2)
+ origin = _messages.StringField(3)
+
+
+class MetricUpdate(_messages.Message):
+ """Describes the state of a metric.
+
+ Fields:
+ cumulative: True if this metric is reported as the total cumulative
+ aggregate value accumulated since the worker started working on this
+ WorkItem. By default this is false, indicating that this metric is
+ reported as a delta that is not associated with any WorkItem.
+ internal: Worker-computed aggregate value for internal use by the Dataflow
+ service.
+ kind: Metric aggregation kind. The possible metric aggregation kinds are
+ "Sum", "Max", "Min", "Mean", "Set", "And", and "Or". The specified
+ aggregation kind is case-insensitive. If omitted, this is not an
+ aggregated value but instead a single metric sample value.
+ meanCount: Worker-computed aggregate value for the "Mean" aggregation
+ kind. This holds the count of the aggregated values and is used in
+ combination with mean_sum above to obtain the actual mean aggregate
+ value. The only possible value type is Long.
+ meanSum: Worker-computed aggregate value for the "Mean" aggregation kind.
+ This holds the sum of the aggregated values and is used in combination
+ with mean_count below to obtain the actual mean aggregate value. The
+ only possible value types are Long and Double.
+ name: Name of the metric.
+ scalar: Worker-computed aggregate value for aggregation kinds "Sum",
+ "Max", "Min", "And", and "Or". The possible value types are Long,
+ Double, and Boolean.
+ set: Worker-computed aggregate value for the "Set" aggregation kind. The
+ only possible value type is a list of Values whose type can be Long,
+ Double, or String, according to the metric's type. All Values in the
+ list must be of the same type.
+ updateTime: Timestamp associated with the metric value. Optional when
+ workers are reporting work progress; it will be filled in responses from
+ the metrics API.
+ """
+
+ cumulative = _messages.BooleanField(1)
+ internal = _messages.MessageField('extra_types.JsonValue', 2)
+ kind = _messages.StringField(3)
+ meanCount = _messages.MessageField('extra_types.JsonValue', 4)
+ meanSum = _messages.MessageField('extra_types.JsonValue', 5)
+ name = _messages.MessageField('MetricStructuredName', 6)
+ scalar = _messages.MessageField('extra_types.JsonValue', 7)
+ set = _messages.MessageField('extra_types.JsonValue', 8)
+ updateTime = _messages.StringField(9)
+
+
+class MountedDataDisk(_messages.Message):
+ """Describes mounted data disk.
+
+ Fields:
+ dataDisk: The name of the data disk. This name is local to the Google
+ Cloud Platform project and uniquely identifies the disk within that
+ project, for example "myproject-1014-104817-4c2-harness-0-disk-1".
+ """
+
+ dataDisk = _messages.StringField(1)
+
+
+class MultiOutputInfo(_messages.Message):
+ """Information about an output of a multi-output DoFn.
+
+ Fields:
+ tag: The id of the tag the user code will emit to this output by; this
+ should correspond to the tag of some SideInputInfo.
+ """
+
+ tag = _messages.StringField(1)
+
+
+class Package(_messages.Message):
+ """Packages that need to be installed in order for a worker to run the steps
+ of the Dataflow job which will be assigned to its worker pool. This is the
+ mechanism by which the SDK causes code to be loaded onto the workers. For
+ example, the Dataflow Java SDK might use this to install jars containing the
+ user's code and all of the various dependencies (libraries, data files, etc)
+ required in order for that code to run.
+
+ Fields:
+ location: The resource to read the package from. The supported resource
+ type is: Google Cloud Storage: storage.googleapis.com/{bucket}
+ bucket.storage.googleapis.com/
+ name: The name of the package.
+ """
+
+ location = _messages.StringField(1)
+ name = _messages.StringField(2)
+
+
+class ParDoInstruction(_messages.Message):
+ """An instruction that does a ParDo operation. Takes one main input and zero
+ or more side inputs, and produces zero or more outputs. Runs user code.
+
+ Messages:
+ UserFnValue: The user function to invoke.
+
+ Fields:
+ input: The input.
+ multiOutputInfos: Information about each of the outputs, if user_fn is a
+ MultiDoFn.
+ numOutputs: The number of outputs.
+ sideInputs: Zero or more side inputs.
+ userFn: The user function to invoke.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class UserFnValue(_messages.Message):
+ """The user function to invoke.
+
+ Messages:
+ AdditionalProperty: An additional property for a UserFnValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a UserFnValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ input = _messages.MessageField('InstructionInput', 1)
+ multiOutputInfos = _messages.MessageField('MultiOutputInfo', 2, repeated=True)
+ numOutputs = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+ sideInputs = _messages.MessageField('SideInputInfo', 4, repeated=True)
+ userFn = _messages.MessageField('UserFnValue', 5)
+
+
+class ParallelInstruction(_messages.Message):
+ """Describes a particular operation comprising a MapTask.
+
+ Fields:
+ flatten: Additional information for Flatten instructions.
+ name: User-provided name of this operation.
+ outputs: Describes the outputs of the instruction.
+ parDo: Additional information for ParDo instructions.
+ partialGroupByKey: Additional information for PartialGroupByKey
+ instructions.
+ read: Additional information for Read instructions.
+ systemName: System-defined name of this operation. Unique across the
+ workflow.
+ write: Additional information for Write instructions.
+ """
+
+ flatten = _messages.MessageField('FlattenInstruction', 1)
+ name = _messages.StringField(2)
+ outputs = _messages.MessageField('InstructionOutput', 3, repeated=True)
+ parDo = _messages.MessageField('ParDoInstruction', 4)
+ partialGroupByKey = _messages.MessageField('PartialGroupByKeyInstruction', 5)
+ read = _messages.MessageField('ReadInstruction', 6)
+ systemName = _messages.StringField(7)
+ write = _messages.MessageField('WriteInstruction', 8)
+
+
+class PartialGroupByKeyInstruction(_messages.Message):
+ """An instruction that does a partial group-by-key. One input and one
+ output.
+
+ Messages:
+ InputElementCodecValue: The codec to use for interpreting an element in
+ the input PTable.
+ ValueCombiningFnValue: The value combining function to invoke.
+
+ Fields:
+ input: Describes the input to the partial group-by-key instruction.
+ inputElementCodec: The codec to use for interpreting an element in the
+ input PTable.
+ sideInputs: Zero or more side inputs.
+ valueCombiningFn: The value combining function to invoke.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class InputElementCodecValue(_messages.Message):
+ """The codec to use for interpreting an element in the input PTable.
+
+ Messages:
+ AdditionalProperty: An additional property for a InputElementCodecValue
+ object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a InputElementCodecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class ValueCombiningFnValue(_messages.Message):
+ """The value combining function to invoke.
+
+ Messages:
+ AdditionalProperty: An additional property for a ValueCombiningFnValue
+ object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a ValueCombiningFnValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ input = _messages.MessageField('InstructionInput', 1)
+ inputElementCodec = _messages.MessageField('InputElementCodecValue', 2)
+ sideInputs = _messages.MessageField('SideInputInfo', 3, repeated=True)
+ valueCombiningFn = _messages.MessageField('ValueCombiningFnValue', 4)
+
+
+class Position(_messages.Message):
+ """Position defines a position within a collection of data. The value can be
+ either the end position, a key (used with ordered collections), a byte
+ offset, or a record index.
+
+ Fields:
+ byteOffset: Position is a byte offset.
+ concatPosition: CloudPosition is a concat position.
+ end: Position is past all other positions. Also useful for the end
+ position of an unbounded range.
+ key: Position is a string key, ordered lexicographically.
+ recordIndex: Position is a record index.
+ shufflePosition: CloudPosition is a base64 encoded BatchShufflePosition
+ (with FIXED sharding).
+ """
+
+ byteOffset = _messages.IntegerField(1)
+ concatPosition = _messages.MessageField('ConcatPosition', 2)
+ end = _messages.BooleanField(3)
+ key = _messages.StringField(4)
+ recordIndex = _messages.IntegerField(5)
+ shufflePosition = _messages.StringField(6)
+
+
+class PubsubLocation(_messages.Message):
+ """Identifies a pubsub location to use for transferring data into or out of
+ a streaming Dataflow job.
+
+ Fields:
+ dropLateData: Indicates whether the pipeline allows late-arriving data.
+ idLabel: If set, contains a pubsub label from which to extract record ids.
+ If left empty, record deduplication will be strictly best effort.
+ subscription: A pubsub subscription, in the form of
+ "pubsub.googleapis.com/subscriptions/ /"
+ timestampLabel: If set, contains a pubsub label from which to extract
+ record timestamps. If left empty, record timestamps will be generated
+ upon arrival.
+ topic: A pubsub topic, in the form of "pubsub.googleapis.com/topics/ /"
+ trackingSubscription: If set, specifies the pubsub subscription that will
+ be used for tracking custom time timestamps for watermark estimation.
+ """
+
+ dropLateData = _messages.BooleanField(1)
+ idLabel = _messages.StringField(2)
+ subscription = _messages.StringField(3)
+ timestampLabel = _messages.StringField(4)
+ topic = _messages.StringField(5)
+ trackingSubscription = _messages.StringField(6)
+
+
+class ReadInstruction(_messages.Message):
+ """An instruction that reads records. Takes no inputs, produces one output.
+
+ Fields:
+ source: The source to read from.
+ """
+
+ source = _messages.MessageField('Source', 1)
+
+
+class ReportWorkItemStatusRequest(_messages.Message):
+ """Request to report the status of WorkItems.
+
+ Fields:
+ currentWorkerTime: The current timestamp at the worker.
+ workItemStatuses: The order is unimportant, except that the order of the
+ WorkItemServiceState messages in the ReportWorkItemStatusResponse
+ corresponds to the order of WorkItemStatus messages here.
+ workerId: The ID of the worker reporting the WorkItem status. If this does
+ not match the ID of the worker which the Dataflow service believes
+ currently has the lease on the WorkItem, the report will be dropped
+ (with an error response).
+ """
+
+ currentWorkerTime = _messages.StringField(1)
+ workItemStatuses = _messages.MessageField('WorkItemStatus', 2, repeated=True)
+ workerId = _messages.StringField(3)
+
+
+class ReportWorkItemStatusResponse(_messages.Message):
+ """Response from a request to report the status of WorkItems.
+
+ Fields:
+ workItemServiceStates: A set of messages indicating the service-side state
+ for each WorkItem whose status was reported, in the same order as the
+ WorkItemStatus messages in the ReportWorkItemStatusRequest which
+ resulting in this response.
+ """
+
+ workItemServiceStates = _messages.MessageField('WorkItemServiceState', 1, repeated=True)
+
+
+class ReportedParallelism(_messages.Message):
+ """Represents the level of parallelism in a WorkItem's input, reported by
+ the worker.
+
+ Fields:
+ isInfinite: Specifies whether the parallelism is infinite. If true,
+ "value" is ignored. Infinite parallelism means the service will assume
+ that the work item can always be split into more non-empty work items by
+ dynamic splitting. This is a work-around for lack of support for
+ infinity by the current JSON-based Java RPC stack.
+ value: Specifies the level of parallelism in case it is finite.
+ """
+
+ isInfinite = _messages.BooleanField(1)
+ value = _messages.FloatField(2)
+
+
+class SendWorkerMessagesRequest(_messages.Message):
+ """A request for sending worker messages to the service.
+
+ Fields:
+ workerMessages: The WorkerMessages to send.
+ """
+
+ workerMessages = _messages.MessageField('WorkerMessage', 1, repeated=True)
+
+
+class SendWorkerMessagesResponse(_messages.Message):
+ """The response to the worker messages.
+
+ Fields:
+ workerMessageResponses: The servers response to the worker messages.
+ """
+
+ workerMessageResponses = _messages.MessageField('WorkerMessageResponse', 1, repeated=True)
+
+
+class SeqMapTask(_messages.Message):
+ """Describes a particular function to invoke.
+
+ Messages:
+ UserFnValue: The user function to invoke.
+
+ Fields:
+ inputs: Information about each of the inputs.
+ name: The user-provided name of the SeqDo operation.
+ outputInfos: Information about each of the outputs.
+ stageName: System-defined name of the stage containing the SeqDo
+ operation. Unique across the workflow.
+ systemName: System-defined name of the SeqDo operation. Unique across the
+ workflow.
+ userFn: The user function to invoke.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class UserFnValue(_messages.Message):
+ """The user function to invoke.
+
+ Messages:
+ AdditionalProperty: An additional property for a UserFnValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a UserFnValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ inputs = _messages.MessageField('SideInputInfo', 1, repeated=True)
+ name = _messages.StringField(2)
+ outputInfos = _messages.MessageField('SeqMapTaskOutputInfo', 3, repeated=True)
+ stageName = _messages.StringField(4)
+ systemName = _messages.StringField(5)
+ userFn = _messages.MessageField('UserFnValue', 6)
+
+
+class SeqMapTaskOutputInfo(_messages.Message):
+ """Information about an output of a SeqMapTask.
+
+ Fields:
+ sink: The sink to write the output value to.
+ tag: The id of the TupleTag the user code will tag the output value by.
+ """
+
+ sink = _messages.MessageField('Sink', 1)
+ tag = _messages.StringField(2)
+
+
+class ShellTask(_messages.Message):
+ """A task which consists of a shell command for the worker to execute.
+
+ Fields:
+ command: The shell command to run.
+ exitCode: Exit code for the task.
+ """
+
+ command = _messages.StringField(1)
+ exitCode = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+
+
+class SideInputInfo(_messages.Message):
+ """Information about a side input of a DoFn or an input of a SeqDoFn.
+
+ Messages:
+ KindValue: How to interpret the source element(s) as a side input value.
+
+ Fields:
+ kind: How to interpret the source element(s) as a side input value.
+ sources: The source(s) to read element(s) from to get the value of this
+ side input. If more than one source, then the elements are taken from
+ the sources, in the specified order if order matters. At least one
+ source is required.
+ tag: The id of the tag the user code will access this side input by; this
+ should correspond to the tag of some MultiOutputInfo.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class KindValue(_messages.Message):
+ """How to interpret the source element(s) as a side input value.
+
+ Messages:
+ AdditionalProperty: An additional property for a KindValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a KindValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ kind = _messages.MessageField('KindValue', 1)
+ sources = _messages.MessageField('Source', 2, repeated=True)
+ tag = _messages.StringField(3)
+
+
+class Sink(_messages.Message):
+ """A sink that records can be encoded and written to.
+
+ Messages:
+ CodecValue: The codec to use to encode data written to the sink.
+ SpecValue: The sink to write to, plus its parameters.
+
+ Fields:
+ codec: The codec to use to encode data written to the sink.
+ spec: The sink to write to, plus its parameters.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class CodecValue(_messages.Message):
+ """The codec to use to encode data written to the sink.
+
+ Messages:
+ AdditionalProperty: An additional property for a CodecValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a CodecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class SpecValue(_messages.Message):
+ """The sink to write to, plus its parameters.
+
+ Messages:
+ AdditionalProperty: An additional property for a SpecValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a SpecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ codec = _messages.MessageField('CodecValue', 1)
+ spec = _messages.MessageField('SpecValue', 2)
+
+
+class Source(_messages.Message):
+ """A source that records can be read and decoded from.
+
+ Messages:
+ BaseSpecsValueListEntry: A BaseSpecsValueListEntry object.
+ CodecValue: The codec to use to decode data read from the source.
+ SpecValue: The source to read from, plus its parameters.
+
+ Fields:
+ baseSpecs: While splitting, sources may specify the produced bundles as
+ differences against another source, in order to save backend-side memory
+ and allow bigger jobs. For details, see SourceSplitRequest. To support
+ this use case, the full set of parameters of the source is logically
+ obtained by taking the latest explicitly specified value of each
+ parameter in the order: base_specs (later items win), spec (overrides
+ anything in base_specs).
+ codec: The codec to use to decode data read from the source.
+ doesNotNeedSplitting: Setting this value to true hints to the framework
+ that the source doesn't need splitting, and using SourceSplitRequest on
+ it would yield SOURCE_SPLIT_OUTCOME_USE_CURRENT. E.g. a file splitter
+ may set this to true when splitting a single file into a set of byte
+ ranges of appropriate size, and set this to false when splitting a
+ filepattern into individual files. However, for efficiency, a file
+ splitter may decide to produce file subranges directly from the
+ filepattern to avoid a splitting round-trip. See SourceSplitRequest for
+ an overview of the splitting process. This field is meaningful only in
+ the Source objects populated by the user (e.g. when filling in a
+ DerivedSource). Source objects supplied by the framework to the user
+ don't have this field populated.
+ metadata: Optionally, metadata for this source can be supplied right away,
+ avoiding a SourceGetMetadataOperation roundtrip (see
+ SourceOperationRequest). This field is meaningful only in the Source
+ objects populated by the user (e.g. when filling in a DerivedSource).
+ Source objects supplied by the framework to the user don't have this
+ field populated.
+ spec: The source to read from, plus its parameters.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class BaseSpecsValueListEntry(_messages.Message):
+ """A BaseSpecsValueListEntry object.
+
+ Messages:
+ AdditionalProperty: An additional property for a BaseSpecsValueListEntry
+ object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a BaseSpecsValueListEntry object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class CodecValue(_messages.Message):
+ """The codec to use to decode data read from the source.
+
+ Messages:
+ AdditionalProperty: An additional property for a CodecValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a CodecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class SpecValue(_messages.Message):
+ """The source to read from, plus its parameters.
+
+ Messages:
+ AdditionalProperty: An additional property for a SpecValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a SpecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ baseSpecs = _messages.MessageField('BaseSpecsValueListEntry', 1, repeated=True)
+ codec = _messages.MessageField('CodecValue', 2)
+ doesNotNeedSplitting = _messages.BooleanField(3)
+ metadata = _messages.MessageField('SourceMetadata', 4)
+ spec = _messages.MessageField('SpecValue', 5)
+
+
+class SourceFork(_messages.Message):
+ """DEPRECATED in favor of DynamicSourceSplit.
+
+ Fields:
+ primary: DEPRECATED
+ primarySource: DEPRECATED
+ residual: DEPRECATED
+ residualSource: DEPRECATED
+ """
+
+ primary = _messages.MessageField('SourceSplitShard', 1)
+ primarySource = _messages.MessageField('DerivedSource', 2)
+ residual = _messages.MessageField('SourceSplitShard', 3)
+ residualSource = _messages.MessageField('DerivedSource', 4)
+
+
+class SourceGetMetadataRequest(_messages.Message):
+ """A request to compute the SourceMetadata of a Source.
+
+ Fields:
+ source: Specification of the source whose metadata should be computed.
+ """
+
+ source = _messages.MessageField('Source', 1)
+
+
+class SourceGetMetadataResponse(_messages.Message):
+ """The result of a SourceGetMetadataOperation.
+
+ Fields:
+ metadata: The computed metadata.
+ """
+
+ metadata = _messages.MessageField('SourceMetadata', 1)
+
+
+class SourceMetadata(_messages.Message):
+ """Metadata about a Source useful for automatically optimizing and tuning
+ the pipeline, etc.
+
+ Fields:
+ estimatedSizeBytes: An estimate of the total size (in bytes) of the data
+ that would be read from this source. This estimate is in terms of
+ external storage size, before any decompression or other processing done
+ by the reader.
+ infinite: Specifies that the size of this source is known to be infinite
+ (this is a streaming source).
+ producesSortedKeys: Whether this source is known to produce key/value
+ pairs with the (encoded) keys in lexicographically sorted order.
+ """
+
+ estimatedSizeBytes = _messages.IntegerField(1)
+ infinite = _messages.BooleanField(2)
+ producesSortedKeys = _messages.BooleanField(3)
+
+
+class SourceOperationRequest(_messages.Message):
+ """A work item that represents the different operations that can be
+ performed on a user-defined Source specification.
+
+ Fields:
+ getMetadata: Information about a request to get metadata about a source.
+ split: Information about a request to split a source.
+ """
+
+ getMetadata = _messages.MessageField('SourceGetMetadataRequest', 1)
+ split = _messages.MessageField('SourceSplitRequest', 2)
+
+
+class SourceOperationResponse(_messages.Message):
+ """The result of a SourceOperationRequest, specified in
+ ReportWorkItemStatusRequest.source_operation when the work item is
+ completed.
+
+ Fields:
+ getMetadata: A response to a request to get metadata about a source.
+ split: A response to a request to split a source.
+ """
+
+ getMetadata = _messages.MessageField('SourceGetMetadataResponse', 1)
+ split = _messages.MessageField('SourceSplitResponse', 2)
+
+
+class SourceSplitOptions(_messages.Message):
+ """Hints for splitting a Source into bundles (parts for parallel processing)
+ using SourceSplitRequest.
+
+ Fields:
+ desiredBundleSizeBytes: The source should be split into a set of bundles
+ where the estimated size of each is approximately this many bytes.
+ desiredShardSizeBytes: DEPRECATED in favor of desired_bundle_size_bytes.
+ """
+
+ desiredBundleSizeBytes = _messages.IntegerField(1)
+ desiredShardSizeBytes = _messages.IntegerField(2)
+
+
+class SourceSplitRequest(_messages.Message):
+ """Represents the operation to split a high-level Source specification into
+ bundles (parts for parallel processing). At a high level, splitting of a
+ source into bundles happens as follows: SourceSplitRequest is applied to the
+ source. If it returns SOURCE_SPLIT_OUTCOME_USE_CURRENT, no further splitting
+ happens and the source is used "as is". Otherwise, splitting is applied
+ recursively to each produced DerivedSource. As an optimization, for any
+ Source, if its does_not_need_splitting is true, the framework assumes that
+ splitting this source would return SOURCE_SPLIT_OUTCOME_USE_CURRENT, and
+ doesn't initiate a SourceSplitRequest. This applies both to the initial
+ source being split and to bundles produced from it.
+
+ Fields:
+ options: Hints for tuning the splitting process.
+ source: Specification of the source to be split.
+ """
+
+ options = _messages.MessageField('SourceSplitOptions', 1)
+ source = _messages.MessageField('Source', 2)
+
+
+class SourceSplitResponse(_messages.Message):
+ """The response to a SourceSplitRequest.
+
+ Enums:
+ OutcomeValueValuesEnum: Indicates whether splitting happened and produced
+ a list of bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current
+ source should be processed "as is" without splitting. "bundles" is
+ ignored in this case. If this is SPLITTING_HAPPENED, then "bundles"
+ contains a list of bundles into which the source was split.
+
+ Fields:
+ bundles: If outcome is SPLITTING_HAPPENED, then this is a list of bundles
+ into which the source was split. Otherwise this field is ignored. This
+ list can be empty, which means the source represents an empty input.
+ outcome: Indicates whether splitting happened and produced a list of
+ bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current source should
+ be processed "as is" without splitting. "bundles" is ignored in this
+ case. If this is SPLITTING_HAPPENED, then "bundles" contains a list of
+ bundles into which the source was split.
+ shards: DEPRECATED in favor of bundles.
+ """
+
+ class OutcomeValueValuesEnum(_messages.Enum):
+ """Indicates whether splitting happened and produced a list of bundles. If
+ this is USE_CURRENT_SOURCE_AS_IS, the current source should be processed
+ "as is" without splitting. "bundles" is ignored in this case. If this is
+ SPLITTING_HAPPENED, then "bundles" contains a list of bundles into which
+ the source was split.
+
+ Values:
+ SOURCE_SPLIT_OUTCOME_UNKNOWN: <no description>
+ SOURCE_SPLIT_OUTCOME_USE_CURRENT: <no description>
+ SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED: <no description>
+ """
+ SOURCE_SPLIT_OUTCOME_UNKNOWN = 0
+ SOURCE_SPLIT_OUTCOME_USE_CURRENT = 1
+ SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED = 2
+
+ bundles = _messages.MessageField('DerivedSource', 1, repeated=True)
+ outcome = _messages.EnumField('OutcomeValueValuesEnum', 2)
+ shards = _messages.MessageField('SourceSplitShard', 3, repeated=True)
+
+
+class SourceSplitShard(_messages.Message):
+ """DEPRECATED in favor of DerivedSource.
+
+ Enums:
+ DerivationModeValueValuesEnum: DEPRECATED
+
+ Fields:
+ derivationMode: DEPRECATED
+ source: DEPRECATED
+ """
+
+ class DerivationModeValueValuesEnum(_messages.Enum):
+ """DEPRECATED
+
+ Values:
+ SOURCE_DERIVATION_MODE_UNKNOWN: <no description>
+ SOURCE_DERIVATION_MODE_INDEPENDENT: <no description>
+ SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: <no description>
+ SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: <no description>
+ """
+ SOURCE_DERIVATION_MODE_UNKNOWN = 0
+ SOURCE_DERIVATION_MODE_INDEPENDENT = 1
+ SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2
+ SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3
+
+ derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1)
+ source = _messages.MessageField('Source', 2)
+
+
+class StandardQueryParameters(_messages.Message):
+ """Query parameters accepted by all methods.
+
+ Fields:
+ f__xgafv: V1 error format.
+ access_token: OAuth access token.
+ alt: Data format for response.
+ bearer_token: OAuth bearer token.
+ callback: JSONP
+ fields: Selector specifying which fields to include in a partial response.
+ key: API key. Your API key identifies your project and provides you with
+ API access, quota, and reports. Required unless you provide an OAuth 2.0
+ token.
+ oauth_token: OAuth 2.0 token for the current user.
+ pp: Pretty-print response.
+ prettyPrint: Returns response with indentations and line breaks.
+ quotaUser: Available to use for quota purposes for server-side
+ applications. Can be any arbitrary string assigned to a user, but should
+ not exceed 40 characters.
+ trace: A tracing token of the form "token:<tokenid>" to include in api
+ requests.
+ uploadType: Legacy upload protocol for media (e.g. "media", "multipart").
+ upload_protocol: Upload protocol for media (e.g. "raw", "multipart").
+ """
+
+ f__xgafv = _messages.StringField(1)
+ access_token = _messages.StringField(2)
+ alt = _messages.StringField(3, default=u'json')
+ bearer_token = _messages.StringField(4)
+ callback = _messages.StringField(5)
+ fields = _messages.StringField(6)
+ key = _messages.StringField(7)
+ oauth_token = _messages.StringField(8)
+ pp = _messages.BooleanField(9, default=True)
+ prettyPrint = _messages.BooleanField(10, default=True)
+ quotaUser = _messages.StringField(11)
+ trace = _messages.StringField(12)
+ uploadType = _messages.StringField(13)
+ upload_protocol = _messages.StringField(14)
+
+
+class StateFamilyConfig(_messages.Message):
+ """State family configuration.
+
+ Fields:
+ isRead: If true, this family corresponds to a read operation.
+ stateFamily: The state family value.
+ """
+
+ isRead = _messages.BooleanField(1)
+ stateFamily = _messages.StringField(2)
+
+
+class Status(_messages.Message):
+ """The `Status` type defines a logical error model that is suitable for
+ different programming environments, including REST APIs and RPC APIs. It is
+ used by [gRPC](https://github.com/grpc). The error model is designed to be:
+ - Simple to use and understand for most users - Flexible enough to meet
+ unexpected needs # Overview The `Status` message contains three pieces of
+ data: error code, error message, and error details. The error code should be
+ an enum value of google.rpc.Code, but it may accept additional error codes
+ if needed. The error message should be a developer-facing English message
+ that helps developers *understand* and *resolve* the error. If a localized
+ user-facing error message is needed, put the localized message in the error
+ details or localize it in the client. The optional error details may contain
+ arbitrary information about the error. There is a predefined set of error
+ detail types in the package `google.rpc` which can be used for common error
+ conditions. # Language mapping The `Status` message is the logical
+ representation of the error model, but it is not necessarily the actual wire
+ format. When the `Status` message is exposed in different client libraries
+ and different wire protocols, it can be mapped differently. For example, it
+ will likely be mapped to some exceptions in Java, but more likely mapped to
+ some error codes in C. # Other uses The error model and the `Status` message
+ can be used in a variety of environments, either with or without APIs, to
+ provide a consistent developer experience across different environments.
+ Example uses of this error model include: - Partial errors. If a service
+ needs to return partial errors to the client, it may embed the `Status` in
+ the normal response to indicate the partial errors. - Workflow errors. A
+ typical workflow has multiple steps. Each step may have a `Status` message
+ for error reporting purpose. - Batch operations. If a client uses batch
+ request and batch response, the `Status` message should be used directly
+ inside batch response, one for each error sub-response. - Asynchronous
+ operations. If an API call embeds asynchronous operation results in its
+ response, the status of those operations should be represented directly
+ using the `Status` message. - Logging. If some API errors are stored in
+ logs, the message `Status` could be used directly after any stripping needed
+ for security/privacy reasons.
+
+ Messages:
+ DetailsValueListEntry: A DetailsValueListEntry object.
+
+ Fields:
+ code: The status code, which should be an enum value of google.rpc.Code.
+ details: A list of messages that carry the error details. There will be a
+ common set of message types for APIs to use.
+ message: A developer-facing error message, which should be in English. Any
+ user-facing error message should be localized and sent in the
+ google.rpc.Status.details field, or localized by the client.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class DetailsValueListEntry(_messages.Message):
+ """A DetailsValueListEntry object.
+
+ Messages:
+ AdditionalProperty: An additional property for a DetailsValueListEntry
+ object.
+
+ Fields:
+ additionalProperties: Properties of the object. Contains field @ype with
+ type URL.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a DetailsValueListEntry object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ code = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+ details = _messages.MessageField('DetailsValueListEntry', 2, repeated=True)
+ message = _messages.StringField(3)
+
+
+class Step(_messages.Message):
+ """Defines a particular step within a Dataflow job. A job consists of
+ multiple steps, each of which performs some specific operation as part of
+ the overall job. Data is typically passed from one step to another as part
+ of the job. Here's an example of a sequence of steps which together
+ implement a Map-Reduce job: * Read a collection of data from some source,
+ parsing the collection's elements. * Validate the elements. * Apply a user-
+ defined function to map each element to some value and extract an element-
+ specific key value. * Group elements with the same key into a single element
+ with that key, transforming a multiply-keyed collection into a uniquely-
+ keyed collection. * Write the elements out to some data sink. (Note that the
+ Dataflow service may be used to run many different types of jobs, not just
+ Map-Reduce).
+
+ Messages:
+ PropertiesValue: Named properties associated with the step. Each kind of
+ predefined step has its own required set of properties.
+
+ Fields:
+ kind: The kind of step in the dataflow Job.
+ name: Name identifying the step. This must be unique for each step with
+ respect to all other steps in the dataflow Job.
+ properties: Named properties associated with the step. Each kind of
+ predefined step has its own required set of properties.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class PropertiesValue(_messages.Message):
+ """Named properties associated with the step. Each kind of predefined step
+ has its own required set of properties.
+
+ Messages:
+ AdditionalProperty: An additional property for a PropertiesValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a PropertiesValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ kind = _messages.StringField(1)
+ name = _messages.StringField(2)
+ properties = _messages.MessageField('PropertiesValue', 3)
+
+
+class StreamLocation(_messages.Message):
+ """Describes a stream of data, either as input to be processed or as output
+ of a streaming Dataflow job.
+
+ Fields:
+ customSourceLocation: The stream is a custom source.
+ pubsubLocation: The stream is a pubsub stream.
+ sideInputLocation: The stream is a streaming side input.
+ streamingStageLocation: The stream is part of another computation within
+ the current streaming Dataflow job.
+ """
+
+ customSourceLocation = _messages.MessageField('CustomSourceLocation', 1)
+ pubsubLocation = _messages.MessageField('PubsubLocation', 2)
+ sideInputLocation = _messages.MessageField('StreamingSideInputLocation', 3)
+ streamingStageLocation = _messages.MessageField('StreamingStageLocation', 4)
+
+
+class StreamingComputationRanges(_messages.Message):
+ """Describes full or partial data disk assignment information of the
+ computation ranges.
+
+ Fields:
+ computationId: The ID of the computation.
+ rangeAssignments: Data disk assignments for ranges from this computation.
+ """
+
+ computationId = _messages.StringField(1)
+ rangeAssignments = _messages.MessageField('KeyRangeDataDiskAssignment', 2, repeated=True)
+
+
+class StreamingComputationTask(_messages.Message):
+ """A task which describes what action should be performed for the specified
+ streaming computation ranges.
+
+ Enums:
+ TaskTypeValueValuesEnum: A type of streaming computation task.
+
+ Fields:
+ computationRanges: Contains ranges of a streaming computation this task
+ should apply to.
+ dataDisks: Describes the set of data disks this task should apply to.
+ taskType: A type of streaming computation task.
+ """
+
+ class TaskTypeValueValuesEnum(_messages.Enum):
+ """A type of streaming computation task.
+
+ Values:
+ STREAMING_COMPUTATION_TASK_UNKNOWN: <no description>
+ STREAMING_COMPUTATION_TASK_STOP: <no description>
+ STREAMING_COMPUTATION_TASK_START: <no description>
+ """
+ STREAMING_COMPUTATION_TASK_UNKNOWN = 0
+ STREAMING_COMPUTATION_TASK_STOP = 1
+ STREAMING_COMPUTATION_TASK_START = 2
+
+ computationRanges = _messages.MessageField('StreamingComputationRanges', 1, repeated=True)
+ dataDisks = _messages.MessageField('MountedDataDisk', 2, repeated=True)
+ taskType = _messages.EnumField('TaskTypeValueValuesEnum', 3)
+
+
+class StreamingSetupTask(_messages.Message):
+ """A task which initializes part of a streaming Dataflow job.
+
+ Fields:
+ drain: The user has requested drain.
+ receiveWorkPort: The TCP port on which the worker should listen for
+ messages from other streaming computation workers.
+ streamingComputationTopology: The global topology of the streaming
+ D
<TRUNCATED>