You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2017/02/17 07:39:18 UTC
[1/6] beam git commit: Moving dataflow runner related code to vendor
directory
Repository: beam
Updated Branches:
refs/heads/master 9df1da493 -> 36ed3f6c5
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers.py
new file mode 100644
index 0000000..4dda47a
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers.py
@@ -0,0 +1,124 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from hamcrest.core.base_matcher import BaseMatcher
+
+
+IGNORED = object()
+
+
+class MetricStructuredNameMatcher(BaseMatcher):
+ """Matches a MetricStructuredName."""
+ def __init__(self,
+ name=IGNORED,
+ origin=IGNORED,
+ context=IGNORED):
+ """Creates a MetricsStructuredNameMatcher.
+
+ Any property not passed in to the constructor will be ignored when matching.
+
+ Args:
+ name: A string with the metric name.
+ origin: A string with the metric namespace.
+ context: A key:value dictionary that will be matched to the
+ structured name.
+ """
+ if context != IGNORED and not isinstance(context, dict):
+ raise ValueError('context must be a Python dictionary.')
+
+ self.name = name
+ self.origin = origin
+ self.context = context
+
+ def _matches(self, item):
+ if self.name != IGNORED and item.name != self.name:
+ return False
+ if self.origin != IGNORED and item.origin != self.origin:
+ return False
+ if self.context != IGNORED:
+ for key, name in self.context.iteritems():
+ if key not in item.context:
+ return False
+ if name != IGNORED and item.context[key] != name:
+ return False
+ return True
+
+ def describe_to(self, description):
+ descriptors = []
+ if self.name != IGNORED:
+ descriptors.append('name is {}'.format(self.name))
+ if self.origin != IGNORED:
+ descriptors.append('origin is {}'.format(self.origin))
+ if self.context != IGNORED:
+ descriptors.append('context is ({})'.format(str(self.context)))
+
+ item_description = ' and '.join(descriptors)
+ description.append(item_description)
+
+
+class MetricUpdateMatcher(BaseMatcher):
+ """Matches a metrics update protocol buffer."""
+ def __init__(self,
+ cumulative=IGNORED,
+ name=IGNORED,
+ scalar=IGNORED,
+ kind=IGNORED):
+ """Creates a MetricUpdateMatcher.
+
+ Any property not passed in to the constructor will be ignored when matching.
+
+ Args:
+ cumulative: A boolean.
+ name: A MetricStructuredNameMatcher object that matches the name.
+ scalar: An integer with the metric update.
+ kind: A string defining the kind of counter.
+ """
+ if name != IGNORED and not isinstance(name, MetricStructuredNameMatcher):
+ raise ValueError('name must be a MetricStructuredNameMatcher.')
+
+ self.cumulative = cumulative
+ self.name = name
+ self.scalar = scalar
+ self.kind = kind
+
+ def _matches(self, item):
+ if self.cumulative != IGNORED and item.cumulative != self.cumulative:
+ return False
+ if self.name != IGNORED and not self.name._matches(item.name):
+ return False
+ if self.kind != IGNORED and item.kind != self.kind:
+ return False
+ if self.scalar != IGNORED:
+ value_property = [p
+ for p in item.scalar.object_value.properties
+ if p.key == 'value']
+ int_value = value_property[0].value.integer_value
+ if self.scalar != int_value:
+ return False
+ return True
+
+ def describe_to(self, description):
+ descriptors = []
+ if self.cumulative != IGNORED:
+ descriptors.append('cumulative is {}'.format(self.cumulative))
+ if self.name != IGNORED:
+ descriptors.append('name is {}'.format(self.name))
+ if self.scalar != IGNORED:
+ descriptors.append('scalar is ({})'.format(str(self.scalar)))
+
+ item_description = ' and '.join(descriptors)
+ description.append(item_description)
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers_test.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers_test.py
new file mode 100644
index 0000000..2b56ae1
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/message_matchers_test.py
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+import hamcrest as hc
+
+import apache_beam.runners.google_cloud_dataflow.internal.clients.dataflow as dataflow
+from apache_beam.internal.json_value import to_json_value
+from apache_beam.runners.google_cloud_dataflow.internal.clients.dataflow import message_matchers
+
+
+class TestMatchers(unittest.TestCase):
+
+ def test_structured_name_matcher_basic(self):
+ metric_name = dataflow.MetricStructuredName()
+ metric_name.name = 'metric1'
+ metric_name.origin = 'origin2'
+
+ matcher = message_matchers.MetricStructuredNameMatcher(
+ name='metric1',
+ origin='origin2')
+ hc.assert_that(metric_name, hc.is_(matcher))
+ with self.assertRaises(AssertionError):
+ matcher = message_matchers.MetricStructuredNameMatcher(
+ name='metric1',
+ origin='origin1')
+ hc.assert_that(metric_name, hc.is_(matcher))
+
+ def test_metric_update_basic(self):
+ metric_update = dataflow.MetricUpdate()
+ metric_update.name = dataflow.MetricStructuredName()
+ metric_update.name.name = 'metric1'
+ metric_update.name.origin = 'origin1'
+
+ metric_update.cumulative = False
+ metric_update.kind = 'sum'
+ metric_update.scalar = to_json_value(1, with_type=True)
+
+ name_matcher = message_matchers.MetricStructuredNameMatcher(
+ name='metric1',
+ origin='origin1')
+ matcher = message_matchers.MetricUpdateMatcher(
+ name=name_matcher,
+ kind='sum',
+ scalar=1)
+
+ hc.assert_that(metric_update, hc.is_(matcher))
+
+ with self.assertRaises(AssertionError):
+ matcher.kind = 'suma'
+ hc.assert_that(metric_update, hc.is_(matcher))
+
+
+if __name__ == '__main__':
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/template_runner_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/template_runner_test.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/template_runner_test.py
new file mode 100644
index 0000000..bbcf340
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/template_runner_test.py
@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for templated pipelines."""
+
+from __future__ import absolute_import
+
+import json
+import tempfile
+import unittest
+
+import apache_beam as beam
+from apache_beam.pipeline import Pipeline
+from apache_beam.runners.google_cloud_dataflow.dataflow_runner import DataflowRunner
+from apache_beam.runners.google_cloud_dataflow.internal import apiclient
+from apache_beam.utils.pipeline_options import PipelineOptions
+
+
+class TemplatingDataflowRunnerTest(unittest.TestCase):
+ """TemplatingDataflow tests."""
+ def test_full_completion(self):
+ # Create dummy file and close it. Note that we need to do this because
+ # Windows does not allow NamedTemporaryFiles to be reopened elsewhere
+ # before the temporary file is closed.
+ dummy_file = tempfile.NamedTemporaryFile(delete=False)
+ dummy_file_name = dummy_file.name
+ dummy_file.close()
+
+ dummy_dir = tempfile.mkdtemp()
+
+ remote_runner = DataflowRunner()
+ pipeline = Pipeline(remote_runner,
+ options=PipelineOptions([
+ '--dataflow_endpoint=ignored',
+ '--sdk_location=' + dummy_file_name,
+ '--job_name=test-job',
+ '--project=test-project',
+ '--staging_location=' + dummy_dir,
+ '--temp_location=/dev/null',
+ '--template_location=' + dummy_file_name,
+ '--no_auth=True']))
+
+ pipeline | beam.Create([1, 2, 3]) | beam.Map(lambda x: x) # pylint: disable=expression-not-assigned
+ pipeline.run().wait_until_finish()
+ with open(dummy_file_name) as template_file:
+ saved_job_dict = json.load(template_file)
+ self.assertEqual(
+ saved_job_dict['environment']['sdkPipelineOptions']
+ ['options']['project'], 'test-project')
+ self.assertEqual(
+ saved_job_dict['environment']['sdkPipelineOptions']
+ ['options']['job_name'], 'test-job')
+
+ def test_bad_path(self):
+ dummy_sdk_file = tempfile.NamedTemporaryFile()
+ remote_runner = DataflowRunner()
+ pipeline = Pipeline(remote_runner,
+ options=PipelineOptions([
+ '--dataflow_endpoint=ignored',
+ '--sdk_location=' + dummy_sdk_file.name,
+ '--job_name=test-job',
+ '--project=test-project',
+ '--staging_location=ignored',
+ '--temp_location=/dev/null',
+ '--template_location=/bad/path',
+ '--no_auth=True']))
+ remote_runner.job = apiclient.Job(pipeline.options)
+
+ with self.assertRaises(IOError):
+ pipeline.run().wait_until_finish()
+
+
+if __name__ == '__main__':
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/runner_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py
index d7a50aa..7ec5d8a 100644
--- a/sdks/python/apache_beam/runners/runner_test.py
+++ b/sdks/python/apache_beam/runners/runner_test.py
@@ -22,30 +22,28 @@ the other unit tests. In this file we choose to test only aspects related to
caching and clearing values that are not tested elsewhere.
"""
-from datetime import datetime
import json
import unittest
+from datetime import datetime
import hamcrest as hc
import apache_beam as beam
-
-from apache_beam.internal import apiclient
+import apache_beam.transforms as ptransform
+from apache_beam.metrics.cells import DistributionData
+from apache_beam.metrics.cells import DistributionResult
+from apache_beam.metrics.execution import MetricKey
+from apache_beam.metrics.execution import MetricResult
+from apache_beam.metrics.metricbase import MetricName
from apache_beam.pipeline import Pipeline
-from apache_beam.runners import create_runner
from apache_beam.runners import DataflowRunner
from apache_beam.runners import DirectRunner
from apache_beam.runners import TestDataflowRunner
-import apache_beam.transforms as ptransform
+from apache_beam.runners import create_runner
+from apache_beam.runners.google_cloud_dataflow.internal import apiclient
from apache_beam.transforms.display import DisplayDataItem
from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.metrics.cells import DistributionData
-from apache_beam.metrics.cells import DistributionResult
-from apache_beam.metrics.execution import MetricResult
-from apache_beam.metrics.execution import MetricKey
-from apache_beam.metrics.metricbase import MetricName
-
class RunnerTest(unittest.TestCase):
default_properties = [
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/template_runner_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/template_runner_test.py b/sdks/python/apache_beam/runners/template_runner_test.py
deleted file mode 100644
index b73645c..0000000
--- a/sdks/python/apache_beam/runners/template_runner_test.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Unit tests for templated pipelines."""
-
-from __future__ import absolute_import
-
-import json
-import tempfile
-import unittest
-
-import apache_beam as beam
-from apache_beam.internal import apiclient
-from apache_beam.pipeline import Pipeline
-from apache_beam.runners.google_cloud_dataflow.dataflow_runner import DataflowRunner
-from apache_beam.utils.pipeline_options import PipelineOptions
-
-
-class TemplatingDataflowRunnerTest(unittest.TestCase):
- """TemplatingDataflow tests."""
- def test_full_completion(self):
- # Create dummy file and close it. Note that we need to do this because
- # Windows does not allow NamedTemporaryFiles to be reopened elsewhere
- # before the temporary file is closed.
- dummy_file = tempfile.NamedTemporaryFile(delete=False)
- dummy_file_name = dummy_file.name
- dummy_file.close()
-
- dummy_dir = tempfile.mkdtemp()
-
- remote_runner = DataflowRunner()
- pipeline = Pipeline(remote_runner,
- options=PipelineOptions([
- '--dataflow_endpoint=ignored',
- '--sdk_location=' + dummy_file_name,
- '--job_name=test-job',
- '--project=test-project',
- '--staging_location=' + dummy_dir,
- '--temp_location=/dev/null',
- '--template_location=' + dummy_file_name,
- '--no_auth=True']))
-
- pipeline | beam.Create([1, 2, 3]) | beam.Map(lambda x: x) # pylint: disable=expression-not-assigned
- pipeline.run().wait_until_finish()
- with open(dummy_file_name) as template_file:
- saved_job_dict = json.load(template_file)
- self.assertEqual(
- saved_job_dict['environment']['sdkPipelineOptions']
- ['options']['project'], 'test-project')
- self.assertEqual(
- saved_job_dict['environment']['sdkPipelineOptions']
- ['options']['job_name'], 'test-job')
-
- def test_bad_path(self):
- dummy_sdk_file = tempfile.NamedTemporaryFile()
- remote_runner = DataflowRunner()
- pipeline = Pipeline(remote_runner,
- options=PipelineOptions([
- '--dataflow_endpoint=ignored',
- '--sdk_location=' + dummy_sdk_file.name,
- '--job_name=test-job',
- '--project=test-project',
- '--staging_location=ignored',
- '--temp_location=/dev/null',
- '--template_location=/bad/path',
- '--no_auth=True']))
- remote_runner.job = apiclient.Job(pipeline.options)
-
- with self.assertRaises(IOError):
- pipeline.run().wait_until_finish()
-
-
-if __name__ == '__main__':
- unittest.main()
[4/6] beam git commit: Moving dataflow runner related code to vendor
directory
Posted by al...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py
deleted file mode 100644
index a42154e..0000000
--- a/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_messages.py
+++ /dev/null
@@ -1,4173 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Generated message classes for dataflow version v1b3.
-
-Develops and executes data processing patterns like ETL, batch computation,
-and continuous computation.
-"""
-# NOTE: This file is autogenerated and should not be edited by hand.
-
-from apitools.base.protorpclite import messages as _messages
-from apitools.base.py import encoding
-from apitools.base.py import extra_types
-
-
-package = 'dataflow'
-
-
-class ApproximateProgress(_messages.Message):
- """Obsolete in favor of ApproximateReportedProgress and
- ApproximateSplitRequest.
-
- Fields:
- percentComplete: Obsolete.
- position: Obsolete.
- remainingTime: Obsolete.
- """
-
- percentComplete = _messages.FloatField(1, variant=_messages.Variant.FLOAT)
- position = _messages.MessageField('Position', 2)
- remainingTime = _messages.StringField(3)
-
-
-class ApproximateReportedProgress(_messages.Message):
- """A progress measurement of a WorkItem by a worker.
-
- Fields:
- consumedParallelism: Total amount of parallelism in the portion of input
- of this task that has already been consumed and is no longer active. In
- the first two examples above (see remaining_parallelism), the value
- should be 29 or 2 respectively. The sum of remaining_parallelism and
- consumed_parallelism should equal the total amount of parallelism in
- this work item. If specified, must be finite.
- fractionConsumed: Completion as fraction of the input consumed, from 0.0
- (beginning, nothing consumed), to 1.0 (end of the input, entire input
- consumed).
- position: A Position within the work to represent a progress.
- remainingParallelism: Total amount of parallelism in the input of this
- task that remains, (i.e. can be delegated to this task and any new tasks
- via dynamic splitting). Always at least 1 for non-finished work items
- and 0 for finished. "Amount of parallelism" refers to how many non-
- empty parts of the input can be read in parallel. This does not
- necessarily equal number of records. An input that can be read in
- parallel down to the individual records is called "perfectly
- splittable". An example of non-perfectly parallelizable input is a
- block-compressed file format where a block of records has to be read as
- a whole, but different blocks can be read in parallel. Examples: * If
- we are processing record #30 (starting at 1) out of 50 in a perfectly
- splittable 50-record input, this value should be 21 (20 remaining + 1
- current). * If we are reading through block 3 in a block-compressed file
- consisting of 5 blocks, this value should be 3 (since blocks 4 and 5
- can be processed in parallel by new tasks via dynamic splitting and
- the current task remains processing block 3). * If we are reading
- through the last block in a block-compressed file, or reading or
- processing the last record in a perfectly splittable input, this value
- should be 1, because apart from the current task, no additional
- remainder can be split off.
- """
-
- consumedParallelism = _messages.MessageField('ReportedParallelism', 1)
- fractionConsumed = _messages.FloatField(2)
- position = _messages.MessageField('Position', 3)
- remainingParallelism = _messages.MessageField('ReportedParallelism', 4)
-
-
-class ApproximateSplitRequest(_messages.Message):
- """A suggestion by the service to the worker to dynamically split the
- WorkItem.
-
- Fields:
- fractionConsumed: A fraction at which to split the work item, from 0.0
- (beginning of the input) to 1.0 (end of the input).
- position: A Position at which to split the work item.
- """
-
- fractionConsumed = _messages.FloatField(1)
- position = _messages.MessageField('Position', 2)
-
-
-class AutoscalingSettings(_messages.Message):
- """Settings for WorkerPool autoscaling.
-
- Enums:
- AlgorithmValueValuesEnum: The algorithm to use for autoscaling.
-
- Fields:
- algorithm: The algorithm to use for autoscaling.
- maxNumWorkers: The maximum number of workers to cap scaling at.
- """
-
- class AlgorithmValueValuesEnum(_messages.Enum):
- """The algorithm to use for autoscaling.
-
- Values:
- AUTOSCALING_ALGORITHM_UNKNOWN: The algorithm is unknown, or unspecified.
- AUTOSCALING_ALGORITHM_NONE: Disable autoscaling.
- AUTOSCALING_ALGORITHM_BASIC: Increase worker count over time to reduce
- job execution time.
- """
- AUTOSCALING_ALGORITHM_UNKNOWN = 0
- AUTOSCALING_ALGORITHM_NONE = 1
- AUTOSCALING_ALGORITHM_BASIC = 2
-
- algorithm = _messages.EnumField('AlgorithmValueValuesEnum', 1)
- maxNumWorkers = _messages.IntegerField(2, variant=_messages.Variant.INT32)
-
-
-class ComputationTopology(_messages.Message):
- """All configuration data for a particular Computation.
-
- Fields:
- computationId: The ID of the computation.
- inputs: The inputs to the computation.
- keyRanges: The key ranges processed by the computation.
- outputs: The outputs from the computation.
- stateFamilies: The state family values.
- systemStageName: The system stage name.
- userStageName: The user stage name.
- """
-
- computationId = _messages.StringField(1)
- inputs = _messages.MessageField('StreamLocation', 2, repeated=True)
- keyRanges = _messages.MessageField('KeyRangeLocation', 3, repeated=True)
- outputs = _messages.MessageField('StreamLocation', 4, repeated=True)
- stateFamilies = _messages.MessageField('StateFamilyConfig', 5, repeated=True)
- systemStageName = _messages.StringField(6)
- userStageName = _messages.StringField(7)
-
-
-class ConcatPosition(_messages.Message):
- """A position that encapsulates an inner position and an index for the inner
- position. A ConcatPosition can be used by a reader of a source that
- encapsulates a set of other sources.
-
- Fields:
- index: Index of the inner source.
- position: Position within the inner source.
- """
-
- index = _messages.IntegerField(1, variant=_messages.Variant.INT32)
- position = _messages.MessageField('Position', 2)
-
-
-class CounterMetadata(_messages.Message):
- """CounterMetadata includes all static non-name non-value counter
- attributes.
-
- Enums:
- KindValueValuesEnum: Counter aggregation kind.
- StandardUnitsValueValuesEnum: System defined Units, see above enum.
-
- Fields:
- description: Human-readable description of the counter semantics.
- kind: Counter aggregation kind.
- otherUnits: A string referring to the unit type.
- standardUnits: System defined Units, see above enum.
- """
-
- class KindValueValuesEnum(_messages.Enum):
- """Counter aggregation kind.
-
- Values:
- INVALID: Counter aggregation kind was not set.
- SUM: Aggregated value is the sum of all contributed values.
- MAX: Aggregated value is the max of all contributed values.
- MIN: Aggregated value is the min of all contributed values.
- MEAN: Aggregated value is the mean of all contributed values.
- OR: Aggregated value represents the logical 'or' of all contributed
- values.
- AND: Aggregated value represents the logical 'and' of all contributed
- values.
- SET: Aggregated value is a set of unique contributed values.
- DISTRIBUTION: Aggregated value captures statistics about a distribution.
- """
- INVALID = 0
- SUM = 1
- MAX = 2
- MIN = 3
- MEAN = 4
- OR = 5
- AND = 6
- SET = 7
- DISTRIBUTION = 8
-
- class StandardUnitsValueValuesEnum(_messages.Enum):
- """System defined Units, see above enum.
-
- Values:
- BYTES: Counter returns a value in bytes.
- BYTES_PER_SEC: Counter returns a value in bytes per second.
- MILLISECONDS: Counter returns a value in milliseconds.
- MICROSECONDS: Counter returns a value in microseconds.
- NANOSECONDS: Counter returns a value in nanoseconds.
- TIMESTAMP_MSEC: Counter returns a timestamp in milliseconds.
- TIMESTAMP_USEC: Counter returns a timestamp in microseconds.
- TIMESTAMP_NSEC: Counter returns a timestamp in nanoseconds.
- """
- BYTES = 0
- BYTES_PER_SEC = 1
- MILLISECONDS = 2
- MICROSECONDS = 3
- NANOSECONDS = 4
- TIMESTAMP_MSEC = 5
- TIMESTAMP_USEC = 6
- TIMESTAMP_NSEC = 7
-
- description = _messages.StringField(1)
- kind = _messages.EnumField('KindValueValuesEnum', 2)
- otherUnits = _messages.StringField(3)
- standardUnits = _messages.EnumField('StandardUnitsValueValuesEnum', 4)
-
-
-class CounterStructuredName(_messages.Message):
- """Identifies a counter within a per-job namespace. Counters whose
- structured names are the same get merged into a single value for the job.
-
- Enums:
- PortionValueValuesEnum: Portion of this counter, either key or value.
- StandardOriginValueValuesEnum: One of the standard Origins defined above.
-
- Fields:
- componentStepName: Name of the optimized step being executed by the
- workers.
- executionStepName: Name of the stage. An execution step contains multiple
- component steps.
- name: Counter name. Not necessarily globally-unique, but unique within the
- context of the other fields. Required.
- originalStepName: System generated name of the original step in the user's
- graph, before optimization.
- otherOrigin: A string containing the origin of the counter.
- portion: Portion of this counter, either key or value.
- standardOrigin: One of the standard Origins defined above.
- workerId: ID of a particular worker.
- """
-
- class PortionValueValuesEnum(_messages.Enum):
- """Portion of this counter, either key or value.
-
- Values:
- ALL: Counter portion has not been set.
- KEY: Counter reports a key.
- VALUE: Counter reports a value.
- """
- ALL = 0
- KEY = 1
- VALUE = 2
-
- class StandardOriginValueValuesEnum(_messages.Enum):
- """One of the standard Origins defined above.
-
- Values:
- DATAFLOW: Counter was created by the Dataflow system.
- USER: Counter was created by the user.
- """
- DATAFLOW = 0
- USER = 1
-
- componentStepName = _messages.StringField(1)
- executionStepName = _messages.StringField(2)
- name = _messages.StringField(3)
- originalStepName = _messages.StringField(4)
- otherOrigin = _messages.StringField(5)
- portion = _messages.EnumField('PortionValueValuesEnum', 6)
- standardOrigin = _messages.EnumField('StandardOriginValueValuesEnum', 7)
- workerId = _messages.StringField(8)
-
-
-class CounterStructuredNameAndMetadata(_messages.Message):
- """A single message which encapsulates structured name and metadata for a
- given counter.
-
- Fields:
- metadata: Metadata associated with a counter
- name: Structured name of the counter.
- """
-
- metadata = _messages.MessageField('CounterMetadata', 1)
- name = _messages.MessageField('CounterStructuredName', 2)
-
-
-class CounterUpdate(_messages.Message):
- """An update to a Counter sent from a worker.
-
- Fields:
- boolean: Boolean value for And, Or.
- cumulative: True if this counter is reported as the total cumulative
- aggregate value accumulated since the worker started working on this
- WorkItem. By default this is false, indicating that this counter is
- reported as a delta.
- distribution: Distribution data
- floatingPoint: Floating point value for Sum, Max, Min.
- floatingPointList: List of floating point numbers, for Set.
- floatingPointMean: Floating point mean aggregation value for Mean.
- integer: Integer value for Sum, Max, Min.
- integerList: List of integers, for Set.
- integerMean: Integer mean aggregation value for Mean.
- internal: Value for internally-defined counters used by the Dataflow
- service.
- nameAndKind: Counter name and aggregation type.
- shortId: The service-generated short identifier for this counter. The
- short_id -> (name, metadata) mapping is constant for the lifetime of a
- job.
- stringList: List of strings, for Set.
- structuredNameAndMetadata: Counter structured name and metadata.
- """
-
- boolean = _messages.BooleanField(1)
- cumulative = _messages.BooleanField(2)
- distribution = _messages.MessageField('DistributionUpdate', 3)
- floatingPoint = _messages.FloatField(4)
- floatingPointList = _messages.MessageField('FloatingPointList', 5)
- floatingPointMean = _messages.MessageField('FloatingPointMean', 6)
- integer = _messages.MessageField('SplitInt64', 7)
- integerList = _messages.MessageField('IntegerList', 8)
- integerMean = _messages.MessageField('IntegerMean', 9)
- internal = _messages.MessageField('extra_types.JsonValue', 10)
- nameAndKind = _messages.MessageField('NameAndKind', 11)
- shortId = _messages.IntegerField(12)
- stringList = _messages.MessageField('StringList', 13)
- structuredNameAndMetadata = _messages.MessageField('CounterStructuredNameAndMetadata', 14)
-
-
-class CreateJobFromTemplateRequest(_messages.Message):
- """A request to create a Cloud Dataflow job from a template.
-
- Messages:
- ParametersValue: The runtime parameters to pass to the job.
-
- Fields:
- environment: The runtime environment for the job.
- gcsPath: Required. A Cloud Storage path to the template from which to
- create the job. Must be a valid Cloud Storage URL, beginning with
- `gs://`.
- jobName: Required. The job name to use for the created job.
- parameters: The runtime parameters to pass to the job.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class ParametersValue(_messages.Message):
- """The runtime parameters to pass to the job.
-
- Messages:
- AdditionalProperty: An additional property for a ParametersValue object.
-
- Fields:
- additionalProperties: Additional properties of type ParametersValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a ParametersValue object.
-
- Fields:
- key: Name of the additional property.
- value: A string attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.StringField(2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- environment = _messages.MessageField('RuntimeEnvironment', 1)
- gcsPath = _messages.StringField(2)
- jobName = _messages.StringField(3)
- parameters = _messages.MessageField('ParametersValue', 4)
-
-
-class CustomSourceLocation(_messages.Message):
- """Identifies the location of a custom souce.
-
- Fields:
- stateful: Whether this source is stateful.
- """
-
- stateful = _messages.BooleanField(1)
-
-
-class DataDiskAssignment(_messages.Message):
- """Data disk assignment for a given VM instance.
-
- Fields:
- dataDisks: Mounted data disks. The order is important a data disk's
- 0-based index in this list defines which persistent directory the disk
- is mounted to, for example the list of {
- "myproject-1014-104817-4c2-harness-0-disk-0" }, {
- "myproject-1014-104817-4c2-harness-0-disk-1" }.
- vmInstance: VM instance name the data disks mounted to, for example
- "myproject-1014-104817-4c2-harness-0".
- """
-
- dataDisks = _messages.StringField(1, repeated=True)
- vmInstance = _messages.StringField(2)
-
-
-class DataflowProjectsJobsCreateRequest(_messages.Message):
- """A DataflowProjectsJobsCreateRequest object.
-
- Enums:
- ViewValueValuesEnum: The level of information requested in response.
-
- Fields:
- job: A Job resource to be passed as the request body.
- location: The location that contains this job.
- projectId: The ID of the Cloud Platform project that the job belongs to.
- replaceJobId: Deprecated. This field is now in the Job message.
- view: The level of information requested in response.
- """
-
- class ViewValueValuesEnum(_messages.Enum):
- """The level of information requested in response.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- job = _messages.MessageField('Job', 1)
- location = _messages.StringField(2)
- projectId = _messages.StringField(3, required=True)
- replaceJobId = _messages.StringField(4)
- view = _messages.EnumField('ViewValueValuesEnum', 5)
-
-
-class DataflowProjectsJobsDebugGetConfigRequest(_messages.Message):
- """A DataflowProjectsJobsDebugGetConfigRequest object.
-
- Fields:
- getDebugConfigRequest: A GetDebugConfigRequest resource to be passed as
- the request body.
- jobId: The job id.
- projectId: The project id.
- """
-
- getDebugConfigRequest = _messages.MessageField('GetDebugConfigRequest', 1)
- jobId = _messages.StringField(2, required=True)
- projectId = _messages.StringField(3, required=True)
-
-
-class DataflowProjectsJobsDebugSendCaptureRequest(_messages.Message):
- """A DataflowProjectsJobsDebugSendCaptureRequest object.
-
- Fields:
- jobId: The job id.
- projectId: The project id.
- sendDebugCaptureRequest: A SendDebugCaptureRequest resource to be passed
- as the request body.
- """
-
- jobId = _messages.StringField(1, required=True)
- projectId = _messages.StringField(2, required=True)
- sendDebugCaptureRequest = _messages.MessageField('SendDebugCaptureRequest', 3)
-
-
-class DataflowProjectsJobsGetMetricsRequest(_messages.Message):
- """A DataflowProjectsJobsGetMetricsRequest object.
-
- Fields:
- jobId: The job to get messages for.
- location: The location which contains the job specified by job_id.
- projectId: A project id.
- startTime: Return only metric data that has changed since this time.
- Default is to return all information about all metrics for the job.
- """
-
- jobId = _messages.StringField(1, required=True)
- location = _messages.StringField(2)
- projectId = _messages.StringField(3, required=True)
- startTime = _messages.StringField(4)
-
-
-class DataflowProjectsJobsGetRequest(_messages.Message):
- """A DataflowProjectsJobsGetRequest object.
-
- Enums:
- ViewValueValuesEnum: The level of information requested in response.
-
- Fields:
- jobId: The job ID.
- location: The location that contains this job.
- projectId: The ID of the Cloud Platform project that the job belongs to.
- view: The level of information requested in response.
- """
-
- class ViewValueValuesEnum(_messages.Enum):
- """The level of information requested in response.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- jobId = _messages.StringField(1, required=True)
- location = _messages.StringField(2)
- projectId = _messages.StringField(3, required=True)
- view = _messages.EnumField('ViewValueValuesEnum', 4)
-
-
-class DataflowProjectsJobsListRequest(_messages.Message):
- """A DataflowProjectsJobsListRequest object.
-
- Enums:
- FilterValueValuesEnum: The kind of filter to use.
- ViewValueValuesEnum: Level of information requested in response. Default
- is `JOB_VIEW_SUMMARY`.
-
- Fields:
- filter: The kind of filter to use.
- location: The location that contains this job.
- pageSize: If there are many jobs, limit response to at most this many. The
- actual number of jobs returned will be the lesser of max_responses and
- an unspecified server-defined limit.
- pageToken: Set this to the 'next_page_token' field of a previous response
- to request additional results in a long list.
- projectId: The project which owns the jobs.
- view: Level of information requested in response. Default is
- `JOB_VIEW_SUMMARY`.
- """
-
- class FilterValueValuesEnum(_messages.Enum):
- """The kind of filter to use.
-
- Values:
- UNKNOWN: <no description>
- ALL: <no description>
- TERMINATED: <no description>
- ACTIVE: <no description>
- """
- UNKNOWN = 0
- ALL = 1
- TERMINATED = 2
- ACTIVE = 3
-
- class ViewValueValuesEnum(_messages.Enum):
- """Level of information requested in response. Default is
- `JOB_VIEW_SUMMARY`.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- filter = _messages.EnumField('FilterValueValuesEnum', 1)
- location = _messages.StringField(2)
- pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32)
- pageToken = _messages.StringField(4)
- projectId = _messages.StringField(5, required=True)
- view = _messages.EnumField('ViewValueValuesEnum', 6)
-
-
-class DataflowProjectsJobsMessagesListRequest(_messages.Message):
- """A DataflowProjectsJobsMessagesListRequest object.
-
- Enums:
- MinimumImportanceValueValuesEnum: Filter to only get messages with
- importance >= level
-
- Fields:
- endTime: Return only messages with timestamps < end_time. The default is
- now (i.e. return up to the latest messages available).
- jobId: The job to get messages about.
- location: The location which contains the job specified by job_id.
- minimumImportance: Filter to only get messages with importance >= level
- pageSize: If specified, determines the maximum number of messages to
- return. If unspecified, the service may choose an appropriate default,
- or may return an arbitrarily large number of results.
- pageToken: If supplied, this should be the value of next_page_token
- returned by an earlier call. This will cause the next page of results to
- be returned.
- projectId: A project id.
- startTime: If specified, return only messages with timestamps >=
- start_time. The default is the job creation time (i.e. beginning of
- messages).
- """
-
- class MinimumImportanceValueValuesEnum(_messages.Enum):
- """Filter to only get messages with importance >= level
-
- Values:
- JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
- JOB_MESSAGE_DEBUG: <no description>
- JOB_MESSAGE_DETAILED: <no description>
- JOB_MESSAGE_BASIC: <no description>
- JOB_MESSAGE_WARNING: <no description>
- JOB_MESSAGE_ERROR: <no description>
- """
- JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
- JOB_MESSAGE_DEBUG = 1
- JOB_MESSAGE_DETAILED = 2
- JOB_MESSAGE_BASIC = 3
- JOB_MESSAGE_WARNING = 4
- JOB_MESSAGE_ERROR = 5
-
- endTime = _messages.StringField(1)
- jobId = _messages.StringField(2, required=True)
- location = _messages.StringField(3)
- minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 4)
- pageSize = _messages.IntegerField(5, variant=_messages.Variant.INT32)
- pageToken = _messages.StringField(6)
- projectId = _messages.StringField(7, required=True)
- startTime = _messages.StringField(8)
-
-
-class DataflowProjectsJobsUpdateRequest(_messages.Message):
- """A DataflowProjectsJobsUpdateRequest object.
-
- Fields:
- job: A Job resource to be passed as the request body.
- jobId: The job ID.
- location: The location that contains this job.
- projectId: The ID of the Cloud Platform project that the job belongs to.
- """
-
- job = _messages.MessageField('Job', 1)
- jobId = _messages.StringField(2, required=True)
- location = _messages.StringField(3)
- projectId = _messages.StringField(4, required=True)
-
-
-class DataflowProjectsJobsWorkItemsLeaseRequest(_messages.Message):
- """A DataflowProjectsJobsWorkItemsLeaseRequest object.
-
- Fields:
- jobId: Identifies the workflow job this worker belongs to.
- leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the
- request body.
- projectId: Identifies the project this worker belongs to.
- """
-
- jobId = _messages.StringField(1, required=True)
- leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2)
- projectId = _messages.StringField(3, required=True)
-
-
-class DataflowProjectsJobsWorkItemsReportStatusRequest(_messages.Message):
- """A DataflowProjectsJobsWorkItemsReportStatusRequest object.
-
- Fields:
- jobId: The job which the WorkItem is part of.
- projectId: The project which owns the WorkItem's job.
- reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be
- passed as the request body.
- """
-
- jobId = _messages.StringField(1, required=True)
- projectId = _messages.StringField(2, required=True)
- reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 3)
-
-
-class DataflowProjectsLocationsJobsCreateRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsCreateRequest object.
-
- Enums:
- ViewValueValuesEnum: The level of information requested in response.
-
- Fields:
- job: A Job resource to be passed as the request body.
- location: The location that contains this job.
- projectId: The ID of the Cloud Platform project that the job belongs to.
- replaceJobId: Deprecated. This field is now in the Job message.
- view: The level of information requested in response.
- """
-
- class ViewValueValuesEnum(_messages.Enum):
- """The level of information requested in response.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- job = _messages.MessageField('Job', 1)
- location = _messages.StringField(2, required=True)
- projectId = _messages.StringField(3, required=True)
- replaceJobId = _messages.StringField(4)
- view = _messages.EnumField('ViewValueValuesEnum', 5)
-
-
-class DataflowProjectsLocationsJobsGetMetricsRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsGetMetricsRequest object.
-
- Fields:
- jobId: The job to get messages for.
- location: The location which contains the job specified by job_id.
- projectId: A project id.
- startTime: Return only metric data that has changed since this time.
- Default is to return all information about all metrics for the job.
- """
-
- jobId = _messages.StringField(1, required=True)
- location = _messages.StringField(2, required=True)
- projectId = _messages.StringField(3, required=True)
- startTime = _messages.StringField(4)
-
-
-class DataflowProjectsLocationsJobsGetRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsGetRequest object.
-
- Enums:
- ViewValueValuesEnum: The level of information requested in response.
-
- Fields:
- jobId: The job ID.
- location: The location that contains this job.
- projectId: The ID of the Cloud Platform project that the job belongs to.
- view: The level of information requested in response.
- """
-
- class ViewValueValuesEnum(_messages.Enum):
- """The level of information requested in response.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- jobId = _messages.StringField(1, required=True)
- location = _messages.StringField(2, required=True)
- projectId = _messages.StringField(3, required=True)
- view = _messages.EnumField('ViewValueValuesEnum', 4)
-
-
-class DataflowProjectsLocationsJobsListRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsListRequest object.
-
- Enums:
- FilterValueValuesEnum: The kind of filter to use.
- ViewValueValuesEnum: Level of information requested in response. Default
- is `JOB_VIEW_SUMMARY`.
-
- Fields:
- filter: The kind of filter to use.
- location: The location that contains this job.
- pageSize: If there are many jobs, limit response to at most this many. The
- actual number of jobs returned will be the lesser of max_responses and
- an unspecified server-defined limit.
- pageToken: Set this to the 'next_page_token' field of a previous response
- to request additional results in a long list.
- projectId: The project which owns the jobs.
- view: Level of information requested in response. Default is
- `JOB_VIEW_SUMMARY`.
- """
-
- class FilterValueValuesEnum(_messages.Enum):
- """The kind of filter to use.
-
- Values:
- UNKNOWN: <no description>
- ALL: <no description>
- TERMINATED: <no description>
- ACTIVE: <no description>
- """
- UNKNOWN = 0
- ALL = 1
- TERMINATED = 2
- ACTIVE = 3
-
- class ViewValueValuesEnum(_messages.Enum):
- """Level of information requested in response. Default is
- `JOB_VIEW_SUMMARY`.
-
- Values:
- JOB_VIEW_UNKNOWN: <no description>
- JOB_VIEW_SUMMARY: <no description>
- JOB_VIEW_ALL: <no description>
- """
- JOB_VIEW_UNKNOWN = 0
- JOB_VIEW_SUMMARY = 1
- JOB_VIEW_ALL = 2
-
- filter = _messages.EnumField('FilterValueValuesEnum', 1)
- location = _messages.StringField(2, required=True)
- pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32)
- pageToken = _messages.StringField(4)
- projectId = _messages.StringField(5, required=True)
- view = _messages.EnumField('ViewValueValuesEnum', 6)
-
-
-class DataflowProjectsLocationsJobsMessagesListRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsMessagesListRequest object.
-
- Enums:
- MinimumImportanceValueValuesEnum: Filter to only get messages with
- importance >= level
-
- Fields:
- endTime: Return only messages with timestamps < end_time. The default is
- now (i.e. return up to the latest messages available).
- jobId: The job to get messages about.
- location: The location which contains the job specified by job_id.
- minimumImportance: Filter to only get messages with importance >= level
- pageSize: If specified, determines the maximum number of messages to
- return. If unspecified, the service may choose an appropriate default,
- or may return an arbitrarily large number of results.
- pageToken: If supplied, this should be the value of next_page_token
- returned by an earlier call. This will cause the next page of results to
- be returned.
- projectId: A project id.
- startTime: If specified, return only messages with timestamps >=
- start_time. The default is the job creation time (i.e. beginning of
- messages).
- """
-
- class MinimumImportanceValueValuesEnum(_messages.Enum):
- """Filter to only get messages with importance >= level
-
- Values:
- JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
- JOB_MESSAGE_DEBUG: <no description>
- JOB_MESSAGE_DETAILED: <no description>
- JOB_MESSAGE_BASIC: <no description>
- JOB_MESSAGE_WARNING: <no description>
- JOB_MESSAGE_ERROR: <no description>
- """
- JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
- JOB_MESSAGE_DEBUG = 1
- JOB_MESSAGE_DETAILED = 2
- JOB_MESSAGE_BASIC = 3
- JOB_MESSAGE_WARNING = 4
- JOB_MESSAGE_ERROR = 5
-
- endTime = _messages.StringField(1)
- jobId = _messages.StringField(2, required=True)
- location = _messages.StringField(3, required=True)
- minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 4)
- pageSize = _messages.IntegerField(5, variant=_messages.Variant.INT32)
- pageToken = _messages.StringField(6)
- projectId = _messages.StringField(7, required=True)
- startTime = _messages.StringField(8)
-
-
-class DataflowProjectsLocationsJobsUpdateRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsUpdateRequest object.
-
- Fields:
- job: A Job resource to be passed as the request body.
- jobId: The job ID.
- location: The location that contains this job.
- projectId: The ID of the Cloud Platform project that the job belongs to.
- """
-
- job = _messages.MessageField('Job', 1)
- jobId = _messages.StringField(2, required=True)
- location = _messages.StringField(3, required=True)
- projectId = _messages.StringField(4, required=True)
-
-
-class DataflowProjectsLocationsJobsWorkItemsLeaseRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsWorkItemsLeaseRequest object.
-
- Fields:
- jobId: Identifies the workflow job this worker belongs to.
- leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the
- request body.
- location: The location which contains the WorkItem's job.
- projectId: Identifies the project this worker belongs to.
- """
-
- jobId = _messages.StringField(1, required=True)
- leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2)
- location = _messages.StringField(3, required=True)
- projectId = _messages.StringField(4, required=True)
-
-
-class DataflowProjectsLocationsJobsWorkItemsReportStatusRequest(_messages.Message):
- """A DataflowProjectsLocationsJobsWorkItemsReportStatusRequest object.
-
- Fields:
- jobId: The job which the WorkItem is part of.
- location: The location which contains the WorkItem's job.
- projectId: The project which owns the WorkItem's job.
- reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be
- passed as the request body.
- """
-
- jobId = _messages.StringField(1, required=True)
- location = _messages.StringField(2, required=True)
- projectId = _messages.StringField(3, required=True)
- reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 4)
-
-
-class DataflowProjectsTemplatesCreateRequest(_messages.Message):
- """A DataflowProjectsTemplatesCreateRequest object.
-
- Fields:
- createJobFromTemplateRequest: A CreateJobFromTemplateRequest resource to
- be passed as the request body.
- projectId: Required. The ID of the Cloud Platform project that the job
- belongs to.
- """
-
- createJobFromTemplateRequest = _messages.MessageField('CreateJobFromTemplateRequest', 1)
- projectId = _messages.StringField(2, required=True)
-
-
-class DataflowProjectsWorkerMessagesRequest(_messages.Message):
- """A DataflowProjectsWorkerMessagesRequest object.
-
- Fields:
- projectId: The project to send the WorkerMessages to.
- sendWorkerMessagesRequest: A SendWorkerMessagesRequest resource to be
- passed as the request body.
- """
-
- projectId = _messages.StringField(1, required=True)
- sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 2)
-
-
-class DerivedSource(_messages.Message):
- """Specification of one of the bundles produced as a result of splitting a
- Source (e.g. when executing a SourceSplitRequest, or when splitting an
- active task using WorkItemStatus.dynamic_source_split), relative to the
- source being split.
-
- Enums:
- DerivationModeValueValuesEnum: What source to base the produced source on
- (if any).
-
- Fields:
- derivationMode: What source to base the produced source on (if any).
- source: Specification of the source.
- """
-
- class DerivationModeValueValuesEnum(_messages.Enum):
- """What source to base the produced source on (if any).
-
- Values:
- SOURCE_DERIVATION_MODE_UNKNOWN: The source derivation is unknown, or
- unspecified.
- SOURCE_DERIVATION_MODE_INDEPENDENT: Produce a completely independent
- Source with no base.
- SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: Produce a Source based on the
- Source being split.
- SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: Produce a Source based on the
- base of the Source being split.
- """
- SOURCE_DERIVATION_MODE_UNKNOWN = 0
- SOURCE_DERIVATION_MODE_INDEPENDENT = 1
- SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2
- SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3
-
- derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1)
- source = _messages.MessageField('Source', 2)
-
-
-class Disk(_messages.Message):
- """Describes the data disk used by a workflow job.
-
- Fields:
- diskType: Disk storage type, as defined by Google Compute Engine. This
- must be a disk type appropriate to the project and zone in which the
- workers will run. If unknown or unspecified, the service will attempt
- to choose a reasonable default. For example, the standard persistent
- disk type is a resource name typically ending in "pd-standard". If SSD
- persistent disks are available, the resource name typically ends with
- "pd-ssd". The actual valid values are defined the Google Compute Engine
- API, not by the Cloud Dataflow API; consult the Google Compute Engine
- documentation for more information about determining the set of
- available disk types for a particular project and zone. Google Compute
- Engine Disk types are local to a particular project in a particular
- zone, and so the resource name will typically look something like this:
- compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-
- standard
- mountPoint: Directory in a VM where disk is mounted.
- sizeGb: Size of disk in GB. If zero or unspecified, the service will
- attempt to choose a reasonable default.
- """
-
- diskType = _messages.StringField(1)
- mountPoint = _messages.StringField(2)
- sizeGb = _messages.IntegerField(3, variant=_messages.Variant.INT32)
-
-
-class DistributionUpdate(_messages.Message):
- """A metric value representing a distribution.
-
- Fields:
- count: The count of the number of elements present in the distribution.
- max: The maximum value present in the distribution.
- min: The minimum value present in the distribution.
- sum: Use an int64 since we'd prefer the added precision. If overflow is a
- common problem we can detect it and use an additional int64 or a double.
- sumOfSquares: Use a double since the sum of squares is likely to overflow
- int64.
- """
-
- count = _messages.MessageField('SplitInt64', 1)
- max = _messages.MessageField('SplitInt64', 2)
- min = _messages.MessageField('SplitInt64', 3)
- sum = _messages.MessageField('SplitInt64', 4)
- sumOfSquares = _messages.FloatField(5)
-
-
-class DynamicSourceSplit(_messages.Message):
- """When a task splits using WorkItemStatus.dynamic_source_split, this
- message describes the two parts of the split relative to the description of
- the current task's input.
-
- Fields:
- primary: Primary part (continued to be processed by worker). Specified
- relative to the previously-current source. Becomes current.
- residual: Residual part (returned to the pool of work). Specified relative
- to the previously-current source.
- """
-
- primary = _messages.MessageField('DerivedSource', 1)
- residual = _messages.MessageField('DerivedSource', 2)
-
-
-class Environment(_messages.Message):
- """Describes the environment in which a Dataflow Job runs.
-
- Messages:
- InternalExperimentsValue: Experimental settings.
- SdkPipelineOptionsValue: The Cloud Dataflow SDK pipeline options specified
- by the user. These options are passed through the service and are used
- to recreate the SDK pipeline options on the worker in a language
- agnostic and platform independent way.
- UserAgentValue: A description of the process that generated the request.
- VersionValue: A structure describing which components and their versions
- of the service are required in order to run the job.
-
- Fields:
- clusterManagerApiService: The type of cluster manager API to use. If
- unknown or unspecified, the service will attempt to choose a reasonable
- default. This should be in the form of the API service name, e.g.
- "compute.googleapis.com".
- dataset: The dataset for the current project where various workflow
- related tables are stored. The supported resource type is: Google
- BigQuery: bigquery.googleapis.com/{dataset}
- experiments: The list of experiments to enable.
- internalExperiments: Experimental settings.
- sdkPipelineOptions: The Cloud Dataflow SDK pipeline options specified by
- the user. These options are passed through the service and are used to
- recreate the SDK pipeline options on the worker in a language agnostic
- and platform independent way.
- serviceAccountEmail: Identity to run virtual machines as. Defaults to the
- default account.
- tempStoragePrefix: The prefix of the resources the system should use for
- temporary storage. The system will append the suffix "/temp-{JOBNAME}
- to this resource prefix, where {JOBNAME} is the value of the job_name
- field. The resulting bucket and object prefix is used as the prefix of
- the resources used to store temporary data needed during the job
- execution. NOTE: This will override the value in taskrunner_settings.
- The supported resource type is: Google Cloud Storage:
- storage.googleapis.com/{bucket}/{object}
- bucket.storage.googleapis.com/{object}
- userAgent: A description of the process that generated the request.
- version: A structure describing which components and their versions of the
- service are required in order to run the job.
- workerPools: The worker pools. At least one "harness" worker pool must be
- specified in order for the job to have workers.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class InternalExperimentsValue(_messages.Message):
- """Experimental settings.
-
- Messages:
- AdditionalProperty: An additional property for a
- InternalExperimentsValue object.
-
- Fields:
- additionalProperties: Properties of the object. Contains field @type
- with type URL.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a InternalExperimentsValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class SdkPipelineOptionsValue(_messages.Message):
- """The Cloud Dataflow SDK pipeline options specified by the user. These
- options are passed through the service and are used to recreate the SDK
- pipeline options on the worker in a language agnostic and platform
- independent way.
-
- Messages:
- AdditionalProperty: An additional property for a SdkPipelineOptionsValue
- object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a SdkPipelineOptionsValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class UserAgentValue(_messages.Message):
- """A description of the process that generated the request.
-
- Messages:
- AdditionalProperty: An additional property for a UserAgentValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a UserAgentValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class VersionValue(_messages.Message):
- """A structure describing which components and their versions of the
- service are required in order to run the job.
-
- Messages:
- AdditionalProperty: An additional property for a VersionValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a VersionValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- clusterManagerApiService = _messages.StringField(1)
- dataset = _messages.StringField(2)
- experiments = _messages.StringField(3, repeated=True)
- internalExperiments = _messages.MessageField('InternalExperimentsValue', 4)
- sdkPipelineOptions = _messages.MessageField('SdkPipelineOptionsValue', 5)
- serviceAccountEmail = _messages.StringField(6)
- tempStoragePrefix = _messages.StringField(7)
- userAgent = _messages.MessageField('UserAgentValue', 8)
- version = _messages.MessageField('VersionValue', 9)
- workerPools = _messages.MessageField('WorkerPool', 10, repeated=True)
-
-
-class FailedLocation(_messages.Message):
- """Indicates which location failed to respond to a request for data.
-
- Fields:
- name: The name of the failed location.
- """
-
- name = _messages.StringField(1)
-
-
-class FlattenInstruction(_messages.Message):
- """An instruction that copies its inputs (zero or more) to its (single)
- output.
-
- Fields:
- inputs: Describes the inputs to the flatten instruction.
- """
-
- inputs = _messages.MessageField('InstructionInput', 1, repeated=True)
-
-
-class FloatingPointList(_messages.Message):
- """A metric value representing a list of floating point numbers.
-
- Fields:
- elements: Elements of the list.
- """
-
- elements = _messages.FloatField(1, repeated=True)
-
-
-class FloatingPointMean(_messages.Message):
- """A representation of a floating point mean metric contribution.
-
- Fields:
- count: The number of values being aggregated.
- sum: The sum of all values being aggregated.
- """
-
- count = _messages.MessageField('SplitInt64', 1)
- sum = _messages.FloatField(2)
-
-
-class GetDebugConfigRequest(_messages.Message):
- """Request to get updated debug configuration for component.
-
- Fields:
- componentId: The internal component id for which debug configuration is
- requested.
- workerId: The worker id, i.e., VM hostname.
- """
-
- componentId = _messages.StringField(1)
- workerId = _messages.StringField(2)
-
-
-class GetDebugConfigResponse(_messages.Message):
- """Response to a get debug configuration request.
-
- Fields:
- config: The encoded debug configuration for the requested component.
- """
-
- config = _messages.StringField(1)
-
-
-class InstructionInput(_messages.Message):
- """An input of an instruction, as a reference to an output of a producer
- instruction.
-
- Fields:
- outputNum: The output index (origin zero) within the producer.
- producerInstructionIndex: The index (origin zero) of the parallel
- instruction that produces the output to be consumed by this input. This
- index is relative to the list of instructions in this input's
- instruction's containing MapTask.
- """
-
- outputNum = _messages.IntegerField(1, variant=_messages.Variant.INT32)
- producerInstructionIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32)
-
-
-class InstructionOutput(_messages.Message):
- """An output of an instruction.
-
- Messages:
- CodecValue: The codec to use to encode data being written via this output.
-
- Fields:
- codec: The codec to use to encode data being written via this output.
- name: The user-provided name of this output.
- onlyCountKeyBytes: For system-generated byte and mean byte metrics,
- certain instructions should only report the key size.
- onlyCountValueBytes: For system-generated byte and mean byte metrics,
- certain instructions should only report the value size.
- originalName: System-defined name for this output in the original workflow
- graph. Outputs that do not contribute to an original instruction do not
- set this.
- systemName: System-defined name of this output. Unique across the
- workflow.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class CodecValue(_messages.Message):
- """The codec to use to encode data being written via this output.
-
- Messages:
- AdditionalProperty: An additional property for a CodecValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a CodecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- codec = _messages.MessageField('CodecValue', 1)
- name = _messages.StringField(2)
- onlyCountKeyBytes = _messages.BooleanField(3)
- onlyCountValueBytes = _messages.BooleanField(4)
- originalName = _messages.StringField(5)
- systemName = _messages.StringField(6)
-
-
-class IntegerList(_messages.Message):
- """A metric value representing a list of integers.
-
- Fields:
- elements: Elements of the list.
- """
-
- elements = _messages.MessageField('SplitInt64', 1, repeated=True)
-
-
-class IntegerMean(_messages.Message):
- """A representation of an integer mean metric contribution.
-
- Fields:
- count: The number of values being aggregated.
- sum: The sum of all values being aggregated.
- """
-
- count = _messages.MessageField('SplitInt64', 1)
- sum = _messages.MessageField('SplitInt64', 2)
-
-
-class Job(_messages.Message):
- """Defines a job to be run by the Cloud Dataflow service.
-
- Enums:
- CurrentStateValueValuesEnum: The current state of the job. Jobs are
- created in the `JOB_STATE_STOPPED` state unless otherwise specified. A
- job in the `JOB_STATE_RUNNING` state may asynchronously enter a terminal
- state. After a job has reached a terminal state, no further state
- updates may be made. This field may be mutated by the Cloud Dataflow
- service; callers cannot mutate it.
- RequestedStateValueValuesEnum: The job's requested state. `UpdateJob` may
- be used to switch between the `JOB_STATE_STOPPED` and
- `JOB_STATE_RUNNING` states, by setting requested_state. `UpdateJob` may
- also be used to directly set a job's requested state to
- `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the
- job if it has not already reached a terminal state.
- TypeValueValuesEnum: The type of Cloud Dataflow job.
-
- Messages:
- LabelsValue: User-defined labels for this job. The labels map can contain
- no more than 64 entries. Entries of the labels map are UTF8 strings
- that comply with the following restrictions: * Keys must conform to
- regexp: \p{Ll}\p{Lo}{0,62} * Values must conform to regexp:
- [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally
- constrained to be <= 128 bytes in size.
- TransformNameMappingValue: The map of transform name prefixes of the job
- to be replaced to the corresponding name prefixes of the new job.
-
- Fields:
- clientRequestId: The client's unique identifier of the job, re-used across
- retried attempts. If this field is set, the service will ensure its
- uniqueness. The request to create a job will fail if the service has
- knowledge of a previously submitted job with the same client's ID and
- job name. The caller may use this field to ensure idempotence of job
- creation across retried attempts to create a job. By default, the field
- is empty and, in that case, the service ignores it.
- createTime: The timestamp when the job was initially created. Immutable
- and set by the Cloud Dataflow service.
- currentState: The current state of the job. Jobs are created in the
- `JOB_STATE_STOPPED` state unless otherwise specified. A job in the
- `JOB_STATE_RUNNING` state may asynchronously enter a terminal state.
- After a job has reached a terminal state, no further state updates may
- be made. This field may be mutated by the Cloud Dataflow service;
- callers cannot mutate it.
- currentStateTime: The timestamp associated with the current state.
- environment: The environment for the job.
- executionInfo: Information about how the Cloud Dataflow service will run
- the job.
- id: The unique ID of this job. This field is set by the Cloud Dataflow
- service when the Job is created, and is immutable for the life of the
- job.
- labels: User-defined labels for this job. The labels map can contain no
- more than 64 entries. Entries of the labels map are UTF8 strings that
- comply with the following restrictions: * Keys must conform to regexp:
- \p{Ll}\p{Lo}{0,62} * Values must conform to regexp:
- [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally
- constrained to be <= 128 bytes in size.
- location: The location that contains this job.
- name: The user-specified Cloud Dataflow job name. Only one Job with a
- given name may exist in a project at any given time. If a caller
- attempts to create a Job with the same name as an already-existing Job,
- the attempt returns the existing Job. The name must match the regular
- expression `[a-z]([-a-z0-9]{0,38}[a-z0-9])?`
- projectId: The ID of the Cloud Platform project that the job belongs to.
- replaceJobId: If this job is an update of an existing job, this field is
- the job ID of the job it replaced. When sending a `CreateJobRequest`,
- you can update a job by specifying it here. The job named here is
- stopped, and its intermediate state is transferred to this job.
- replacedByJobId: If another job is an update of this job (and thus, this
- job is in `JOB_STATE_UPDATED`), this field contains the ID of that job.
- requestedState: The job's requested state. `UpdateJob` may be used to
- switch between the `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING` states,
- by setting requested_state. `UpdateJob` may also be used to directly
- set a job's requested state to `JOB_STATE_CANCELLED` or
- `JOB_STATE_DONE`, irrevocably terminating the job if it has not already
- reached a terminal state.
- steps: The top-level steps that constitute the entire job.
- tempFiles: A set of files the system should be aware of that are used for
- temporary storage. These temporary files will be removed on job
- completion. No duplicates are allowed. No file patterns are supported.
- The supported files are: Google Cloud Storage:
- storage.googleapis.com/{bucket}/{object}
- bucket.storage.googleapis.com/{object}
- transformNameMapping: The map of transform name prefixes of the job to be
- replaced to the corresponding name prefixes of the new job.
- type: The type of Cloud Dataflow job.
- """
-
- class CurrentStateValueValuesEnum(_messages.Enum):
- """The current state of the job. Jobs are created in the
- `JOB_STATE_STOPPED` state unless otherwise specified. A job in the
- `JOB_STATE_RUNNING` state may asynchronously enter a terminal state. After
- a job has reached a terminal state, no further state updates may be made.
- This field may be mutated by the Cloud Dataflow service; callers cannot
- mutate it.
-
- Values:
- JOB_STATE_UNKNOWN: The job's run state isn't specified.
- JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not
- yet started to run.
- JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is
- currently running.
- JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully
- completed. This is a terminal job state. This state may be set by the
- Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It
- may also be set via a Cloud Dataflow `UpdateJob` call, if the job has
- not yet reached a terminal state.
- JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed.
- This is a terminal job state. This state may only be set by the Cloud
- Dataflow service, and only as a transition from `JOB_STATE_RUNNING`.
- JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has
- been explicitly cancelled. This is a terminal job state. This state
- may only be set via a Cloud Dataflow `UpdateJob` call, and only if the
- job has not yet reached another terminal state.
- JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was
- successfully updated, meaning that this job was stopped and another
- job was started, inheriting state from this one. This is a terminal
- job state. This state may only be set by the Cloud Dataflow service,
- and only as a transition from `JOB_STATE_RUNNING`.
- JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in
- the process of draining. A draining job has stopped pulling from its
- input sources and is processing any data that remains in-flight. This
- state may be set via a Cloud Dataflow `UpdateJob` call, but only as a
- transition from `JOB_STATE_RUNNING`. Jobs that are draining may only
- transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or
- `JOB_STATE_FAILED`.
- JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been
- drained. A drained job terminated by stopping pulling from its input
- sources and processing any data that remained in-flight when draining
- was requested. This state is a terminal state, may only be set by the
- Cloud Dataflow service, and only as a transition from
- `JOB_STATE_DRAINING`.
- """
- JOB_STATE_UNKNOWN = 0
- JOB_STATE_STOPPED = 1
- JOB_STATE_RUNNING = 2
- JOB_STATE_DONE = 3
- JOB_STATE_FAILED = 4
- JOB_STATE_CANCELLED = 5
- JOB_STATE_UPDATED = 6
- JOB_STATE_DRAINING = 7
- JOB_STATE_DRAINED = 8
-
- class RequestedStateValueValuesEnum(_messages.Enum):
- """The job's requested state. `UpdateJob` may be used to switch between
- the `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING` states, by setting
- requested_state. `UpdateJob` may also be used to directly set a job's
- requested state to `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably
- terminating the job if it has not already reached a terminal state.
-
- Values:
- JOB_STATE_UNKNOWN: The job's run state isn't specified.
- JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not
- yet started to run.
- JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is
- currently running.
- JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully
- completed. This is a terminal job state. This state may be set by the
- Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It
- may also be set via a Cloud Dataflow `UpdateJob` call, if the job has
- not yet reached a terminal state.
- JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed.
- This is a terminal job state. This state may only be set by the Cloud
- Dataflow service, and only as a transition from `JOB_STATE_RUNNING`.
- JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has
- been explicitly cancelled. This is a terminal job state. This state
- may only be set via a Cloud Dataflow `UpdateJob` call, and only if the
- job has not yet reached another terminal state.
- JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was
- successfully updated, meaning that this job was stopped and another
- job was started, inheriting state from this one. This is a terminal
- job state. This state may only be set by the Cloud Dataflow service,
- and only as a transition from `JOB_STATE_RUNNING`.
- JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in
- the process of draining. A draining job has stopped pulling from its
- input sources and is processing any data that remains in-flight. This
- state may be set via a Cloud Dataflow `UpdateJob` call, but only as a
- transition from `JOB_STATE_RUNNING`. Jobs that are draining may only
- transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or
- `JOB_STATE_FAILED`.
- JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been
- drained. A drained job terminated by stopping pulling from its input
- sources and processing any data that remained in-flight when draining
- was requested. This state is a terminal state, may only be set by the
- Cloud Dataflow service, and only as a transition from
- `JOB_STATE_DRAINING`.
- """
- JOB_STATE_UNKNOWN = 0
- JOB_STATE_STOPPED = 1
- JOB_STATE_RUNNING = 2
- JOB_STATE_DONE = 3
- JOB_STATE_FAILED = 4
- JOB_STATE_CANCELLED = 5
- JOB_STATE_UPDATED = 6
- JOB_STATE_DRAINING = 7
- JOB_STATE_DRAINED = 8
-
- class TypeValueValuesEnum(_messages.Enum):
- """The type of Cloud Dataflow job.
-
- Values:
- JOB_TYPE_UNKNOWN: The type of the job is unspecified, or unknown.
- JOB_TYPE_BATCH: A batch job with a well-defined end point: data is read,
- data is processed, data is written, and the job is done.
- JOB_TYPE_STREAMING: A continuously streaming job with no end: data is
- read, processed, and written continuously.
- """
- JOB_TYPE_UNKNOWN = 0
- JOB_TYPE_BATCH = 1
- JOB_TYPE_STREAMING = 2
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class LabelsValue(_messages.Message):
- """User-defined labels for this job. The labels map can contain no more
- than 64 entries. Entries of the labels map are UTF8 strings that comply
- with the following restrictions: * Keys must conform to regexp:
- \p{Ll}\p{Lo}{0,62} * Values must conform to regexp:
- [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally
- constrained to be <= 128 bytes in size.
-
- Messages:
- AdditionalProperty: An additional property for a LabelsValue object.
-
- Fields:
- additionalProperties: Additional properties of type LabelsValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a LabelsValue object.
-
- Fields:
- key: Name of the additional property.
- value: A string attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.StringField(2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class TransformNameMappingValue(_messages.Message):
- """The map of transform name prefixes of the job to be replaced to the
- corresponding name prefixes of the new job.
-
- Messages:
- AdditionalProperty: An additional property for a
- TransformNameMappingValue object.
-
- Fields:
- additionalProperties: Additional properties of type
- TransformNameMappingValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a TransformNameMappingValue object.
-
- Fields:
- key: Name of the additional property.
- value: A string attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.StringField(2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- clientRequestId = _messages.StringField(1)
- createTime = _messages.StringField(2)
- currentState = _messages.EnumField('CurrentStateValueValuesEnum', 3)
- currentStateTime = _messages.StringField(4)
- environment = _messages.MessageField('Environment', 5)
- executionInfo = _messages.MessageField('JobExecutionInfo', 6)
- id = _messages.StringField(7)
- labels = _messages.MessageField('LabelsValue', 8)
- location = _messages.StringField(9)
- name = _messages.StringField(10)
- projectId = _messages.StringField(11)
- replaceJobId = _messages.StringField(12)
- replacedByJobId = _messages.StringField(13)
- requestedState = _messages.EnumField('RequestedStateValueValuesEnum', 14)
- steps = _messages.MessageField('Step', 15, repeated=True)
- tempFiles = _messages.StringField(16, repeated=True)
- transformNameMapping = _messages.MessageField('TransformNameMappingValue', 17)
- type = _messages.EnumField('TypeValueValuesEnum', 18)
-
-
-class JobExecutionInfo(_messages.Message):
- """Additional information about how a Cloud Dataflow job will be executed
- that isn't contained in the submitted job.
-
- Messages:
- StagesValue: A mapping from each stage to the information about that
- stage.
-
- Fields:
- stages: A mapping from each stage to the information about that stage.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class StagesValue(_messages.Message):
- """A mapping from each stage to the information about that stage.
-
- Messages:
- AdditionalProperty: An additional property for a StagesValue object.
-
- Fields:
- additionalProperties: Additional properties of type StagesValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a StagesValue object.
-
- Fields:
- key: Name of the additional property.
- value: A JobExecutionStageInfo attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('JobExecutionStageInfo', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- stages = _messages.MessageField('StagesValue', 1)
-
-
-class JobExecutionStageInfo(_messages.Message):
- """Contains information about how a particular google.dataflow.v1beta3.Step
- will be executed.
-
- Fields:
- stepName: The steps associated with the execution stage. Note that stages
- may have several steps, and that a given step might be run by more than
- one stage.
- """
-
- stepName = _messages.StringField(1, repeated=True)
-
-
-class JobMessage(_messages.Message):
- """A particular message pertaining to a Dataflow job.
-
- Enums:
- MessageImportanceValueValuesEnum: Importance level of the message.
-
- Fields:
- id: Identifies the message. This is automatically generated by the
- service; the caller should treat it as an opaque string.
- messageImportance: Importance level of the message.
- messageText: The text of the message.
- time: The timestamp of the message.
- """
-
- class MessageImportanceValueValuesEnum(_messages.Enum):
- """Importance level of the message.
-
- Values:
- JOB_MESSAGE_IMPORTANCE_UNKNOWN: The message importance isn't specified,
- or is unknown.
- JOB_MESSAGE_DEBUG: The message is at the 'debug' level: typically only
- useful for software engineers working on the code the job is running.
- Typically, Dataflow pipeline runners do not display log messages at
- this level by default.
- JOB_MESSAGE_DETAILED: The message is at the 'detailed' level: somewhat
- verbose, but potentially useful to users. Typically, Dataflow
- pipeline runners do not display log messages at this level by default.
- These messages are displayed by default in the Dataflow monitoring UI.
- JOB_MESSAGE_BASIC: The message is at the 'basic' level: useful for
- keeping track of the execution of a Dataflow pipeline. Typically,
- Dataflow pipeline runners display log messages at this level by
- default, and these messages are displayed by default in the Dataflow
- monitoring UI.
- JOB_MESSAGE_WARNING: The message is at the 'warning' level: indicating a
- condition pertaining to a job which may require human intervention.
- Typically, Dataflow pipeline runners display log messages at this
- level by default, and these messages are displayed by default in the
- Dataflow monitoring UI.
- JOB_MESSAGE_ERROR: The message is at the 'error' level: indicating a
- condition preventing a job from succeeding. Typically, Dataflow
- pipeline runners display log messages at this level by default, and
- these messages are displayed by default in the Dataflow monitoring UI.
- """
- JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
- JOB_MESSAGE_DEBUG = 1
- JOB_MESSAGE_DETAILED = 2
- JOB_MESSAGE_BASIC = 3
- JOB_MESSAGE_WARNING = 4
- JOB_MESSAGE_ERROR = 5
-
- id = _messages.StringField(1)
- messageImportance = _messages.EnumField('MessageImportanceValueValuesEnum', 2)
- messageText = _messages.StringField(3)
- time = _messages.StringField(4)
-
-
-class JobMetrics(_messages.Message):
- """JobMetrics contains a collection of metrics descibing the detailed
- progress of a Dataflow job. Metrics correspond to user-defined and system-
- defined metrics in the job. This resource captures only the most recent
- values of each metric; time-series data can be queried for them (under the
- same metric names) from Cloud Monitoring.
-
- Fields:
- metricTime: Timestamp as of which metric values are current.
- metrics: All metrics for this job.
- """
-
- metricTime = _messages.StringField(1)
- metrics = _messages.MessageField('MetricUpdate', 2, repeated=True)
-
-
-class KeyRangeDataDiskAssignment(_messages.Message):
- """Data disk assignment information for a specific key-range of a sharded
- computation. Currently we only support UTF-8 character splits to simplify
- encoding into JSON.
-
- Fields:
- dataDisk: The name of the data disk where data for this range is stored.
- This name is local to the Google Cloud Platform project and uniquely
- identifies the disk within that project, for example
- "myproject-1014-104817-4c2-harness-0-disk-1".
- end: The end (exclusive) of the key range.
- start: The start (inclusive) of the key range.
- """
-
- dataDisk = _messages.StringField(1)
- end = _messages.StringField(2)
- start = _messages.StringField(3)
-
-
-class KeyRangeLocation(_messages.Message):
- """Location information for a specific key-range of a sharded computation.
- Currently we only support UTF-8 character splits to simplify encoding into
- JSON.
-
- Fields:
- dataDisk: The name of the data disk where data for this range is stored.
- This name is local to the Google Cloud Platform project and uniquely
- identifies the disk within that project, for example
- "myproject-1014-104817-4c2-harness-0-disk-1".
- deliveryEndpoint: The physical location of this range assignment to be
- used for streaming computation cross-worker message delivery.
- end: The end (exclusive) of the key range.
- persistentDirectory: The location of the persistent state for this range,
- as a persistent directory in the worker local filesystem.
- start: The start (inclusive) of the key range.
- """
-
- dataDisk = _messages.StringField(1)
- deliveryEndpoint = _messages.StringField(2)
- end = _messages.StringField(3)
- persistentDirectory = _messages.StringField(4)
- start = _messages.StringField(5)
-
-
-class LeaseWorkItemRequest(_messages.Message):
- """Request to lease WorkItems.
-
- Fields:
- currentWorkerTime: The current timestamp at the worker.
- location: The location which contains the WorkItem's job.
- requestedLeaseDuration: The initial lease period.
- workItemTypes: Filter for WorkItem type.
- workerCapabilities: Worker capabilities. WorkItems might be limited to
- workers with specific capabilities.
- workerId: Identifies the worker leasing work -- typically the ID of the
- virtual machine running the worker.
- """
-
- currentWorkerTime = _messages.StringField(1)
- location = _messages.StringField(2)
- requestedLeaseDuration = _messages.StringField(3)
- workItemTypes = _messages.StringField(4, repeated=True)
- workerCapabilities = _messages.StringField(5, repeated=True)
- workerId = _messages.StringField(6)
-
-
-class LeaseWorkItemResponse(_messages.Message):
- """Response to a request to lease WorkItems.
-
- Fields:
- workItems: A list of the leased WorkItems.
- """
-
- workItems = _messages.MessageField('WorkItem', 1, repeated=True)
-
-
-class ListJobMessagesResponse(_messages.Message):
- """Response to a request to list job messages.
-
- Fields:
- jobMessages: Messages in ascending timestamp order.
- nextPageToken: The token to obtain the next page of results if there are
- more.
- """
-
- jobMessages = _messages.MessageField('JobMessage', 1, repeated=True)
- nextPageToken = _messages.StringField(2)
-
-
-class ListJobsResponse(_messages.Message):
- """Response to a request to list Cloud Dataflow jobs. This may be a partial
- response, depending on the page size in the ListJobsRequest.
-
- Fields:
- failedLocation: Zero or more messages describing locations that failed to
- respond.
- jobs: A subset of the requested job information.
- nextPageToken: Set if there may be more results than fit in this response.
- """
-
- failedLocation = _messages.MessageField('FailedLocation', 1, repeated=True)
- jobs = _messages.MessageField('Job', 2, repeated=True)
- nextPageToken = _messages.StringField(3)
-
-
-class MapTask(_messages.Message):
- """MapTask consists of an ordered set of instructions, each of which
- describes one particular low-level operation for the worker to perform in
- order to accomplish the MapTask's WorkItem. Each instruction must appear in
- the list before any instructions which depends on its output.
-
- Fields:
- instructions: The instructions in the MapTask.
- stageName: System-defined name of the stage containing this MapTask.
- Unique across the workflow.
- systemName: System-defined name of this MapTask. Unique across the
- workflow.
- """
-
- instructions = _messages.MessageField('ParallelInstruction', 1, repeated=True)
- stageName = _messages.StringField(2)
- systemName = _messages.StringField(3)
-
-
-class MetricShortId(_messages.Message):
- """The metric short id is returned to the user alongside an offset into
- ReportWorkItemStatusRequest
-
- Fields:
- metricIndex: The index of the corresponding metric in the
- ReportWorkItemStatusRequest. Required.
- shortId: The service-generated short identifier for the metric.
- """
-
- metricIndex = _messages.IntegerField(1, variant=_messages.Variant.INT32)
- shortId = _messages.IntegerField(2)
-
-
-class MetricStructuredName(_messages.Message):
- """Identifies a metric, by describing the source which generated the metric.
-
- Messages:
- ContextValue: Zero or more labeled fields which identify the part of the
- job this metric is associated with, such as the name of a step or
- collection. For example, built-in counters associated with steps will
- have context['step'] = <step-name>. Counters associated with
- PCollections in the SDK will have context['pcollection'] = <pcollection-
- name>.
-
- Fields:
- context: Zero or more labeled fields which identify the part of the job
- this metric is associated with, such as the name of a step or
- collection. For example, built-in counters associated with steps will
- have context['step'] = <step-name>. Counters associated with
- PCollections in the SDK will have context['pcollection'] = <pcollection-
- name>.
- name: Worker-defined metric name.
- origin: Origin (namespace) of metric name. May be blank for user-define
- metrics; will be "dataflow" for metrics defined by the Dataflow service
- or SDK.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class ContextValue(_messages.Message):
- """Zero or more labeled fields which identify the part of the job this
- metric is associated with, such as the name of a step or collection. For
- example, built-in counters associated with steps will have context['step']
- = <step-name>. Counters associated with PCollections in the SDK will have
- context['pcollection'] = <pcollection-name>.
-
- Messages:
- AdditionalProperty: An additional property for a ContextValue object.
-
- Fields:
- additionalProperties: Additional properties of type ContextValue
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a ContextValue object.
-
- Fields:
- key: Name of the additional property.
- value: A string attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.StringField(2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- context = _messages.MessageField('ContextValue', 1)
- name = _messages.StringField(2)
- origin = _messages.StringField(3)
-
-
-class MetricUpdate(_messages.Message):
- """Describes the state of a metric.
-
- Fields:
- cumulative: True if this metric is reported as the total cumulative
- aggregate value accumulated since the worker started working on this
- WorkItem. By default this is false, indicating that this metric is
- reported as a delta that is not associated with any WorkItem.
- internal: Worker-computed aggregate value for internal use by the Dataflow
- service.
- kind: Metric aggregation kind. The possible metric aggregation kinds are
- "Sum", "Max", "Min", "Mean", "Set", "And", and "Or". The specified
- aggregation kind is case-insensitive. If omitted, this is not an
- aggregated value but instead a single metric sample value.
- meanCount: Worker-computed aggregate value for the "Mean" aggregation
- kind. This holds the count of the aggregated values and is used in
- combination with mean_sum above to obtain the actual mean aggregate
- value. The only possible value type is Long.
- meanSum: Worker-computed aggregate value for the "Mean" aggregation kind.
- This holds the sum of the aggregated values and is used in combination
- with mean_count below to obtain the actual mean aggregate value. The
- only possible value types are Long and Double.
- name: Name of the metric.
- scalar: Worker-computed aggregate value for aggregation kinds "Sum",
- "Max", "Min", "And", and "Or". The possible value types are Long,
- Double, and Boolean.
- set: Worker-computed aggregate value for the "Set" aggregation kind. The
- only possible value type is a list of Values whose type can be Long,
- Double, or String, according to the metric's type. All Values in the
- list must be of the same type.
- updateTime: Timestamp associated with the metric value. Optional when
- workers are reporting work progress; it will be filled in responses from
- the metrics API.
- """
-
- cumulative = _messages.BooleanField(1)
- internal = _messages.MessageField('extra_types.JsonValue', 2)
- kind = _messages.StringField(3)
- meanCount = _messages.MessageField('extra_types.JsonValue', 4)
- meanSum = _messages.MessageField('extra_types.JsonValue', 5)
- name = _messages.MessageField('MetricStructuredName', 6)
- scalar = _messages.MessageField('extra_types.JsonValue', 7)
- set = _messages.MessageField('extra_types.JsonValue', 8)
- updateTime = _messages.StringField(9)
-
-
-class MountedDataDisk(_messages.Message):
- """Describes mounted data disk.
-
- Fields:
- dataDisk: The name of the data disk. This name is local to the Google
- Cloud Platform project and uniquely identifies the disk within that
- project, for example "myproject-1014-104817-4c2-harness-0-disk-1".
- """
-
- dataDisk = _messages.StringField(1)
-
-
-class MultiOutputInfo(_messages.Message):
- """Information about an output of a multi-output DoFn.
-
- Fields:
- tag: The id of the tag the user code will emit to this output by; this
- should correspond to the tag of some SideInputInfo.
- """
-
- tag = _messages.StringField(1)
-
-
-class NameAndKind(_messages.Message):
- """Basic metadata about a counter.
-
- Enums:
- KindValueValuesEnum: Counter aggregation kind.
-
- Fields:
- kind: Counter aggregation kind.
- name: Name of the counter.
- """
-
- class KindValueValuesEnum(_messages.Enum):
- """Counter aggregation kind.
-
- Values:
- INVALID: Counter aggregation kind was not set.
- SUM: Aggregated value is the sum of all contributed values.
- MAX: Aggregated value is the max of all contributed values.
- MIN: Aggregated value is the min of all contributed values.
- MEAN: Aggregated value is the mean of all contributed values.
- OR: Aggregated value represents the logical 'or' of all contributed
- values.
- AND: Aggregated value represents the logical 'and' of all contributed
- values.
- SET: Aggregated value is a set of unique contributed values.
- DISTRIBUTION: Aggregated value captures statistics about a distribution.
- """
- INVALID = 0
- SUM = 1
- MAX = 2
- MIN = 3
- MEAN = 4
- OR = 5
- AND = 6
- SET = 7
- DISTRIBUTION = 8
-
- kind = _messages.EnumField('KindValueValuesEnum', 1)
- name = _messages.StringField(2)
-
-
-class Package(_messages.Message):
- """The packages that must be installed in order for a worker to run the
- steps of the Cloud Dataflow job that will be assigned to its worker pool.
- This is the mechanism by which the Cloud Dataflow SDK causes code to be
- loaded onto the workers. For example, the Cloud Dataflow Java SDK might use
- this to install jars containing the user's code and all of the various
- dependencies (libraries, data files, etc.) required in order for that code
- to run.
-
- Fields:
- location: The resource to read the package from. The supported resource
- type is: Google Cloud Storage: storage.googleapis.com/{bucket}
- bucket.storage.googleapis.com/
- name: The name of the package.
- """
-
- location = _messages.StringField(1)
- name = _messages.StringField(2)
-
-
-class ParDoInstruction(_messages.Message):
- """An instruction that does a ParDo operation. Takes one main input and zero
- or more side inputs, and produces zero or more outputs. Runs user code.
-
- Messages:
- UserFnValue: The user function to invoke.
-
- Fields:
- input: The input.
- multiOutputInfos: Information about each of the outputs, if user_fn is a
- MultiDoFn.
- numOutputs: The number of outputs.
- sideInputs: Zero or more side inputs.
- userFn: The user function to invoke.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class UserFnValue(_messages.Message):
- """The user function to invoke.
-
- Messages:
- AdditionalProperty: An additional property for a UserFnValue object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a UserFnValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- input = _messages.MessageField('InstructionInput', 1)
- multiOutputInfos = _messages.MessageField('MultiOutputInfo', 2, repeated=True)
- numOutputs = _messages.IntegerField(3, variant=_messages.Variant.INT32)
- sideInputs = _messages.MessageField('SideInputInfo', 4, repeated=True)
- userFn = _messages.MessageField('UserFnValue', 5)
-
-
-class ParallelInstruction(_messages.Message):
- """Describes a particular operation comprising a MapTask.
-
- Fields:
- flatten: Additional information for Flatten instructions.
- name: User-provided name of this operation.
- originalName: System-defined name for the operation in the original
- workflow graph.
- outputs: Describes the outputs of the instruction.
- parDo: Additional information for ParDo instructions.
- partialGroupByKey: Additional information for PartialGroupByKey
- instructions.
- read: Additional information for Read instructions.
- systemName: System-defined name of this operation. Unique across the
- workflow.
- write: Additional information for Write instructions.
- """
-
- flatten = _messages.MessageField('FlattenInstruction', 1)
- name = _messages.StringField(2)
- originalName = _messages.StringField(3)
- outputs = _messages.MessageField('InstructionOutput', 4, repeated=True)
- parDo = _messages.MessageField('ParDoInstruction', 5)
- partialGroupByKey = _messages.MessageField('PartialGroupByKeyInstruction', 6)
- read = _messages.MessageField('ReadInstruction', 7)
- systemName = _messages.StringField(8)
- write = _messages.MessageField('WriteInstruction', 9)
-
-
-class PartialGroupByKeyInstruction(_messages.Message):
- """An instruction that does a partial group-by-key. One input and one
- output.
-
- Messages:
- InputElementCodecValue: The codec to use for interpreting an element in
- the input PTable.
- ValueCombiningFnValue: The value combining function to invoke.
-
- Fields:
- input: Describes the input to the partial group-by-key instruction.
- inputElementCodec: The codec to use for interpreting an element in the
- input PTable.
- originalCombineValuesInputStoreName: If this instruction includes a
- combining function this is the name of the intermediate store between
- the GBK and the CombineValues.
- originalCombineValuesStepName: If this instruction includes a combining
- function, this is the name of the CombineValues instruction lifted into
- this instruction.
- sideInputs: Zero or more side inputs.
- valueCombiningFn: The value combining function to invoke.
- """
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class InputElementCodecValue(_messages.Message):
- """The codec to use for interpreting an element in the input PTable.
-
- Messages:
- AdditionalProperty: An additional property for a InputElementCodecValue
- object.
-
- Fields:
- additionalProperties: Properties of the object.
- """
-
- class AdditionalProperty(_messages.Message):
- """An additional property for a InputElementCodecValue object.
-
- Fields:
- key: Name of the additional property.
- value: A extra_types.JsonValue attribute.
- """
-
- key = _messages.StringField(1)
- value = _messages.MessageField('extra_types.JsonValue', 2)
-
- additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
-
- @encoding.MapUnrecognizedFields('additionalProperties')
- class ValueCombiningFnValue(_messages.Message):
- """The value combining function to invoke.
-
- Messages:
- AdditionalProperty: An additional property for a ValueCombiningFnValue
- object.
-
-
<TRUNCATED>
[2/6] beam git commit: Moving dataflow runner related code to vendor
directory
Posted by al...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
new file mode 100644
index 0000000..a42154e
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
@@ -0,0 +1,4173 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Generated message classes for dataflow version v1b3.
+
+Develops and executes data processing patterns like ETL, batch computation,
+and continuous computation.
+"""
+# NOTE: This file is autogenerated and should not be edited by hand.
+
+from apitools.base.protorpclite import messages as _messages
+from apitools.base.py import encoding
+from apitools.base.py import extra_types
+
+
+package = 'dataflow'
+
+
+class ApproximateProgress(_messages.Message):
+ """Obsolete in favor of ApproximateReportedProgress and
+ ApproximateSplitRequest.
+
+ Fields:
+ percentComplete: Obsolete.
+ position: Obsolete.
+ remainingTime: Obsolete.
+ """
+
+ percentComplete = _messages.FloatField(1, variant=_messages.Variant.FLOAT)
+ position = _messages.MessageField('Position', 2)
+ remainingTime = _messages.StringField(3)
+
+
+class ApproximateReportedProgress(_messages.Message):
+ """A progress measurement of a WorkItem by a worker.
+
+ Fields:
+ consumedParallelism: Total amount of parallelism in the portion of input
+ of this task that has already been consumed and is no longer active. In
+ the first two examples above (see remaining_parallelism), the value
+ should be 29 or 2 respectively. The sum of remaining_parallelism and
+ consumed_parallelism should equal the total amount of parallelism in
+ this work item. If specified, must be finite.
+ fractionConsumed: Completion as fraction of the input consumed, from 0.0
+ (beginning, nothing consumed), to 1.0 (end of the input, entire input
+ consumed).
+ position: A Position within the work to represent a progress.
+ remainingParallelism: Total amount of parallelism in the input of this
+ task that remains, (i.e. can be delegated to this task and any new tasks
+ via dynamic splitting). Always at least 1 for non-finished work items
+ and 0 for finished. "Amount of parallelism" refers to how many non-
+ empty parts of the input can be read in parallel. This does not
+ necessarily equal number of records. An input that can be read in
+ parallel down to the individual records is called "perfectly
+ splittable". An example of non-perfectly parallelizable input is a
+ block-compressed file format where a block of records has to be read as
+ a whole, but different blocks can be read in parallel. Examples: * If
+ we are processing record #30 (starting at 1) out of 50 in a perfectly
+ splittable 50-record input, this value should be 21 (20 remaining + 1
+ current). * If we are reading through block 3 in a block-compressed file
+ consisting of 5 blocks, this value should be 3 (since blocks 4 and 5
+ can be processed in parallel by new tasks via dynamic splitting and
+ the current task remains processing block 3). * If we are reading
+ through the last block in a block-compressed file, or reading or
+ processing the last record in a perfectly splittable input, this value
+ should be 1, because apart from the current task, no additional
+ remainder can be split off.
+ """
+
+ consumedParallelism = _messages.MessageField('ReportedParallelism', 1)
+ fractionConsumed = _messages.FloatField(2)
+ position = _messages.MessageField('Position', 3)
+ remainingParallelism = _messages.MessageField('ReportedParallelism', 4)
+
+
+class ApproximateSplitRequest(_messages.Message):
+ """A suggestion by the service to the worker to dynamically split the
+ WorkItem.
+
+ Fields:
+ fractionConsumed: A fraction at which to split the work item, from 0.0
+ (beginning of the input) to 1.0 (end of the input).
+ position: A Position at which to split the work item.
+ """
+
+ fractionConsumed = _messages.FloatField(1)
+ position = _messages.MessageField('Position', 2)
+
+
+class AutoscalingSettings(_messages.Message):
+ """Settings for WorkerPool autoscaling.
+
+ Enums:
+ AlgorithmValueValuesEnum: The algorithm to use for autoscaling.
+
+ Fields:
+ algorithm: The algorithm to use for autoscaling.
+ maxNumWorkers: The maximum number of workers to cap scaling at.
+ """
+
+ class AlgorithmValueValuesEnum(_messages.Enum):
+ """The algorithm to use for autoscaling.
+
+ Values:
+ AUTOSCALING_ALGORITHM_UNKNOWN: The algorithm is unknown, or unspecified.
+ AUTOSCALING_ALGORITHM_NONE: Disable autoscaling.
+ AUTOSCALING_ALGORITHM_BASIC: Increase worker count over time to reduce
+ job execution time.
+ """
+ AUTOSCALING_ALGORITHM_UNKNOWN = 0
+ AUTOSCALING_ALGORITHM_NONE = 1
+ AUTOSCALING_ALGORITHM_BASIC = 2
+
+ algorithm = _messages.EnumField('AlgorithmValueValuesEnum', 1)
+ maxNumWorkers = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+
+
+class ComputationTopology(_messages.Message):
+ """All configuration data for a particular Computation.
+
+ Fields:
+ computationId: The ID of the computation.
+ inputs: The inputs to the computation.
+ keyRanges: The key ranges processed by the computation.
+ outputs: The outputs from the computation.
+ stateFamilies: The state family values.
+ systemStageName: The system stage name.
+ userStageName: The user stage name.
+ """
+
+ computationId = _messages.StringField(1)
+ inputs = _messages.MessageField('StreamLocation', 2, repeated=True)
+ keyRanges = _messages.MessageField('KeyRangeLocation', 3, repeated=True)
+ outputs = _messages.MessageField('StreamLocation', 4, repeated=True)
+ stateFamilies = _messages.MessageField('StateFamilyConfig', 5, repeated=True)
+ systemStageName = _messages.StringField(6)
+ userStageName = _messages.StringField(7)
+
+
+class ConcatPosition(_messages.Message):
+ """A position that encapsulates an inner position and an index for the inner
+ position. A ConcatPosition can be used by a reader of a source that
+ encapsulates a set of other sources.
+
+ Fields:
+ index: Index of the inner source.
+ position: Position within the inner source.
+ """
+
+ index = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+ position = _messages.MessageField('Position', 2)
+
+
+class CounterMetadata(_messages.Message):
+ """CounterMetadata includes all static non-name non-value counter
+ attributes.
+
+ Enums:
+ KindValueValuesEnum: Counter aggregation kind.
+ StandardUnitsValueValuesEnum: System defined Units, see above enum.
+
+ Fields:
+ description: Human-readable description of the counter semantics.
+ kind: Counter aggregation kind.
+ otherUnits: A string referring to the unit type.
+ standardUnits: System defined Units, see above enum.
+ """
+
+ class KindValueValuesEnum(_messages.Enum):
+ """Counter aggregation kind.
+
+ Values:
+ INVALID: Counter aggregation kind was not set.
+ SUM: Aggregated value is the sum of all contributed values.
+ MAX: Aggregated value is the max of all contributed values.
+ MIN: Aggregated value is the min of all contributed values.
+ MEAN: Aggregated value is the mean of all contributed values.
+ OR: Aggregated value represents the logical 'or' of all contributed
+ values.
+ AND: Aggregated value represents the logical 'and' of all contributed
+ values.
+ SET: Aggregated value is a set of unique contributed values.
+ DISTRIBUTION: Aggregated value captures statistics about a distribution.
+ """
+ INVALID = 0
+ SUM = 1
+ MAX = 2
+ MIN = 3
+ MEAN = 4
+ OR = 5
+ AND = 6
+ SET = 7
+ DISTRIBUTION = 8
+
+ class StandardUnitsValueValuesEnum(_messages.Enum):
+ """System defined Units, see above enum.
+
+ Values:
+ BYTES: Counter returns a value in bytes.
+ BYTES_PER_SEC: Counter returns a value in bytes per second.
+ MILLISECONDS: Counter returns a value in milliseconds.
+ MICROSECONDS: Counter returns a value in microseconds.
+ NANOSECONDS: Counter returns a value in nanoseconds.
+ TIMESTAMP_MSEC: Counter returns a timestamp in milliseconds.
+ TIMESTAMP_USEC: Counter returns a timestamp in microseconds.
+ TIMESTAMP_NSEC: Counter returns a timestamp in nanoseconds.
+ """
+ BYTES = 0
+ BYTES_PER_SEC = 1
+ MILLISECONDS = 2
+ MICROSECONDS = 3
+ NANOSECONDS = 4
+ TIMESTAMP_MSEC = 5
+ TIMESTAMP_USEC = 6
+ TIMESTAMP_NSEC = 7
+
+ description = _messages.StringField(1)
+ kind = _messages.EnumField('KindValueValuesEnum', 2)
+ otherUnits = _messages.StringField(3)
+ standardUnits = _messages.EnumField('StandardUnitsValueValuesEnum', 4)
+
+
+class CounterStructuredName(_messages.Message):
+ """Identifies a counter within a per-job namespace. Counters whose
+ structured names are the same get merged into a single value for the job.
+
+ Enums:
+ PortionValueValuesEnum: Portion of this counter, either key or value.
+ StandardOriginValueValuesEnum: One of the standard Origins defined above.
+
+ Fields:
+ componentStepName: Name of the optimized step being executed by the
+ workers.
+ executionStepName: Name of the stage. An execution step contains multiple
+ component steps.
+ name: Counter name. Not necessarily globally-unique, but unique within the
+ context of the other fields. Required.
+ originalStepName: System generated name of the original step in the user's
+ graph, before optimization.
+ otherOrigin: A string containing the origin of the counter.
+ portion: Portion of this counter, either key or value.
+ standardOrigin: One of the standard Origins defined above.
+ workerId: ID of a particular worker.
+ """
+
+ class PortionValueValuesEnum(_messages.Enum):
+ """Portion of this counter, either key or value.
+
+ Values:
+ ALL: Counter portion has not been set.
+ KEY: Counter reports a key.
+ VALUE: Counter reports a value.
+ """
+ ALL = 0
+ KEY = 1
+ VALUE = 2
+
+ class StandardOriginValueValuesEnum(_messages.Enum):
+ """One of the standard Origins defined above.
+
+ Values:
+ DATAFLOW: Counter was created by the Dataflow system.
+ USER: Counter was created by the user.
+ """
+ DATAFLOW = 0
+ USER = 1
+
+ componentStepName = _messages.StringField(1)
+ executionStepName = _messages.StringField(2)
+ name = _messages.StringField(3)
+ originalStepName = _messages.StringField(4)
+ otherOrigin = _messages.StringField(5)
+ portion = _messages.EnumField('PortionValueValuesEnum', 6)
+ standardOrigin = _messages.EnumField('StandardOriginValueValuesEnum', 7)
+ workerId = _messages.StringField(8)
+
+
+class CounterStructuredNameAndMetadata(_messages.Message):
+ """A single message which encapsulates structured name and metadata for a
+ given counter.
+
+ Fields:
+ metadata: Metadata associated with a counter
+ name: Structured name of the counter.
+ """
+
+ metadata = _messages.MessageField('CounterMetadata', 1)
+ name = _messages.MessageField('CounterStructuredName', 2)
+
+
+class CounterUpdate(_messages.Message):
+ """An update to a Counter sent from a worker.
+
+ Fields:
+ boolean: Boolean value for And, Or.
+ cumulative: True if this counter is reported as the total cumulative
+ aggregate value accumulated since the worker started working on this
+ WorkItem. By default this is false, indicating that this counter is
+ reported as a delta.
+ distribution: Distribution data
+ floatingPoint: Floating point value for Sum, Max, Min.
+ floatingPointList: List of floating point numbers, for Set.
+ floatingPointMean: Floating point mean aggregation value for Mean.
+ integer: Integer value for Sum, Max, Min.
+ integerList: List of integers, for Set.
+ integerMean: Integer mean aggregation value for Mean.
+ internal: Value for internally-defined counters used by the Dataflow
+ service.
+ nameAndKind: Counter name and aggregation type.
+ shortId: The service-generated short identifier for this counter. The
+ short_id -> (name, metadata) mapping is constant for the lifetime of a
+ job.
+ stringList: List of strings, for Set.
+ structuredNameAndMetadata: Counter structured name and metadata.
+ """
+
+ boolean = _messages.BooleanField(1)
+ cumulative = _messages.BooleanField(2)
+ distribution = _messages.MessageField('DistributionUpdate', 3)
+ floatingPoint = _messages.FloatField(4)
+ floatingPointList = _messages.MessageField('FloatingPointList', 5)
+ floatingPointMean = _messages.MessageField('FloatingPointMean', 6)
+ integer = _messages.MessageField('SplitInt64', 7)
+ integerList = _messages.MessageField('IntegerList', 8)
+ integerMean = _messages.MessageField('IntegerMean', 9)
+ internal = _messages.MessageField('extra_types.JsonValue', 10)
+ nameAndKind = _messages.MessageField('NameAndKind', 11)
+ shortId = _messages.IntegerField(12)
+ stringList = _messages.MessageField('StringList', 13)
+ structuredNameAndMetadata = _messages.MessageField('CounterStructuredNameAndMetadata', 14)
+
+
+class CreateJobFromTemplateRequest(_messages.Message):
+ """A request to create a Cloud Dataflow job from a template.
+
+ Messages:
+ ParametersValue: The runtime parameters to pass to the job.
+
+ Fields:
+ environment: The runtime environment for the job.
+ gcsPath: Required. A Cloud Storage path to the template from which to
+ create the job. Must be a valid Cloud Storage URL, beginning with
+ `gs://`.
+ jobName: Required. The job name to use for the created job.
+ parameters: The runtime parameters to pass to the job.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class ParametersValue(_messages.Message):
+ """The runtime parameters to pass to the job.
+
+ Messages:
+ AdditionalProperty: An additional property for a ParametersValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type ParametersValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a ParametersValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A string attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.StringField(2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ environment = _messages.MessageField('RuntimeEnvironment', 1)
+ gcsPath = _messages.StringField(2)
+ jobName = _messages.StringField(3)
+ parameters = _messages.MessageField('ParametersValue', 4)
+
+
+class CustomSourceLocation(_messages.Message):
+ """Identifies the location of a custom souce.
+
+ Fields:
+ stateful: Whether this source is stateful.
+ """
+
+ stateful = _messages.BooleanField(1)
+
+
+class DataDiskAssignment(_messages.Message):
+ """Data disk assignment for a given VM instance.
+
+ Fields:
+ dataDisks: Mounted data disks. The order is important a data disk's
+ 0-based index in this list defines which persistent directory the disk
+ is mounted to, for example the list of {
+ "myproject-1014-104817-4c2-harness-0-disk-0" }, {
+ "myproject-1014-104817-4c2-harness-0-disk-1" }.
+ vmInstance: VM instance name the data disks mounted to, for example
+ "myproject-1014-104817-4c2-harness-0".
+ """
+
+ dataDisks = _messages.StringField(1, repeated=True)
+ vmInstance = _messages.StringField(2)
+
+
+class DataflowProjectsJobsCreateRequest(_messages.Message):
+ """A DataflowProjectsJobsCreateRequest object.
+
+ Enums:
+ ViewValueValuesEnum: The level of information requested in response.
+
+ Fields:
+ job: A Job resource to be passed as the request body.
+ location: The location that contains this job.
+ projectId: The ID of the Cloud Platform project that the job belongs to.
+ replaceJobId: Deprecated. This field is now in the Job message.
+ view: The level of information requested in response.
+ """
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """The level of information requested in response.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ job = _messages.MessageField('Job', 1)
+ location = _messages.StringField(2)
+ projectId = _messages.StringField(3, required=True)
+ replaceJobId = _messages.StringField(4)
+ view = _messages.EnumField('ViewValueValuesEnum', 5)
+
+
+class DataflowProjectsJobsDebugGetConfigRequest(_messages.Message):
+ """A DataflowProjectsJobsDebugGetConfigRequest object.
+
+ Fields:
+ getDebugConfigRequest: A GetDebugConfigRequest resource to be passed as
+ the request body.
+ jobId: The job id.
+ projectId: The project id.
+ """
+
+ getDebugConfigRequest = _messages.MessageField('GetDebugConfigRequest', 1)
+ jobId = _messages.StringField(2, required=True)
+ projectId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsJobsDebugSendCaptureRequest(_messages.Message):
+ """A DataflowProjectsJobsDebugSendCaptureRequest object.
+
+ Fields:
+ jobId: The job id.
+ projectId: The project id.
+ sendDebugCaptureRequest: A SendDebugCaptureRequest resource to be passed
+ as the request body.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ sendDebugCaptureRequest = _messages.MessageField('SendDebugCaptureRequest', 3)
+
+
+class DataflowProjectsJobsGetMetricsRequest(_messages.Message):
+ """A DataflowProjectsJobsGetMetricsRequest object.
+
+ Fields:
+ jobId: The job to get messages for.
+ location: The location which contains the job specified by job_id.
+ projectId: A project id.
+ startTime: Return only metric data that has changed since this time.
+ Default is to return all information about all metrics for the job.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ location = _messages.StringField(2)
+ projectId = _messages.StringField(3, required=True)
+ startTime = _messages.StringField(4)
+
+
+class DataflowProjectsJobsGetRequest(_messages.Message):
+ """A DataflowProjectsJobsGetRequest object.
+
+ Enums:
+ ViewValueValuesEnum: The level of information requested in response.
+
+ Fields:
+ jobId: The job ID.
+ location: The location that contains this job.
+ projectId: The ID of the Cloud Platform project that the job belongs to.
+ view: The level of information requested in response.
+ """
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """The level of information requested in response.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ jobId = _messages.StringField(1, required=True)
+ location = _messages.StringField(2)
+ projectId = _messages.StringField(3, required=True)
+ view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class DataflowProjectsJobsListRequest(_messages.Message):
+ """A DataflowProjectsJobsListRequest object.
+
+ Enums:
+ FilterValueValuesEnum: The kind of filter to use.
+ ViewValueValuesEnum: Level of information requested in response. Default
+ is `JOB_VIEW_SUMMARY`.
+
+ Fields:
+ filter: The kind of filter to use.
+ location: The location that contains this job.
+ pageSize: If there are many jobs, limit response to at most this many. The
+ actual number of jobs returned will be the lesser of max_responses and
+ an unspecified server-defined limit.
+ pageToken: Set this to the 'next_page_token' field of a previous response
+ to request additional results in a long list.
+ projectId: The project which owns the jobs.
+ view: Level of information requested in response. Default is
+ `JOB_VIEW_SUMMARY`.
+ """
+
+ class FilterValueValuesEnum(_messages.Enum):
+ """The kind of filter to use.
+
+ Values:
+ UNKNOWN: <no description>
+ ALL: <no description>
+ TERMINATED: <no description>
+ ACTIVE: <no description>
+ """
+ UNKNOWN = 0
+ ALL = 1
+ TERMINATED = 2
+ ACTIVE = 3
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """Level of information requested in response. Default is
+ `JOB_VIEW_SUMMARY`.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ filter = _messages.EnumField('FilterValueValuesEnum', 1)
+ location = _messages.StringField(2)
+ pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+ pageToken = _messages.StringField(4)
+ projectId = _messages.StringField(5, required=True)
+ view = _messages.EnumField('ViewValueValuesEnum', 6)
+
+
+class DataflowProjectsJobsMessagesListRequest(_messages.Message):
+ """A DataflowProjectsJobsMessagesListRequest object.
+
+ Enums:
+ MinimumImportanceValueValuesEnum: Filter to only get messages with
+ importance >= level
+
+ Fields:
+ endTime: Return only messages with timestamps < end_time. The default is
+ now (i.e. return up to the latest messages available).
+ jobId: The job to get messages about.
+ location: The location which contains the job specified by job_id.
+ minimumImportance: Filter to only get messages with importance >= level
+ pageSize: If specified, determines the maximum number of messages to
+ return. If unspecified, the service may choose an appropriate default,
+ or may return an arbitrarily large number of results.
+ pageToken: If supplied, this should be the value of next_page_token
+ returned by an earlier call. This will cause the next page of results to
+ be returned.
+ projectId: A project id.
+ startTime: If specified, return only messages with timestamps >=
+ start_time. The default is the job creation time (i.e. beginning of
+ messages).
+ """
+
+ class MinimumImportanceValueValuesEnum(_messages.Enum):
+ """Filter to only get messages with importance >= level
+
+ Values:
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
+ JOB_MESSAGE_DEBUG: <no description>
+ JOB_MESSAGE_DETAILED: <no description>
+ JOB_MESSAGE_BASIC: <no description>
+ JOB_MESSAGE_WARNING: <no description>
+ JOB_MESSAGE_ERROR: <no description>
+ """
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
+ JOB_MESSAGE_DEBUG = 1
+ JOB_MESSAGE_DETAILED = 2
+ JOB_MESSAGE_BASIC = 3
+ JOB_MESSAGE_WARNING = 4
+ JOB_MESSAGE_ERROR = 5
+
+ endTime = _messages.StringField(1)
+ jobId = _messages.StringField(2, required=True)
+ location = _messages.StringField(3)
+ minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 4)
+ pageSize = _messages.IntegerField(5, variant=_messages.Variant.INT32)
+ pageToken = _messages.StringField(6)
+ projectId = _messages.StringField(7, required=True)
+ startTime = _messages.StringField(8)
+
+
+class DataflowProjectsJobsUpdateRequest(_messages.Message):
+ """A DataflowProjectsJobsUpdateRequest object.
+
+ Fields:
+ job: A Job resource to be passed as the request body.
+ jobId: The job ID.
+ location: The location that contains this job.
+ projectId: The ID of the Cloud Platform project that the job belongs to.
+ """
+
+ job = _messages.MessageField('Job', 1)
+ jobId = _messages.StringField(2, required=True)
+ location = _messages.StringField(3)
+ projectId = _messages.StringField(4, required=True)
+
+
+class DataflowProjectsJobsWorkItemsLeaseRequest(_messages.Message):
+ """A DataflowProjectsJobsWorkItemsLeaseRequest object.
+
+ Fields:
+ jobId: Identifies the workflow job this worker belongs to.
+ leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the
+ request body.
+ projectId: Identifies the project this worker belongs to.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2)
+ projectId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsJobsWorkItemsReportStatusRequest(_messages.Message):
+ """A DataflowProjectsJobsWorkItemsReportStatusRequest object.
+
+ Fields:
+ jobId: The job which the WorkItem is part of.
+ projectId: The project which owns the WorkItem's job.
+ reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be
+ passed as the request body.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ projectId = _messages.StringField(2, required=True)
+ reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 3)
+
+
+class DataflowProjectsLocationsJobsCreateRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsCreateRequest object.
+
+ Enums:
+ ViewValueValuesEnum: The level of information requested in response.
+
+ Fields:
+ job: A Job resource to be passed as the request body.
+ location: The location that contains this job.
+ projectId: The ID of the Cloud Platform project that the job belongs to.
+ replaceJobId: Deprecated. This field is now in the Job message.
+ view: The level of information requested in response.
+ """
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """The level of information requested in response.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ job = _messages.MessageField('Job', 1)
+ location = _messages.StringField(2, required=True)
+ projectId = _messages.StringField(3, required=True)
+ replaceJobId = _messages.StringField(4)
+ view = _messages.EnumField('ViewValueValuesEnum', 5)
+
+
+class DataflowProjectsLocationsJobsGetMetricsRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsGetMetricsRequest object.
+
+ Fields:
+ jobId: The job to get messages for.
+ location: The location which contains the job specified by job_id.
+ projectId: A project id.
+ startTime: Return only metric data that has changed since this time.
+ Default is to return all information about all metrics for the job.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ location = _messages.StringField(2, required=True)
+ projectId = _messages.StringField(3, required=True)
+ startTime = _messages.StringField(4)
+
+
+class DataflowProjectsLocationsJobsGetRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsGetRequest object.
+
+ Enums:
+ ViewValueValuesEnum: The level of information requested in response.
+
+ Fields:
+ jobId: The job ID.
+ location: The location that contains this job.
+ projectId: The ID of the Cloud Platform project that the job belongs to.
+ view: The level of information requested in response.
+ """
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """The level of information requested in response.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ jobId = _messages.StringField(1, required=True)
+ location = _messages.StringField(2, required=True)
+ projectId = _messages.StringField(3, required=True)
+ view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class DataflowProjectsLocationsJobsListRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsListRequest object.
+
+ Enums:
+ FilterValueValuesEnum: The kind of filter to use.
+ ViewValueValuesEnum: Level of information requested in response. Default
+ is `JOB_VIEW_SUMMARY`.
+
+ Fields:
+ filter: The kind of filter to use.
+ location: The location that contains this job.
+ pageSize: If there are many jobs, limit response to at most this many. The
+ actual number of jobs returned will be the lesser of max_responses and
+ an unspecified server-defined limit.
+ pageToken: Set this to the 'next_page_token' field of a previous response
+ to request additional results in a long list.
+ projectId: The project which owns the jobs.
+ view: Level of information requested in response. Default is
+ `JOB_VIEW_SUMMARY`.
+ """
+
+ class FilterValueValuesEnum(_messages.Enum):
+ """The kind of filter to use.
+
+ Values:
+ UNKNOWN: <no description>
+ ALL: <no description>
+ TERMINATED: <no description>
+ ACTIVE: <no description>
+ """
+ UNKNOWN = 0
+ ALL = 1
+ TERMINATED = 2
+ ACTIVE = 3
+
+ class ViewValueValuesEnum(_messages.Enum):
+ """Level of information requested in response. Default is
+ `JOB_VIEW_SUMMARY`.
+
+ Values:
+ JOB_VIEW_UNKNOWN: <no description>
+ JOB_VIEW_SUMMARY: <no description>
+ JOB_VIEW_ALL: <no description>
+ """
+ JOB_VIEW_UNKNOWN = 0
+ JOB_VIEW_SUMMARY = 1
+ JOB_VIEW_ALL = 2
+
+ filter = _messages.EnumField('FilterValueValuesEnum', 1)
+ location = _messages.StringField(2, required=True)
+ pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+ pageToken = _messages.StringField(4)
+ projectId = _messages.StringField(5, required=True)
+ view = _messages.EnumField('ViewValueValuesEnum', 6)
+
+
+class DataflowProjectsLocationsJobsMessagesListRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsMessagesListRequest object.
+
+ Enums:
+ MinimumImportanceValueValuesEnum: Filter to only get messages with
+ importance >= level
+
+ Fields:
+ endTime: Return only messages with timestamps < end_time. The default is
+ now (i.e. return up to the latest messages available).
+ jobId: The job to get messages about.
+ location: The location which contains the job specified by job_id.
+ minimumImportance: Filter to only get messages with importance >= level
+ pageSize: If specified, determines the maximum number of messages to
+ return. If unspecified, the service may choose an appropriate default,
+ or may return an arbitrarily large number of results.
+ pageToken: If supplied, this should be the value of next_page_token
+ returned by an earlier call. This will cause the next page of results to
+ be returned.
+ projectId: A project id.
+ startTime: If specified, return only messages with timestamps >=
+ start_time. The default is the job creation time (i.e. beginning of
+ messages).
+ """
+
+ class MinimumImportanceValueValuesEnum(_messages.Enum):
+ """Filter to only get messages with importance >= level
+
+ Values:
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description>
+ JOB_MESSAGE_DEBUG: <no description>
+ JOB_MESSAGE_DETAILED: <no description>
+ JOB_MESSAGE_BASIC: <no description>
+ JOB_MESSAGE_WARNING: <no description>
+ JOB_MESSAGE_ERROR: <no description>
+ """
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
+ JOB_MESSAGE_DEBUG = 1
+ JOB_MESSAGE_DETAILED = 2
+ JOB_MESSAGE_BASIC = 3
+ JOB_MESSAGE_WARNING = 4
+ JOB_MESSAGE_ERROR = 5
+
+ endTime = _messages.StringField(1)
+ jobId = _messages.StringField(2, required=True)
+ location = _messages.StringField(3, required=True)
+ minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 4)
+ pageSize = _messages.IntegerField(5, variant=_messages.Variant.INT32)
+ pageToken = _messages.StringField(6)
+ projectId = _messages.StringField(7, required=True)
+ startTime = _messages.StringField(8)
+
+
+class DataflowProjectsLocationsJobsUpdateRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsUpdateRequest object.
+
+ Fields:
+ job: A Job resource to be passed as the request body.
+ jobId: The job ID.
+ location: The location that contains this job.
+ projectId: The ID of the Cloud Platform project that the job belongs to.
+ """
+
+ job = _messages.MessageField('Job', 1)
+ jobId = _messages.StringField(2, required=True)
+ location = _messages.StringField(3, required=True)
+ projectId = _messages.StringField(4, required=True)
+
+
+class DataflowProjectsLocationsJobsWorkItemsLeaseRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsWorkItemsLeaseRequest object.
+
+ Fields:
+ jobId: Identifies the workflow job this worker belongs to.
+ leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the
+ request body.
+ location: The location which contains the WorkItem's job.
+ projectId: Identifies the project this worker belongs to.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2)
+ location = _messages.StringField(3, required=True)
+ projectId = _messages.StringField(4, required=True)
+
+
+class DataflowProjectsLocationsJobsWorkItemsReportStatusRequest(_messages.Message):
+ """A DataflowProjectsLocationsJobsWorkItemsReportStatusRequest object.
+
+ Fields:
+ jobId: The job which the WorkItem is part of.
+ location: The location which contains the WorkItem's job.
+ projectId: The project which owns the WorkItem's job.
+ reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be
+ passed as the request body.
+ """
+
+ jobId = _messages.StringField(1, required=True)
+ location = _messages.StringField(2, required=True)
+ projectId = _messages.StringField(3, required=True)
+ reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 4)
+
+
+class DataflowProjectsTemplatesCreateRequest(_messages.Message):
+ """A DataflowProjectsTemplatesCreateRequest object.
+
+ Fields:
+ createJobFromTemplateRequest: A CreateJobFromTemplateRequest resource to
+ be passed as the request body.
+ projectId: Required. The ID of the Cloud Platform project that the job
+ belongs to.
+ """
+
+ createJobFromTemplateRequest = _messages.MessageField('CreateJobFromTemplateRequest', 1)
+ projectId = _messages.StringField(2, required=True)
+
+
+class DataflowProjectsWorkerMessagesRequest(_messages.Message):
+ """A DataflowProjectsWorkerMessagesRequest object.
+
+ Fields:
+ projectId: The project to send the WorkerMessages to.
+ sendWorkerMessagesRequest: A SendWorkerMessagesRequest resource to be
+ passed as the request body.
+ """
+
+ projectId = _messages.StringField(1, required=True)
+ sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 2)
+
+
+class DerivedSource(_messages.Message):
+ """Specification of one of the bundles produced as a result of splitting a
+ Source (e.g. when executing a SourceSplitRequest, or when splitting an
+ active task using WorkItemStatus.dynamic_source_split), relative to the
+ source being split.
+
+ Enums:
+ DerivationModeValueValuesEnum: What source to base the produced source on
+ (if any).
+
+ Fields:
+ derivationMode: What source to base the produced source on (if any).
+ source: Specification of the source.
+ """
+
+ class DerivationModeValueValuesEnum(_messages.Enum):
+ """What source to base the produced source on (if any).
+
+ Values:
+ SOURCE_DERIVATION_MODE_UNKNOWN: The source derivation is unknown, or
+ unspecified.
+ SOURCE_DERIVATION_MODE_INDEPENDENT: Produce a completely independent
+ Source with no base.
+ SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: Produce a Source based on the
+ Source being split.
+ SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: Produce a Source based on the
+ base of the Source being split.
+ """
+ SOURCE_DERIVATION_MODE_UNKNOWN = 0
+ SOURCE_DERIVATION_MODE_INDEPENDENT = 1
+ SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2
+ SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3
+
+ derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1)
+ source = _messages.MessageField('Source', 2)
+
+
+class Disk(_messages.Message):
+ """Describes the data disk used by a workflow job.
+
+ Fields:
+ diskType: Disk storage type, as defined by Google Compute Engine. This
+ must be a disk type appropriate to the project and zone in which the
+ workers will run. If unknown or unspecified, the service will attempt
+ to choose a reasonable default. For example, the standard persistent
+ disk type is a resource name typically ending in "pd-standard". If SSD
+ persistent disks are available, the resource name typically ends with
+ "pd-ssd". The actual valid values are defined the Google Compute Engine
+ API, not by the Cloud Dataflow API; consult the Google Compute Engine
+ documentation for more information about determining the set of
+ available disk types for a particular project and zone. Google Compute
+ Engine Disk types are local to a particular project in a particular
+ zone, and so the resource name will typically look something like this:
+ compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-
+ standard
+ mountPoint: Directory in a VM where disk is mounted.
+ sizeGb: Size of disk in GB. If zero or unspecified, the service will
+ attempt to choose a reasonable default.
+ """
+
+ diskType = _messages.StringField(1)
+ mountPoint = _messages.StringField(2)
+ sizeGb = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+
+
+class DistributionUpdate(_messages.Message):
+ """A metric value representing a distribution.
+
+ Fields:
+ count: The count of the number of elements present in the distribution.
+ max: The maximum value present in the distribution.
+ min: The minimum value present in the distribution.
+ sum: Use an int64 since we'd prefer the added precision. If overflow is a
+ common problem we can detect it and use an additional int64 or a double.
+ sumOfSquares: Use a double since the sum of squares is likely to overflow
+ int64.
+ """
+
+ count = _messages.MessageField('SplitInt64', 1)
+ max = _messages.MessageField('SplitInt64', 2)
+ min = _messages.MessageField('SplitInt64', 3)
+ sum = _messages.MessageField('SplitInt64', 4)
+ sumOfSquares = _messages.FloatField(5)
+
+
+class DynamicSourceSplit(_messages.Message):
+ """When a task splits using WorkItemStatus.dynamic_source_split, this
+ message describes the two parts of the split relative to the description of
+ the current task's input.
+
+ Fields:
+ primary: Primary part (continued to be processed by worker). Specified
+ relative to the previously-current source. Becomes current.
+ residual: Residual part (returned to the pool of work). Specified relative
+ to the previously-current source.
+ """
+
+ primary = _messages.MessageField('DerivedSource', 1)
+ residual = _messages.MessageField('DerivedSource', 2)
+
+
+class Environment(_messages.Message):
+ """Describes the environment in which a Dataflow Job runs.
+
+ Messages:
+ InternalExperimentsValue: Experimental settings.
+ SdkPipelineOptionsValue: The Cloud Dataflow SDK pipeline options specified
+ by the user. These options are passed through the service and are used
+ to recreate the SDK pipeline options on the worker in a language
+ agnostic and platform independent way.
+ UserAgentValue: A description of the process that generated the request.
+ VersionValue: A structure describing which components and their versions
+ of the service are required in order to run the job.
+
+ Fields:
+ clusterManagerApiService: The type of cluster manager API to use. If
+ unknown or unspecified, the service will attempt to choose a reasonable
+ default. This should be in the form of the API service name, e.g.
+ "compute.googleapis.com".
+ dataset: The dataset for the current project where various workflow
+ related tables are stored. The supported resource type is: Google
+ BigQuery: bigquery.googleapis.com/{dataset}
+ experiments: The list of experiments to enable.
+ internalExperiments: Experimental settings.
+ sdkPipelineOptions: The Cloud Dataflow SDK pipeline options specified by
+ the user. These options are passed through the service and are used to
+ recreate the SDK pipeline options on the worker in a language agnostic
+ and platform independent way.
+ serviceAccountEmail: Identity to run virtual machines as. Defaults to the
+ default account.
+ tempStoragePrefix: The prefix of the resources the system should use for
+ temporary storage. The system will append the suffix "/temp-{JOBNAME}
+ to this resource prefix, where {JOBNAME} is the value of the job_name
+ field. The resulting bucket and object prefix is used as the prefix of
+ the resources used to store temporary data needed during the job
+ execution. NOTE: This will override the value in taskrunner_settings.
+ The supported resource type is: Google Cloud Storage:
+ storage.googleapis.com/{bucket}/{object}
+ bucket.storage.googleapis.com/{object}
+ userAgent: A description of the process that generated the request.
+ version: A structure describing which components and their versions of the
+ service are required in order to run the job.
+ workerPools: The worker pools. At least one "harness" worker pool must be
+ specified in order for the job to have workers.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class InternalExperimentsValue(_messages.Message):
+ """Experimental settings.
+
+ Messages:
+ AdditionalProperty: An additional property for a
+ InternalExperimentsValue object.
+
+ Fields:
+ additionalProperties: Properties of the object. Contains field @type
+ with type URL.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a InternalExperimentsValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class SdkPipelineOptionsValue(_messages.Message):
+ """The Cloud Dataflow SDK pipeline options specified by the user. These
+ options are passed through the service and are used to recreate the SDK
+ pipeline options on the worker in a language agnostic and platform
+ independent way.
+
+ Messages:
+ AdditionalProperty: An additional property for a SdkPipelineOptionsValue
+ object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a SdkPipelineOptionsValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class UserAgentValue(_messages.Message):
+ """A description of the process that generated the request.
+
+ Messages:
+ AdditionalProperty: An additional property for a UserAgentValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a UserAgentValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class VersionValue(_messages.Message):
+ """A structure describing which components and their versions of the
+ service are required in order to run the job.
+
+ Messages:
+ AdditionalProperty: An additional property for a VersionValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a VersionValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ clusterManagerApiService = _messages.StringField(1)
+ dataset = _messages.StringField(2)
+ experiments = _messages.StringField(3, repeated=True)
+ internalExperiments = _messages.MessageField('InternalExperimentsValue', 4)
+ sdkPipelineOptions = _messages.MessageField('SdkPipelineOptionsValue', 5)
+ serviceAccountEmail = _messages.StringField(6)
+ tempStoragePrefix = _messages.StringField(7)
+ userAgent = _messages.MessageField('UserAgentValue', 8)
+ version = _messages.MessageField('VersionValue', 9)
+ workerPools = _messages.MessageField('WorkerPool', 10, repeated=True)
+
+
+class FailedLocation(_messages.Message):
+ """Indicates which location failed to respond to a request for data.
+
+ Fields:
+ name: The name of the failed location.
+ """
+
+ name = _messages.StringField(1)
+
+
+class FlattenInstruction(_messages.Message):
+ """An instruction that copies its inputs (zero or more) to its (single)
+ output.
+
+ Fields:
+ inputs: Describes the inputs to the flatten instruction.
+ """
+
+ inputs = _messages.MessageField('InstructionInput', 1, repeated=True)
+
+
+class FloatingPointList(_messages.Message):
+ """A metric value representing a list of floating point numbers.
+
+ Fields:
+ elements: Elements of the list.
+ """
+
+ elements = _messages.FloatField(1, repeated=True)
+
+
+class FloatingPointMean(_messages.Message):
+ """A representation of a floating point mean metric contribution.
+
+ Fields:
+ count: The number of values being aggregated.
+ sum: The sum of all values being aggregated.
+ """
+
+ count = _messages.MessageField('SplitInt64', 1)
+ sum = _messages.FloatField(2)
+
+
+class GetDebugConfigRequest(_messages.Message):
+ """Request to get updated debug configuration for component.
+
+ Fields:
+ componentId: The internal component id for which debug configuration is
+ requested.
+ workerId: The worker id, i.e., VM hostname.
+ """
+
+ componentId = _messages.StringField(1)
+ workerId = _messages.StringField(2)
+
+
+class GetDebugConfigResponse(_messages.Message):
+ """Response to a get debug configuration request.
+
+ Fields:
+ config: The encoded debug configuration for the requested component.
+ """
+
+ config = _messages.StringField(1)
+
+
+class InstructionInput(_messages.Message):
+ """An input of an instruction, as a reference to an output of a producer
+ instruction.
+
+ Fields:
+ outputNum: The output index (origin zero) within the producer.
+ producerInstructionIndex: The index (origin zero) of the parallel
+ instruction that produces the output to be consumed by this input. This
+ index is relative to the list of instructions in this input's
+ instruction's containing MapTask.
+ """
+
+ outputNum = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+ producerInstructionIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+
+
+class InstructionOutput(_messages.Message):
+ """An output of an instruction.
+
+ Messages:
+ CodecValue: The codec to use to encode data being written via this output.
+
+ Fields:
+ codec: The codec to use to encode data being written via this output.
+ name: The user-provided name of this output.
+ onlyCountKeyBytes: For system-generated byte and mean byte metrics,
+ certain instructions should only report the key size.
+ onlyCountValueBytes: For system-generated byte and mean byte metrics,
+ certain instructions should only report the value size.
+ originalName: System-defined name for this output in the original workflow
+ graph. Outputs that do not contribute to an original instruction do not
+ set this.
+ systemName: System-defined name of this output. Unique across the
+ workflow.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class CodecValue(_messages.Message):
+ """The codec to use to encode data being written via this output.
+
+ Messages:
+ AdditionalProperty: An additional property for a CodecValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a CodecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ codec = _messages.MessageField('CodecValue', 1)
+ name = _messages.StringField(2)
+ onlyCountKeyBytes = _messages.BooleanField(3)
+ onlyCountValueBytes = _messages.BooleanField(4)
+ originalName = _messages.StringField(5)
+ systemName = _messages.StringField(6)
+
+
+class IntegerList(_messages.Message):
+ """A metric value representing a list of integers.
+
+ Fields:
+ elements: Elements of the list.
+ """
+
+ elements = _messages.MessageField('SplitInt64', 1, repeated=True)
+
+
+class IntegerMean(_messages.Message):
+ """A representation of an integer mean metric contribution.
+
+ Fields:
+ count: The number of values being aggregated.
+ sum: The sum of all values being aggregated.
+ """
+
+ count = _messages.MessageField('SplitInt64', 1)
+ sum = _messages.MessageField('SplitInt64', 2)
+
+
+class Job(_messages.Message):
+ """Defines a job to be run by the Cloud Dataflow service.
+
+ Enums:
+ CurrentStateValueValuesEnum: The current state of the job. Jobs are
+ created in the `JOB_STATE_STOPPED` state unless otherwise specified. A
+ job in the `JOB_STATE_RUNNING` state may asynchronously enter a terminal
+ state. After a job has reached a terminal state, no further state
+ updates may be made. This field may be mutated by the Cloud Dataflow
+ service; callers cannot mutate it.
+ RequestedStateValueValuesEnum: The job's requested state. `UpdateJob` may
+ be used to switch between the `JOB_STATE_STOPPED` and
+ `JOB_STATE_RUNNING` states, by setting requested_state. `UpdateJob` may
+ also be used to directly set a job's requested state to
+ `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the
+ job if it has not already reached a terminal state.
+ TypeValueValuesEnum: The type of Cloud Dataflow job.
+
+ Messages:
+ LabelsValue: User-defined labels for this job. The labels map can contain
+ no more than 64 entries. Entries of the labels map are UTF8 strings
+ that comply with the following restrictions: * Keys must conform to
+ regexp: \p{Ll}\p{Lo}{0,62} * Values must conform to regexp:
+ [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally
+ constrained to be <= 128 bytes in size.
+ TransformNameMappingValue: The map of transform name prefixes of the job
+ to be replaced to the corresponding name prefixes of the new job.
+
+ Fields:
+ clientRequestId: The client's unique identifier of the job, re-used across
+ retried attempts. If this field is set, the service will ensure its
+ uniqueness. The request to create a job will fail if the service has
+ knowledge of a previously submitted job with the same client's ID and
+ job name. The caller may use this field to ensure idempotence of job
+ creation across retried attempts to create a job. By default, the field
+ is empty and, in that case, the service ignores it.
+ createTime: The timestamp when the job was initially created. Immutable
+ and set by the Cloud Dataflow service.
+ currentState: The current state of the job. Jobs are created in the
+ `JOB_STATE_STOPPED` state unless otherwise specified. A job in the
+ `JOB_STATE_RUNNING` state may asynchronously enter a terminal state.
+ After a job has reached a terminal state, no further state updates may
+ be made. This field may be mutated by the Cloud Dataflow service;
+ callers cannot mutate it.
+ currentStateTime: The timestamp associated with the current state.
+ environment: The environment for the job.
+ executionInfo: Information about how the Cloud Dataflow service will run
+ the job.
+ id: The unique ID of this job. This field is set by the Cloud Dataflow
+ service when the Job is created, and is immutable for the life of the
+ job.
+ labels: User-defined labels for this job. The labels map can contain no
+ more than 64 entries. Entries of the labels map are UTF8 strings that
+ comply with the following restrictions: * Keys must conform to regexp:
+ \p{Ll}\p{Lo}{0,62} * Values must conform to regexp:
+ [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally
+ constrained to be <= 128 bytes in size.
+ location: The location that contains this job.
+ name: The user-specified Cloud Dataflow job name. Only one Job with a
+ given name may exist in a project at any given time. If a caller
+ attempts to create a Job with the same name as an already-existing Job,
+ the attempt returns the existing Job. The name must match the regular
+ expression `[a-z]([-a-z0-9]{0,38}[a-z0-9])?`
+ projectId: The ID of the Cloud Platform project that the job belongs to.
+ replaceJobId: If this job is an update of an existing job, this field is
+ the job ID of the job it replaced. When sending a `CreateJobRequest`,
+ you can update a job by specifying it here. The job named here is
+ stopped, and its intermediate state is transferred to this job.
+ replacedByJobId: If another job is an update of this job (and thus, this
+ job is in `JOB_STATE_UPDATED`), this field contains the ID of that job.
+ requestedState: The job's requested state. `UpdateJob` may be used to
+ switch between the `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING` states,
+ by setting requested_state. `UpdateJob` may also be used to directly
+ set a job's requested state to `JOB_STATE_CANCELLED` or
+ `JOB_STATE_DONE`, irrevocably terminating the job if it has not already
+ reached a terminal state.
+ steps: The top-level steps that constitute the entire job.
+ tempFiles: A set of files the system should be aware of that are used for
+ temporary storage. These temporary files will be removed on job
+ completion. No duplicates are allowed. No file patterns are supported.
+ The supported files are: Google Cloud Storage:
+ storage.googleapis.com/{bucket}/{object}
+ bucket.storage.googleapis.com/{object}
+ transformNameMapping: The map of transform name prefixes of the job to be
+ replaced to the corresponding name prefixes of the new job.
+ type: The type of Cloud Dataflow job.
+ """
+
+ class CurrentStateValueValuesEnum(_messages.Enum):
+ """The current state of the job. Jobs are created in the
+ `JOB_STATE_STOPPED` state unless otherwise specified. A job in the
+ `JOB_STATE_RUNNING` state may asynchronously enter a terminal state. After
+ a job has reached a terminal state, no further state updates may be made.
+ This field may be mutated by the Cloud Dataflow service; callers cannot
+ mutate it.
+
+ Values:
+ JOB_STATE_UNKNOWN: The job's run state isn't specified.
+ JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not
+ yet started to run.
+ JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is
+ currently running.
+ JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully
+ completed. This is a terminal job state. This state may be set by the
+ Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It
+ may also be set via a Cloud Dataflow `UpdateJob` call, if the job has
+ not yet reached a terminal state.
+ JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed.
+ This is a terminal job state. This state may only be set by the Cloud
+ Dataflow service, and only as a transition from `JOB_STATE_RUNNING`.
+ JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has
+ been explicitly cancelled. This is a terminal job state. This state
+ may only be set via a Cloud Dataflow `UpdateJob` call, and only if the
+ job has not yet reached another terminal state.
+ JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was
+ successfully updated, meaning that this job was stopped and another
+ job was started, inheriting state from this one. This is a terminal
+ job state. This state may only be set by the Cloud Dataflow service,
+ and only as a transition from `JOB_STATE_RUNNING`.
+ JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in
+ the process of draining. A draining job has stopped pulling from its
+ input sources and is processing any data that remains in-flight. This
+ state may be set via a Cloud Dataflow `UpdateJob` call, but only as a
+ transition from `JOB_STATE_RUNNING`. Jobs that are draining may only
+ transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or
+ `JOB_STATE_FAILED`.
+ JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been
+ drained. A drained job terminated by stopping pulling from its input
+ sources and processing any data that remained in-flight when draining
+ was requested. This state is a terminal state, may only be set by the
+ Cloud Dataflow service, and only as a transition from
+ `JOB_STATE_DRAINING`.
+ """
+ JOB_STATE_UNKNOWN = 0
+ JOB_STATE_STOPPED = 1
+ JOB_STATE_RUNNING = 2
+ JOB_STATE_DONE = 3
+ JOB_STATE_FAILED = 4
+ JOB_STATE_CANCELLED = 5
+ JOB_STATE_UPDATED = 6
+ JOB_STATE_DRAINING = 7
+ JOB_STATE_DRAINED = 8
+
+ class RequestedStateValueValuesEnum(_messages.Enum):
+ """The job's requested state. `UpdateJob` may be used to switch between
+ the `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING` states, by setting
+ requested_state. `UpdateJob` may also be used to directly set a job's
+ requested state to `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably
+ terminating the job if it has not already reached a terminal state.
+
+ Values:
+ JOB_STATE_UNKNOWN: The job's run state isn't specified.
+ JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not
+ yet started to run.
+ JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is
+ currently running.
+ JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully
+ completed. This is a terminal job state. This state may be set by the
+ Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It
+ may also be set via a Cloud Dataflow `UpdateJob` call, if the job has
+ not yet reached a terminal state.
+ JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed.
+ This is a terminal job state. This state may only be set by the Cloud
+ Dataflow service, and only as a transition from `JOB_STATE_RUNNING`.
+ JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has
+ been explicitly cancelled. This is a terminal job state. This state
+ may only be set via a Cloud Dataflow `UpdateJob` call, and only if the
+ job has not yet reached another terminal state.
+ JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was
+ successfully updated, meaning that this job was stopped and another
+ job was started, inheriting state from this one. This is a terminal
+ job state. This state may only be set by the Cloud Dataflow service,
+ and only as a transition from `JOB_STATE_RUNNING`.
+ JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in
+ the process of draining. A draining job has stopped pulling from its
+ input sources and is processing any data that remains in-flight. This
+ state may be set via a Cloud Dataflow `UpdateJob` call, but only as a
+ transition from `JOB_STATE_RUNNING`. Jobs that are draining may only
+ transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or
+ `JOB_STATE_FAILED`.
+ JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been
+ drained. A drained job terminated by stopping pulling from its input
+ sources and processing any data that remained in-flight when draining
+ was requested. This state is a terminal state, may only be set by the
+ Cloud Dataflow service, and only as a transition from
+ `JOB_STATE_DRAINING`.
+ """
+ JOB_STATE_UNKNOWN = 0
+ JOB_STATE_STOPPED = 1
+ JOB_STATE_RUNNING = 2
+ JOB_STATE_DONE = 3
+ JOB_STATE_FAILED = 4
+ JOB_STATE_CANCELLED = 5
+ JOB_STATE_UPDATED = 6
+ JOB_STATE_DRAINING = 7
+ JOB_STATE_DRAINED = 8
+
+ class TypeValueValuesEnum(_messages.Enum):
+ """The type of Cloud Dataflow job.
+
+ Values:
+ JOB_TYPE_UNKNOWN: The type of the job is unspecified, or unknown.
+ JOB_TYPE_BATCH: A batch job with a well-defined end point: data is read,
+ data is processed, data is written, and the job is done.
+ JOB_TYPE_STREAMING: A continuously streaming job with no end: data is
+ read, processed, and written continuously.
+ """
+ JOB_TYPE_UNKNOWN = 0
+ JOB_TYPE_BATCH = 1
+ JOB_TYPE_STREAMING = 2
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class LabelsValue(_messages.Message):
+ """User-defined labels for this job. The labels map can contain no more
+ than 64 entries. Entries of the labels map are UTF8 strings that comply
+ with the following restrictions: * Keys must conform to regexp:
+ \p{Ll}\p{Lo}{0,62} * Values must conform to regexp:
+ [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally
+ constrained to be <= 128 bytes in size.
+
+ Messages:
+ AdditionalProperty: An additional property for a LabelsValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type LabelsValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a LabelsValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A string attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.StringField(2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class TransformNameMappingValue(_messages.Message):
+ """The map of transform name prefixes of the job to be replaced to the
+ corresponding name prefixes of the new job.
+
+ Messages:
+ AdditionalProperty: An additional property for a
+ TransformNameMappingValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type
+ TransformNameMappingValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a TransformNameMappingValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A string attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.StringField(2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ clientRequestId = _messages.StringField(1)
+ createTime = _messages.StringField(2)
+ currentState = _messages.EnumField('CurrentStateValueValuesEnum', 3)
+ currentStateTime = _messages.StringField(4)
+ environment = _messages.MessageField('Environment', 5)
+ executionInfo = _messages.MessageField('JobExecutionInfo', 6)
+ id = _messages.StringField(7)
+ labels = _messages.MessageField('LabelsValue', 8)
+ location = _messages.StringField(9)
+ name = _messages.StringField(10)
+ projectId = _messages.StringField(11)
+ replaceJobId = _messages.StringField(12)
+ replacedByJobId = _messages.StringField(13)
+ requestedState = _messages.EnumField('RequestedStateValueValuesEnum', 14)
+ steps = _messages.MessageField('Step', 15, repeated=True)
+ tempFiles = _messages.StringField(16, repeated=True)
+ transformNameMapping = _messages.MessageField('TransformNameMappingValue', 17)
+ type = _messages.EnumField('TypeValueValuesEnum', 18)
+
+
+class JobExecutionInfo(_messages.Message):
+ """Additional information about how a Cloud Dataflow job will be executed
+ that isn't contained in the submitted job.
+
+ Messages:
+ StagesValue: A mapping from each stage to the information about that
+ stage.
+
+ Fields:
+ stages: A mapping from each stage to the information about that stage.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class StagesValue(_messages.Message):
+ """A mapping from each stage to the information about that stage.
+
+ Messages:
+ AdditionalProperty: An additional property for a StagesValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type StagesValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a StagesValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A JobExecutionStageInfo attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('JobExecutionStageInfo', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ stages = _messages.MessageField('StagesValue', 1)
+
+
+class JobExecutionStageInfo(_messages.Message):
+ """Contains information about how a particular google.dataflow.v1beta3.Step
+ will be executed.
+
+ Fields:
+ stepName: The steps associated with the execution stage. Note that stages
+ may have several steps, and that a given step might be run by more than
+ one stage.
+ """
+
+ stepName = _messages.StringField(1, repeated=True)
+
+
+class JobMessage(_messages.Message):
+ """A particular message pertaining to a Dataflow job.
+
+ Enums:
+ MessageImportanceValueValuesEnum: Importance level of the message.
+
+ Fields:
+ id: Identifies the message. This is automatically generated by the
+ service; the caller should treat it as an opaque string.
+ messageImportance: Importance level of the message.
+ messageText: The text of the message.
+ time: The timestamp of the message.
+ """
+
+ class MessageImportanceValueValuesEnum(_messages.Enum):
+ """Importance level of the message.
+
+ Values:
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN: The message importance isn't specified,
+ or is unknown.
+ JOB_MESSAGE_DEBUG: The message is at the 'debug' level: typically only
+ useful for software engineers working on the code the job is running.
+ Typically, Dataflow pipeline runners do not display log messages at
+ this level by default.
+ JOB_MESSAGE_DETAILED: The message is at the 'detailed' level: somewhat
+ verbose, but potentially useful to users. Typically, Dataflow
+ pipeline runners do not display log messages at this level by default.
+ These messages are displayed by default in the Dataflow monitoring UI.
+ JOB_MESSAGE_BASIC: The message is at the 'basic' level: useful for
+ keeping track of the execution of a Dataflow pipeline. Typically,
+ Dataflow pipeline runners display log messages at this level by
+ default, and these messages are displayed by default in the Dataflow
+ monitoring UI.
+ JOB_MESSAGE_WARNING: The message is at the 'warning' level: indicating a
+ condition pertaining to a job which may require human intervention.
+ Typically, Dataflow pipeline runners display log messages at this
+ level by default, and these messages are displayed by default in the
+ Dataflow monitoring UI.
+ JOB_MESSAGE_ERROR: The message is at the 'error' level: indicating a
+ condition preventing a job from succeeding. Typically, Dataflow
+ pipeline runners display log messages at this level by default, and
+ these messages are displayed by default in the Dataflow monitoring UI.
+ """
+ JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0
+ JOB_MESSAGE_DEBUG = 1
+ JOB_MESSAGE_DETAILED = 2
+ JOB_MESSAGE_BASIC = 3
+ JOB_MESSAGE_WARNING = 4
+ JOB_MESSAGE_ERROR = 5
+
+ id = _messages.StringField(1)
+ messageImportance = _messages.EnumField('MessageImportanceValueValuesEnum', 2)
+ messageText = _messages.StringField(3)
+ time = _messages.StringField(4)
+
+
+class JobMetrics(_messages.Message):
+ """JobMetrics contains a collection of metrics descibing the detailed
+ progress of a Dataflow job. Metrics correspond to user-defined and system-
+ defined metrics in the job. This resource captures only the most recent
+ values of each metric; time-series data can be queried for them (under the
+ same metric names) from Cloud Monitoring.
+
+ Fields:
+ metricTime: Timestamp as of which metric values are current.
+ metrics: All metrics for this job.
+ """
+
+ metricTime = _messages.StringField(1)
+ metrics = _messages.MessageField('MetricUpdate', 2, repeated=True)
+
+
+class KeyRangeDataDiskAssignment(_messages.Message):
+ """Data disk assignment information for a specific key-range of a sharded
+ computation. Currently we only support UTF-8 character splits to simplify
+ encoding into JSON.
+
+ Fields:
+ dataDisk: The name of the data disk where data for this range is stored.
+ This name is local to the Google Cloud Platform project and uniquely
+ identifies the disk within that project, for example
+ "myproject-1014-104817-4c2-harness-0-disk-1".
+ end: The end (exclusive) of the key range.
+ start: The start (inclusive) of the key range.
+ """
+
+ dataDisk = _messages.StringField(1)
+ end = _messages.StringField(2)
+ start = _messages.StringField(3)
+
+
+class KeyRangeLocation(_messages.Message):
+ """Location information for a specific key-range of a sharded computation.
+ Currently we only support UTF-8 character splits to simplify encoding into
+ JSON.
+
+ Fields:
+ dataDisk: The name of the data disk where data for this range is stored.
+ This name is local to the Google Cloud Platform project and uniquely
+ identifies the disk within that project, for example
+ "myproject-1014-104817-4c2-harness-0-disk-1".
+ deliveryEndpoint: The physical location of this range assignment to be
+ used for streaming computation cross-worker message delivery.
+ end: The end (exclusive) of the key range.
+ persistentDirectory: The location of the persistent state for this range,
+ as a persistent directory in the worker local filesystem.
+ start: The start (inclusive) of the key range.
+ """
+
+ dataDisk = _messages.StringField(1)
+ deliveryEndpoint = _messages.StringField(2)
+ end = _messages.StringField(3)
+ persistentDirectory = _messages.StringField(4)
+ start = _messages.StringField(5)
+
+
+class LeaseWorkItemRequest(_messages.Message):
+ """Request to lease WorkItems.
+
+ Fields:
+ currentWorkerTime: The current timestamp at the worker.
+ location: The location which contains the WorkItem's job.
+ requestedLeaseDuration: The initial lease period.
+ workItemTypes: Filter for WorkItem type.
+ workerCapabilities: Worker capabilities. WorkItems might be limited to
+ workers with specific capabilities.
+ workerId: Identifies the worker leasing work -- typically the ID of the
+ virtual machine running the worker.
+ """
+
+ currentWorkerTime = _messages.StringField(1)
+ location = _messages.StringField(2)
+ requestedLeaseDuration = _messages.StringField(3)
+ workItemTypes = _messages.StringField(4, repeated=True)
+ workerCapabilities = _messages.StringField(5, repeated=True)
+ workerId = _messages.StringField(6)
+
+
+class LeaseWorkItemResponse(_messages.Message):
+ """Response to a request to lease WorkItems.
+
+ Fields:
+ workItems: A list of the leased WorkItems.
+ """
+
+ workItems = _messages.MessageField('WorkItem', 1, repeated=True)
+
+
+class ListJobMessagesResponse(_messages.Message):
+ """Response to a request to list job messages.
+
+ Fields:
+ jobMessages: Messages in ascending timestamp order.
+ nextPageToken: The token to obtain the next page of results if there are
+ more.
+ """
+
+ jobMessages = _messages.MessageField('JobMessage', 1, repeated=True)
+ nextPageToken = _messages.StringField(2)
+
+
+class ListJobsResponse(_messages.Message):
+ """Response to a request to list Cloud Dataflow jobs. This may be a partial
+ response, depending on the page size in the ListJobsRequest.
+
+ Fields:
+ failedLocation: Zero or more messages describing locations that failed to
+ respond.
+ jobs: A subset of the requested job information.
+ nextPageToken: Set if there may be more results than fit in this response.
+ """
+
+ failedLocation = _messages.MessageField('FailedLocation', 1, repeated=True)
+ jobs = _messages.MessageField('Job', 2, repeated=True)
+ nextPageToken = _messages.StringField(3)
+
+
+class MapTask(_messages.Message):
+ """MapTask consists of an ordered set of instructions, each of which
+ describes one particular low-level operation for the worker to perform in
+ order to accomplish the MapTask's WorkItem. Each instruction must appear in
+ the list before any instructions which depends on its output.
+
+ Fields:
+ instructions: The instructions in the MapTask.
+ stageName: System-defined name of the stage containing this MapTask.
+ Unique across the workflow.
+ systemName: System-defined name of this MapTask. Unique across the
+ workflow.
+ """
+
+ instructions = _messages.MessageField('ParallelInstruction', 1, repeated=True)
+ stageName = _messages.StringField(2)
+ systemName = _messages.StringField(3)
+
+
+class MetricShortId(_messages.Message):
+ """The metric short id is returned to the user alongside an offset into
+ ReportWorkItemStatusRequest
+
+ Fields:
+ metricIndex: The index of the corresponding metric in the
+ ReportWorkItemStatusRequest. Required.
+ shortId: The service-generated short identifier for the metric.
+ """
+
+ metricIndex = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+ shortId = _messages.IntegerField(2)
+
+
+class MetricStructuredName(_messages.Message):
+ """Identifies a metric, by describing the source which generated the metric.
+
+ Messages:
+ ContextValue: Zero or more labeled fields which identify the part of the
+ job this metric is associated with, such as the name of a step or
+ collection. For example, built-in counters associated with steps will
+ have context['step'] = <step-name>. Counters associated with
+ PCollections in the SDK will have context['pcollection'] = <pcollection-
+ name>.
+
+ Fields:
+ context: Zero or more labeled fields which identify the part of the job
+ this metric is associated with, such as the name of a step or
+ collection. For example, built-in counters associated with steps will
+ have context['step'] = <step-name>. Counters associated with
+ PCollections in the SDK will have context['pcollection'] = <pcollection-
+ name>.
+ name: Worker-defined metric name.
+ origin: Origin (namespace) of metric name. May be blank for user-define
+ metrics; will be "dataflow" for metrics defined by the Dataflow service
+ or SDK.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class ContextValue(_messages.Message):
+ """Zero or more labeled fields which identify the part of the job this
+ metric is associated with, such as the name of a step or collection. For
+ example, built-in counters associated with steps will have context['step']
+ = <step-name>. Counters associated with PCollections in the SDK will have
+ context['pcollection'] = <pcollection-name>.
+
+ Messages:
+ AdditionalProperty: An additional property for a ContextValue object.
+
+ Fields:
+ additionalProperties: Additional properties of type ContextValue
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a ContextValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A string attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.StringField(2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ context = _messages.MessageField('ContextValue', 1)
+ name = _messages.StringField(2)
+ origin = _messages.StringField(3)
+
+
+class MetricUpdate(_messages.Message):
+ """Describes the state of a metric.
+
+ Fields:
+ cumulative: True if this metric is reported as the total cumulative
+ aggregate value accumulated since the worker started working on this
+ WorkItem. By default this is false, indicating that this metric is
+ reported as a delta that is not associated with any WorkItem.
+ internal: Worker-computed aggregate value for internal use by the Dataflow
+ service.
+ kind: Metric aggregation kind. The possible metric aggregation kinds are
+ "Sum", "Max", "Min", "Mean", "Set", "And", and "Or". The specified
+ aggregation kind is case-insensitive. If omitted, this is not an
+ aggregated value but instead a single metric sample value.
+ meanCount: Worker-computed aggregate value for the "Mean" aggregation
+ kind. This holds the count of the aggregated values and is used in
+ combination with mean_sum above to obtain the actual mean aggregate
+ value. The only possible value type is Long.
+ meanSum: Worker-computed aggregate value for the "Mean" aggregation kind.
+ This holds the sum of the aggregated values and is used in combination
+ with mean_count below to obtain the actual mean aggregate value. The
+ only possible value types are Long and Double.
+ name: Name of the metric.
+ scalar: Worker-computed aggregate value for aggregation kinds "Sum",
+ "Max", "Min", "And", and "Or". The possible value types are Long,
+ Double, and Boolean.
+ set: Worker-computed aggregate value for the "Set" aggregation kind. The
+ only possible value type is a list of Values whose type can be Long,
+ Double, or String, according to the metric's type. All Values in the
+ list must be of the same type.
+ updateTime: Timestamp associated with the metric value. Optional when
+ workers are reporting work progress; it will be filled in responses from
+ the metrics API.
+ """
+
+ cumulative = _messages.BooleanField(1)
+ internal = _messages.MessageField('extra_types.JsonValue', 2)
+ kind = _messages.StringField(3)
+ meanCount = _messages.MessageField('extra_types.JsonValue', 4)
+ meanSum = _messages.MessageField('extra_types.JsonValue', 5)
+ name = _messages.MessageField('MetricStructuredName', 6)
+ scalar = _messages.MessageField('extra_types.JsonValue', 7)
+ set = _messages.MessageField('extra_types.JsonValue', 8)
+ updateTime = _messages.StringField(9)
+
+
+class MountedDataDisk(_messages.Message):
+ """Describes mounted data disk.
+
+ Fields:
+ dataDisk: The name of the data disk. This name is local to the Google
+ Cloud Platform project and uniquely identifies the disk within that
+ project, for example "myproject-1014-104817-4c2-harness-0-disk-1".
+ """
+
+ dataDisk = _messages.StringField(1)
+
+
+class MultiOutputInfo(_messages.Message):
+ """Information about an output of a multi-output DoFn.
+
+ Fields:
+ tag: The id of the tag the user code will emit to this output by; this
+ should correspond to the tag of some SideInputInfo.
+ """
+
+ tag = _messages.StringField(1)
+
+
+class NameAndKind(_messages.Message):
+ """Basic metadata about a counter.
+
+ Enums:
+ KindValueValuesEnum: Counter aggregation kind.
+
+ Fields:
+ kind: Counter aggregation kind.
+ name: Name of the counter.
+ """
+
+ class KindValueValuesEnum(_messages.Enum):
+ """Counter aggregation kind.
+
+ Values:
+ INVALID: Counter aggregation kind was not set.
+ SUM: Aggregated value is the sum of all contributed values.
+ MAX: Aggregated value is the max of all contributed values.
+ MIN: Aggregated value is the min of all contributed values.
+ MEAN: Aggregated value is the mean of all contributed values.
+ OR: Aggregated value represents the logical 'or' of all contributed
+ values.
+ AND: Aggregated value represents the logical 'and' of all contributed
+ values.
+ SET: Aggregated value is a set of unique contributed values.
+ DISTRIBUTION: Aggregated value captures statistics about a distribution.
+ """
+ INVALID = 0
+ SUM = 1
+ MAX = 2
+ MIN = 3
+ MEAN = 4
+ OR = 5
+ AND = 6
+ SET = 7
+ DISTRIBUTION = 8
+
+ kind = _messages.EnumField('KindValueValuesEnum', 1)
+ name = _messages.StringField(2)
+
+
+class Package(_messages.Message):
+ """The packages that must be installed in order for a worker to run the
+ steps of the Cloud Dataflow job that will be assigned to its worker pool.
+ This is the mechanism by which the Cloud Dataflow SDK causes code to be
+ loaded onto the workers. For example, the Cloud Dataflow Java SDK might use
+ this to install jars containing the user's code and all of the various
+ dependencies (libraries, data files, etc.) required in order for that code
+ to run.
+
+ Fields:
+ location: The resource to read the package from. The supported resource
+ type is: Google Cloud Storage: storage.googleapis.com/{bucket}
+ bucket.storage.googleapis.com/
+ name: The name of the package.
+ """
+
+ location = _messages.StringField(1)
+ name = _messages.StringField(2)
+
+
+class ParDoInstruction(_messages.Message):
+ """An instruction that does a ParDo operation. Takes one main input and zero
+ or more side inputs, and produces zero or more outputs. Runs user code.
+
+ Messages:
+ UserFnValue: The user function to invoke.
+
+ Fields:
+ input: The input.
+ multiOutputInfos: Information about each of the outputs, if user_fn is a
+ MultiDoFn.
+ numOutputs: The number of outputs.
+ sideInputs: Zero or more side inputs.
+ userFn: The user function to invoke.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class UserFnValue(_messages.Message):
+ """The user function to invoke.
+
+ Messages:
+ AdditionalProperty: An additional property for a UserFnValue object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a UserFnValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ input = _messages.MessageField('InstructionInput', 1)
+ multiOutputInfos = _messages.MessageField('MultiOutputInfo', 2, repeated=True)
+ numOutputs = _messages.IntegerField(3, variant=_messages.Variant.INT32)
+ sideInputs = _messages.MessageField('SideInputInfo', 4, repeated=True)
+ userFn = _messages.MessageField('UserFnValue', 5)
+
+
+class ParallelInstruction(_messages.Message):
+ """Describes a particular operation comprising a MapTask.
+
+ Fields:
+ flatten: Additional information for Flatten instructions.
+ name: User-provided name of this operation.
+ originalName: System-defined name for the operation in the original
+ workflow graph.
+ outputs: Describes the outputs of the instruction.
+ parDo: Additional information for ParDo instructions.
+ partialGroupByKey: Additional information for PartialGroupByKey
+ instructions.
+ read: Additional information for Read instructions.
+ systemName: System-defined name of this operation. Unique across the
+ workflow.
+ write: Additional information for Write instructions.
+ """
+
+ flatten = _messages.MessageField('FlattenInstruction', 1)
+ name = _messages.StringField(2)
+ originalName = _messages.StringField(3)
+ outputs = _messages.MessageField('InstructionOutput', 4, repeated=True)
+ parDo = _messages.MessageField('ParDoInstruction', 5)
+ partialGroupByKey = _messages.MessageField('PartialGroupByKeyInstruction', 6)
+ read = _messages.MessageField('ReadInstruction', 7)
+ systemName = _messages.StringField(8)
+ write = _messages.MessageField('WriteInstruction', 9)
+
+
+class PartialGroupByKeyInstruction(_messages.Message):
+ """An instruction that does a partial group-by-key. One input and one
+ output.
+
+ Messages:
+ InputElementCodecValue: The codec to use for interpreting an element in
+ the input PTable.
+ ValueCombiningFnValue: The value combining function to invoke.
+
+ Fields:
+ input: Describes the input to the partial group-by-key instruction.
+ inputElementCodec: The codec to use for interpreting an element in the
+ input PTable.
+ originalCombineValuesInputStoreName: If this instruction includes a
+ combining function this is the name of the intermediate store between
+ the GBK and the CombineValues.
+ originalCombineValuesStepName: If this instruction includes a combining
+ function, this is the name of the CombineValues instruction lifted into
+ this instruction.
+ sideInputs: Zero or more side inputs.
+ valueCombiningFn: The value combining function to invoke.
+ """
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class InputElementCodecValue(_messages.Message):
+ """The codec to use for interpreting an element in the input PTable.
+
+ Messages:
+ AdditionalProperty: An additional property for a InputElementCodecValue
+ object.
+
+ Fields:
+ additionalProperties: Properties of the object.
+ """
+
+ class AdditionalProperty(_messages.Message):
+ """An additional property for a InputElementCodecValue object.
+
+ Fields:
+ key: Name of the additional property.
+ value: A extra_types.JsonValue attribute.
+ """
+
+ key = _messages.StringField(1)
+ value = _messages.MessageField('extra_types.JsonValue', 2)
+
+ additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+ @encoding.MapUnrecognizedFields('additionalProperties')
+ class ValueCombiningFnValue(_messages.Message):
+ """The value combining function to invoke.
<TRUNCATED>
[5/6] beam git commit: Moving dataflow runner related code to vendor
directory
Posted by al...@apache.org.
Moving dataflow runner related code to vendor directory
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4337c3ec
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4337c3ec
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4337c3ec
Branch: refs/heads/master
Commit: 4337c3ec46f2d513a6518bde6efc26e6c1b74cb9
Parents: 9df1da4
Author: Ahmet Altay <al...@google.com>
Authored: Thu Feb 16 15:42:49 2017 -0800
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Feb 16 23:09:02 2017 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/internal/apiclient.py | 726 ---
.../apache_beam/internal/apiclient_test.py | 87 -
.../internal/clients/dataflow/__init__.py | 27 -
.../clients/dataflow/dataflow_v1b3_client.py | 683 ---
.../clients/dataflow/dataflow_v1b3_messages.py | 4173 ------------------
.../clients/dataflow/message_matchers.py | 124 -
.../clients/dataflow/message_matchers_test.py | 69 -
.../google_cloud_dataflow/dataflow_runner.py | 8 +-
.../dataflow_runner_test.py | 2 +-
.../google_cloud_dataflow/internal/__init__.py | 16 +
.../google_cloud_dataflow/internal/apiclient.py | 726 +++
.../internal/apiclient_test.py | 87 +
.../internal/clients/__init__.py | 16 +
.../internal/clients/dataflow/__init__.py | 27 +
.../clients/dataflow/dataflow_v1b3_client.py | 684 +++
.../clients/dataflow/dataflow_v1b3_messages.py | 4173 ++++++++++++++++++
.../clients/dataflow/message_matchers.py | 124 +
.../clients/dataflow/message_matchers_test.py | 69 +
.../template_runner_test.py | 88 +
sdks/python/apache_beam/runners/runner_test.py | 20 +-
.../apache_beam/runners/template_runner_test.py | 88 -
21 files changed, 6024 insertions(+), 5993 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/internal/apiclient.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/apiclient.py b/sdks/python/apache_beam/internal/apiclient.py
deleted file mode 100644
index 71b670f..0000000
--- a/sdks/python/apache_beam/internal/apiclient.py
+++ /dev/null
@@ -1,726 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Dataflow client utility functions."""
-
-import codecs
-from datetime import datetime
-import getpass
-import json
-import logging
-import os
-import re
-from StringIO import StringIO
-import time
-
-from apache_beam import utils
-from apache_beam.internal.auth import get_service_credentials
-from apache_beam.internal.clients import dataflow
-from apache_beam.internal.clients import storage
-from apache_beam.internal.json_value import to_json_value
-from apache_beam.transforms import cy_combiners
-from apache_beam.transforms.display import DisplayData
-from apache_beam.utils import dependency
-from apache_beam.utils import retry
-from apache_beam.utils.dependency import get_required_container_version
-from apache_beam.utils.dependency import get_sdk_name_and_version
-from apache_beam.utils.names import PropertyNames
-from apache_beam.utils.pipeline_options import DebugOptions
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import StandardOptions
-from apache_beam.utils.pipeline_options import WorkerOptions
-
-from apitools.base.py import encoding
-from apitools.base.py import exceptions
-
-
-class Step(object):
- """Wrapper for a dataflow Step protobuf."""
-
- def __init__(self, step_kind, step_name, additional_properties=None):
- self.step_kind = step_kind
- self.step_name = step_name
- self.proto = dataflow.Step(kind=step_kind, name=step_name)
- self.proto.properties = {}
- self._additional_properties = []
-
- if additional_properties is not None:
- for (n, v, t) in additional_properties:
- self.add_property(n, v, t)
-
- def add_property(self, name, value, with_type=False):
- self._additional_properties.append((name, value, with_type))
- self.proto.properties.additionalProperties.append(
- dataflow.Step.PropertiesValue.AdditionalProperty(
- key=name, value=to_json_value(value, with_type=with_type)))
-
- def _get_outputs(self):
- """Returns a list of all output labels for a step."""
- outputs = []
- for p in self.proto.properties.additionalProperties:
- if p.key == PropertyNames.OUTPUT_INFO:
- for entry in p.value.array_value.entries:
- for entry_prop in entry.object_value.properties:
- if entry_prop.key == PropertyNames.OUTPUT_NAME:
- outputs.append(entry_prop.value.string_value)
- return outputs
-
- def __reduce__(self):
- """Reduce hook for pickling the Step class more easily."""
- return (Step, (self.step_kind, self.step_name, self._additional_properties))
-
- def get_output(self, tag=None):
- """Returns name if it is one of the outputs or first output if name is None.
-
- Args:
- tag: tag of the output as a string or None if we want to get the
- name of the first output.
-
- Returns:
- The name of the output associated with the tag or the first output
- if tag was None.
-
- Raises:
- ValueError: if the tag does not exist within outputs.
- """
- outputs = self._get_outputs()
- if tag is None:
- return outputs[0]
- else:
- name = '%s_%s' % (PropertyNames.OUT, tag)
- if name not in outputs:
- raise ValueError(
- 'Cannot find named output: %s in %s.' % (name, outputs))
- return name
-
-
-class Environment(object):
- """Wrapper for a dataflow Environment protobuf."""
-
- def __init__(self, packages, options, environment_version):
- self.standard_options = options.view_as(StandardOptions)
- self.google_cloud_options = options.view_as(GoogleCloudOptions)
- self.worker_options = options.view_as(WorkerOptions)
- self.debug_options = options.view_as(DebugOptions)
- self.proto = dataflow.Environment()
- self.proto.clusterManagerApiService = GoogleCloudOptions.COMPUTE_API_SERVICE
- self.proto.dataset = '{}/cloud_dataflow'.format(
- GoogleCloudOptions.BIGQUERY_API_SERVICE)
- self.proto.tempStoragePrefix = (
- self.google_cloud_options.temp_location.replace(
- 'gs:/',
- GoogleCloudOptions.STORAGE_API_SERVICE))
- # User agent information.
- self.proto.userAgent = dataflow.Environment.UserAgentValue()
- self.local = 'localhost' in self.google_cloud_options.dataflow_endpoint
-
- if self.google_cloud_options.service_account_email:
- self.proto.serviceAccountEmail = (
- self.google_cloud_options.service_account_email)
-
- sdk_name, version_string = get_sdk_name_and_version()
-
- self.proto.userAgent.additionalProperties.extend([
- dataflow.Environment.UserAgentValue.AdditionalProperty(
- key='name',
- value=to_json_value(sdk_name)),
- dataflow.Environment.UserAgentValue.AdditionalProperty(
- key='version', value=to_json_value(version_string))])
- # Version information.
- self.proto.version = dataflow.Environment.VersionValue()
- if self.standard_options.streaming:
- job_type = 'PYTHON_STREAMING'
- else:
- job_type = 'PYTHON_BATCH'
- self.proto.version.additionalProperties.extend([
- dataflow.Environment.VersionValue.AdditionalProperty(
- key='job_type',
- value=to_json_value(job_type)),
- dataflow.Environment.VersionValue.AdditionalProperty(
- key='major', value=to_json_value(environment_version))])
- # Experiments
- if self.debug_options.experiments:
- for experiment in self.debug_options.experiments:
- self.proto.experiments.append(experiment)
- # Worker pool(s) information.
- package_descriptors = []
- for package in packages:
- package_descriptors.append(
- dataflow.Package(
- location='%s/%s' % (
- self.google_cloud_options.staging_location.replace(
- 'gs:/', GoogleCloudOptions.STORAGE_API_SERVICE),
- package),
- name=package))
-
- pool = dataflow.WorkerPool(
- kind='local' if self.local else 'harness',
- packages=package_descriptors,
- taskrunnerSettings=dataflow.TaskRunnerSettings(
- parallelWorkerSettings=dataflow.WorkerSettings(
- baseUrl=GoogleCloudOptions.DATAFLOW_ENDPOINT,
- servicePath=self.google_cloud_options.dataflow_endpoint)))
- pool.autoscalingSettings = dataflow.AutoscalingSettings()
- # Set worker pool options received through command line.
- if self.worker_options.num_workers:
- pool.numWorkers = self.worker_options.num_workers
- if self.worker_options.max_num_workers:
- pool.autoscalingSettings.maxNumWorkers = (
- self.worker_options.max_num_workers)
- if self.worker_options.autoscaling_algorithm:
- values_enum = dataflow.AutoscalingSettings.AlgorithmValueValuesEnum
- pool.autoscalingSettings.algorithm = {
- 'NONE': values_enum.AUTOSCALING_ALGORITHM_NONE,
- 'THROUGHPUT_BASED': values_enum.AUTOSCALING_ALGORITHM_BASIC,
- }.get(self.worker_options.autoscaling_algorithm)
- if self.worker_options.machine_type:
- pool.machineType = self.worker_options.machine_type
- if self.worker_options.disk_size_gb:
- pool.diskSizeGb = self.worker_options.disk_size_gb
- if self.worker_options.disk_type:
- pool.diskType = self.worker_options.disk_type
- if self.worker_options.zone:
- pool.zone = self.worker_options.zone
- if self.worker_options.network:
- pool.network = self.worker_options.network
- if self.worker_options.worker_harness_container_image:
- pool.workerHarnessContainerImage = (
- self.worker_options.worker_harness_container_image)
- else:
- # Default to using the worker harness container image for the current SDK
- # version.
- pool.workerHarnessContainerImage = (
- 'dataflow.gcr.io/v1beta3/python:%s' %
- get_required_container_version())
- if self.worker_options.use_public_ips is not None:
- if self.worker_options.use_public_ips:
- pool.ipConfiguration = (
- dataflow.WorkerPool
- .IpConfigurationValueValuesEnum.WORKER_IP_PUBLIC)
- else:
- pool.ipConfiguration = (
- dataflow.WorkerPool
- .IpConfigurationValueValuesEnum.WORKER_IP_PRIVATE)
-
- if self.standard_options.streaming:
- # Use separate data disk for streaming.
- disk = dataflow.Disk()
- if self.local:
- disk.diskType = 'local'
- # TODO(ccy): allow customization of disk.
- pool.dataDisks.append(disk)
- self.proto.workerPools.append(pool)
-
- sdk_pipeline_options = options.get_all_options()
- if sdk_pipeline_options:
- self.proto.sdkPipelineOptions = (
- dataflow.Environment.SdkPipelineOptionsValue())
-
- options_dict = {k: v
- for k, v in sdk_pipeline_options.iteritems()
- if v is not None}
- self.proto.sdkPipelineOptions.additionalProperties.append(
- dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(
- key='options', value=to_json_value(options_dict)))
-
- dd = DisplayData.create_from_options(options)
- items = [item.get_dict() for item in dd.items]
- self.proto.sdkPipelineOptions.additionalProperties.append(
- dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(
- key='display_data', value=to_json_value(items)))
-
-
-class Job(object):
- """Wrapper for a dataflow Job protobuf."""
-
- def __str__(self):
- def encode_shortstrings(input_buffer, errors='strict'):
- """Encoder (from Unicode) that suppresses long base64 strings."""
- original_len = len(input_buffer)
- if original_len > 150:
- if self.base64_str_re.match(input_buffer):
- input_buffer = '<string of %d bytes>' % original_len
- input_buffer = input_buffer.encode('ascii', errors=errors)
- else:
- matched = self.coder_str_re.match(input_buffer)
- if matched:
- input_buffer = '%s<string of %d bytes>' % (
- matched.group(1), matched.end(2) - matched.start(2))
- input_buffer = input_buffer.encode('ascii', errors=errors)
- return input_buffer, original_len
-
- def decode_shortstrings(input_buffer, errors='strict'):
- """Decoder (to Unicode) that suppresses long base64 strings."""
- shortened, length = encode_shortstrings(input_buffer, errors)
- return unicode(shortened), length
-
- def shortstrings_registerer(encoding_name):
- if encoding_name == 'shortstrings':
- return codecs.CodecInfo(name='shortstrings',
- encode=encode_shortstrings,
- decode=decode_shortstrings)
- return None
-
- codecs.register(shortstrings_registerer)
-
- # Use json "dump string" method to get readable formatting;
- # further modify it to not output too-long strings, aimed at the
- # 10,000+ character hex-encoded "serialized_fn" values.
- return json.dumps(
- json.loads(encoding.MessageToJson(self.proto), encoding='shortstrings'),
- indent=2, sort_keys=True)
-
- @staticmethod
- def default_job_name(job_name):
- if job_name is None:
- user_name = getpass.getuser().lower()
- date_component = datetime.utcnow().strftime('%m%d%H%M%S-%f')
- app_name = 'beamapp'
- job_name = '{}-{}-{}'.format(app_name, user_name, date_component)
- return job_name
-
- def __init__(self, options):
- self.options = options
- self.google_cloud_options = options.view_as(GoogleCloudOptions)
- if not self.google_cloud_options.job_name:
- self.google_cloud_options.job_name = self.default_job_name(
- self.google_cloud_options.job_name)
-
- required_google_cloud_options = ['project', 'job_name', 'temp_location']
- missing = [
- option for option in required_google_cloud_options
- if not getattr(self.google_cloud_options, option)]
- if missing:
- raise ValueError(
- 'Missing required configuration parameters: %s' % missing)
-
- if not self.google_cloud_options.staging_location:
- logging.info('Defaulting to the temp_location as staging_location: %s',
- self.google_cloud_options.temp_location)
- (self.google_cloud_options
- .staging_location) = self.google_cloud_options.temp_location
-
- # Make the staging and temp locations job name and time specific. This is
- # needed to avoid clashes between job submissions using the same staging
- # area or team members using same job names. This method is not entirely
- # foolproof since two job submissions with same name can happen at exactly
- # the same time. However the window is extremely small given that
- # time.time() has at least microseconds granularity. We add the suffix only
- # for GCS staging locations where the potential for such clashes is high.
- if self.google_cloud_options.staging_location.startswith('gs://'):
- path_suffix = '%s.%f' % (self.google_cloud_options.job_name, time.time())
- self.google_cloud_options.staging_location = utils.path.join(
- self.google_cloud_options.staging_location, path_suffix)
- self.google_cloud_options.temp_location = utils.path.join(
- self.google_cloud_options.temp_location, path_suffix)
- self.proto = dataflow.Job(name=self.google_cloud_options.job_name)
- if self.options.view_as(StandardOptions).streaming:
- self.proto.type = dataflow.Job.TypeValueValuesEnum.JOB_TYPE_STREAMING
- else:
- self.proto.type = dataflow.Job.TypeValueValuesEnum.JOB_TYPE_BATCH
- self.base64_str_re = re.compile(r'^[A-Za-z0-9+/]*=*$')
- self.coder_str_re = re.compile(r'^([A-Za-z]+\$)([A-Za-z0-9+/]*=*)$')
-
- def json(self):
- return encoding.MessageToJson(self.proto)
-
- def __reduce__(self):
- """Reduce hook for pickling the Job class more easily."""
- return (Job, (self.options,))
-
-
-class DataflowApplicationClient(object):
- """A Dataflow API client used by application code to create and query jobs."""
-
- def __init__(self, options, environment_version):
- """Initializes a Dataflow API client object."""
- self.standard_options = options.view_as(StandardOptions)
- self.google_cloud_options = options.view_as(GoogleCloudOptions)
- self.environment_version = environment_version
- if self.google_cloud_options.no_auth:
- credentials = None
- else:
- credentials = get_service_credentials()
- self._client = dataflow.DataflowV1b3(
- url=self.google_cloud_options.dataflow_endpoint,
- credentials=credentials,
- get_credentials=(not self.google_cloud_options.no_auth))
- self._storage_client = storage.StorageV1(
- url='https://www.googleapis.com/storage/v1',
- credentials=credentials,
- get_credentials=(not self.google_cloud_options.no_auth))
-
- # TODO(silviuc): Refactor so that retry logic can be applied.
- @retry.no_retries # Using no_retries marks this as an integration point.
- def _gcs_file_copy(self, from_path, to_path):
- to_folder, to_name = os.path.split(to_path)
- with open(from_path, 'rb') as f:
- self.stage_file(to_folder, to_name, f)
-
- def stage_file(self, gcs_or_local_path, file_name, stream,
- mime_type='application/octet-stream'):
- """Stages a file at a GCS or local path with stream-supplied contents."""
- if not gcs_or_local_path.startswith('gs://'):
- local_path = os.path.join(gcs_or_local_path, file_name)
- logging.info('Staging file locally to %s', local_path)
- with open(local_path, 'wb') as f:
- f.write(stream.read())
- return
- gcs_location = gcs_or_local_path + '/' + file_name
- bucket, name = gcs_location[5:].split('/', 1)
-
- request = storage.StorageObjectsInsertRequest(
- bucket=bucket, name=name)
- logging.info('Starting GCS upload to %s...', gcs_location)
- upload = storage.Upload(stream, mime_type)
- try:
- response = self._storage_client.objects.Insert(request, upload=upload)
- except exceptions.HttpError as e:
- reportable_errors = {
- 403: 'access denied',
- 404: 'bucket not found',
- }
- if e.status_code in reportable_errors:
- raise IOError(('Could not upload to GCS path %s: %s. Please verify '
- 'that credentials are valid and that you have write '
- 'access to the specified path. Stale credentials can be '
- 'refreshed by executing "gcloud auth login".') %
- (gcs_or_local_path, reportable_errors[e.status_code]))
- raise
- logging.info('Completed GCS upload to %s', gcs_location)
- return response
-
- # TODO(silviuc): Refactor so that retry logic can be applied.
- @retry.no_retries # Using no_retries marks this as an integration point.
- def create_job(self, job):
- """Creates job description. May stage and/or submit for remote execution."""
- self.create_job_description(job)
-
- # Stage and submit the job when necessary
- dataflow_job_file = job.options.view_as(DebugOptions).dataflow_job_file
- template_location = (
- job.options.view_as(GoogleCloudOptions).template_location)
-
- job_location = template_location or dataflow_job_file
- if job_location:
- gcs_or_local_path = os.path.dirname(job_location)
- file_name = os.path.basename(job_location)
- self.stage_file(gcs_or_local_path, file_name, StringIO(job.json()))
-
- if not template_location:
- return self.submit_job_description(job)
- else:
- return None
-
- def create_job_description(self, job):
- """Creates a job described by the workflow proto."""
- resources = dependency.stage_job_resources(
- job.options, file_copy=self._gcs_file_copy)
- job.proto.environment = Environment(
- packages=resources, options=job.options,
- environment_version=self.environment_version).proto
- # TODO(silviuc): Remove the debug logging eventually.
- logging.info('JOB: %s', job)
-
- def submit_job_description(self, job):
- """Creates and excutes a job request."""
- request = dataflow.DataflowProjectsJobsCreateRequest()
- request.projectId = self.google_cloud_options.project
- request.job = job.proto
-
- try:
- response = self._client.projects_jobs.Create(request)
- except exceptions.BadStatusCodeError as e:
- logging.error('HTTP status %d trying to create job'
- ' at dataflow service endpoint %s',
- e.response.status,
- self.google_cloud_options.dataflow_endpoint)
- logging.fatal('details of server error: %s', e)
- raise
- logging.info('Create job: %s', response)
- # The response is a Job proto with the id for the new job.
- logging.info('Created job with id: [%s]', response.id)
- logging.info(
- 'To access the Dataflow monitoring console, please navigate to '
- 'https://console.developers.google.com/project/%s/dataflow/job/%s',
- self.google_cloud_options.project, response.id)
-
- return response
-
- @retry.with_exponential_backoff() # Using retry defaults from utils/retry.py
- def modify_job_state(self, job_id, new_state):
- """Modify the run state of the job.
-
- Args:
- job_id: The id of the job.
- new_state: A string representing the new desired state. It could be set to
- either 'JOB_STATE_DONE', 'JOB_STATE_CANCELLED' or 'JOB_STATE_DRAINING'.
-
- Returns:
- True if the job was modified successfully.
- """
- if new_state == 'JOB_STATE_DONE':
- new_state = dataflow.Job.RequestedStateValueValuesEnum.JOB_STATE_DONE
- elif new_state == 'JOB_STATE_CANCELLED':
- new_state = dataflow.Job.RequestedStateValueValuesEnum.JOB_STATE_CANCELLED
- elif new_state == 'JOB_STATE_DRAINING':
- new_state = dataflow.Job.RequestedStateValueValuesEnum.JOB_STATE_DRAINING
- else:
- # Other states could only be set by the service.
- return False
-
- request = dataflow.DataflowProjectsJobsUpdateRequest()
- request.jobId = job_id
- request.projectId = self.google_cloud_options.project
- request.job = dataflow.Job(requestedState=new_state)
-
- self._client.projects_jobs.Update(request)
- return True
-
- @retry.with_exponential_backoff() # Using retry defaults from utils/retry.py
- def get_job(self, job_id):
- """Gets the job status for a submitted job.
-
- Args:
- job_id: A string representing the job_id for the workflow as returned
- by the a create_job() request.
-
- Returns:
- A Job proto. See below for interesting fields.
-
- The Job proto returned from a get_job() request contains some interesting
- fields:
- currentState: An object representing the current state of the job. The
- string representation of the object (str() result) has the following
- possible values: JOB_STATE_UNKNONW, JOB_STATE_STOPPED,
- JOB_STATE_RUNNING, JOB_STATE_DONE, JOB_STATE_FAILED,
- JOB_STATE_CANCELLED.
- createTime: UTC time when the job was created
- (e.g. '2015-03-10T00:01:53.074Z')
- currentStateTime: UTC time for the current state of the job.
- """
- request = dataflow.DataflowProjectsJobsGetRequest()
- request.jobId = job_id
- request.projectId = self.google_cloud_options.project
- response = self._client.projects_jobs.Get(request)
- return response
-
- @retry.with_exponential_backoff() # Using retry defaults from utils/retry.py
- def list_messages(
- self, job_id, start_time=None, end_time=None, page_token=None,
- minimum_importance=None):
- """List messages associated with the execution of a job.
-
- Args:
- job_id: A string representing the job_id for the workflow as returned
- by the a create_job() request.
- start_time: If specified, only messages generated after the start time
- will be returned, otherwise all messages since job started will be
- returned. The value is a string representing UTC time
- (e.g., '2015-08-18T21:03:50.644Z')
- end_time: If specified, only messages generated before the end time
- will be returned, otherwise all messages up to current time will be
- returned. The value is a string representing UTC time
- (e.g., '2015-08-18T21:03:50.644Z')
- page_token: A string to be used as next page token if the list call
- returned paginated results.
- minimum_importance: Filter for messages based on importance. The possible
- string values in increasing order of importance are: JOB_MESSAGE_DEBUG,
- JOB_MESSAGE_DETAILED, JOB_MESSAGE_BASIC, JOB_MESSAGE_WARNING,
- JOB_MESSAGE_ERROR. For example, a filter set on warning will allow only
- warnings and errors and exclude all others.
-
- Returns:
- A tuple consisting of a list of JobMessage instances and a
- next page token string.
-
- Raises:
- RuntimeError: if an unexpected value for the message_importance argument
- is used.
-
- The JobMessage objects returned by the call contain the following fields:
- id: A unique string identifier for the message.
- time: A string representing the UTC time of the message
- (e.g., '2015-08-18T21:03:50.644Z')
- messageImportance: An enumeration value for the message importance. The
- value if converted to string will have the following possible values:
- JOB_MESSAGE_DEBUG, JOB_MESSAGE_DETAILED, JOB_MESSAGE_BASIC,
- JOB_MESSAGE_WARNING, JOB_MESSAGE_ERROR.
- messageText: A message string.
- """
- request = dataflow.DataflowProjectsJobsMessagesListRequest(
- jobId=job_id, projectId=self.google_cloud_options.project)
- if page_token is not None:
- request.pageToken = page_token
- if start_time is not None:
- request.startTime = start_time
- if end_time is not None:
- request.endTime = end_time
- if minimum_importance is not None:
- if minimum_importance == 'JOB_MESSAGE_DEBUG':
- request.minimumImportance = (
- dataflow.DataflowProjectsJobsMessagesListRequest
- .MinimumImportanceValueValuesEnum
- .JOB_MESSAGE_DEBUG)
- elif minimum_importance == 'JOB_MESSAGE_DETAILED':
- request.minimumImportance = (
- dataflow.DataflowProjectsJobsMessagesListRequest
- .MinimumImportanceValueValuesEnum
- .JOB_MESSAGE_DETAILED)
- elif minimum_importance == 'JOB_MESSAGE_BASIC':
- request.minimumImportance = (
- dataflow.DataflowProjectsJobsMessagesListRequest
- .MinimumImportanceValueValuesEnum
- .JOB_MESSAGE_BASIC)
- elif minimum_importance == 'JOB_MESSAGE_WARNING':
- request.minimumImportance = (
- dataflow.DataflowProjectsJobsMessagesListRequest
- .MinimumImportanceValueValuesEnum
- .JOB_MESSAGE_WARNING)
- elif minimum_importance == 'JOB_MESSAGE_ERROR':
- request.minimumImportance = (
- dataflow.DataflowProjectsJobsMessagesListRequest
- .MinimumImportanceValueValuesEnum
- .JOB_MESSAGE_ERROR)
- else:
- raise RuntimeError(
- 'Unexpected value for minimum_importance argument: %r',
- minimum_importance)
- response = self._client.projects_jobs_messages.List(request)
- return response.jobMessages, response.nextPageToken
-
-
-class MetricUpdateTranslators(object):
- """Translators between accumulators and dataflow metric updates."""
-
- @staticmethod
- def translate_boolean(accumulator, metric_update_proto):
- metric_update_proto.boolean = accumulator.value
-
- @staticmethod
- def translate_scalar_mean_int(accumulator, metric_update_proto):
- if accumulator.count:
- metric_update_proto.integerMean = dataflow.IntegerMean()
- metric_update_proto.integerMean.sum = to_split_int(accumulator.sum)
- metric_update_proto.integerMean.count = to_split_int(accumulator.count)
- else:
- metric_update_proto.nameAndKind.kind = None
-
- @staticmethod
- def translate_scalar_mean_float(accumulator, metric_update_proto):
- if accumulator.count:
- metric_update_proto.floatingPointMean = dataflow.FloatingPointMean()
- metric_update_proto.floatingPointMean.sum = accumulator.sum
- metric_update_proto.floatingPointMean.count = to_split_int(
- accumulator.count)
- else:
- metric_update_proto.nameAndKind.kind = None
-
- @staticmethod
- def translate_scalar_counter_int(accumulator, metric_update_proto):
- metric_update_proto.integer = to_split_int(accumulator.value)
-
- @staticmethod
- def translate_scalar_counter_float(accumulator, metric_update_proto):
- metric_update_proto.floatingPoint = accumulator.value
-
-
-def to_split_int(n):
- res = dataflow.SplitInt64()
- res.lowBits = n & 0xffffffff
- res.highBits = n >> 32
- return res
-
-
-def translate_distribution(distribution_update, metric_update_proto):
- """Translate metrics DistributionUpdate to dataflow distribution update."""
- dist_update_proto = dataflow.DistributionUpdate()
- dist_update_proto.min = to_split_int(distribution_update.min)
- dist_update_proto.max = to_split_int(distribution_update.max)
- dist_update_proto.count = to_split_int(distribution_update.count)
- dist_update_proto.sum = to_split_int(distribution_update.sum)
- metric_update_proto.distribution = dist_update_proto
-
-
-def translate_value(value, metric_update_proto):
- metric_update_proto.integer = to_split_int(value)
-
-
-def translate_scalar(accumulator, metric_update):
- metric_update.scalar = to_json_value(accumulator.value, with_type=True)
-
-
-def translate_mean(accumulator, metric_update):
- if accumulator.count:
- metric_update.meanSum = to_json_value(accumulator.sum, with_type=True)
- metric_update.meanCount = to_json_value(accumulator.count, with_type=True)
- else:
- # A denominator of 0 will raise an error in the service.
- # What it means is we have nothing to report yet, so don't.
- metric_update.kind = None
-
-
-# To enable a counter on the service, add it to this dictionary.
-metric_translations = {
- cy_combiners.CountCombineFn: ('sum', translate_scalar),
- cy_combiners.SumInt64Fn: ('sum', translate_scalar),
- cy_combiners.MinInt64Fn: ('min', translate_scalar),
- cy_combiners.MaxInt64Fn: ('max', translate_scalar),
- cy_combiners.MeanInt64Fn: ('mean', translate_mean),
- cy_combiners.SumFloatFn: ('sum', translate_scalar),
- cy_combiners.MinFloatFn: ('min', translate_scalar),
- cy_combiners.MaxFloatFn: ('max', translate_scalar),
- cy_combiners.MeanFloatFn: ('mean', translate_mean),
- cy_combiners.AllCombineFn: ('and', translate_scalar),
- cy_combiners.AnyCombineFn: ('or', translate_scalar),
-}
-
-counter_translations = {
- cy_combiners.CountCombineFn: (
- dataflow.NameAndKind.KindValueValuesEnum.SUM,
- MetricUpdateTranslators.translate_scalar_counter_int),
- cy_combiners.SumInt64Fn: (
- dataflow.NameAndKind.KindValueValuesEnum.SUM,
- MetricUpdateTranslators.translate_scalar_counter_int),
- cy_combiners.MinInt64Fn: (
- dataflow.NameAndKind.KindValueValuesEnum.MIN,
- MetricUpdateTranslators.translate_scalar_counter_int),
- cy_combiners.MaxInt64Fn: (
- dataflow.NameAndKind.KindValueValuesEnum.MAX,
- MetricUpdateTranslators.translate_scalar_counter_int),
- cy_combiners.MeanInt64Fn: (
- dataflow.NameAndKind.KindValueValuesEnum.MEAN,
- MetricUpdateTranslators.translate_scalar_mean_int),
- cy_combiners.SumFloatFn: (
- dataflow.NameAndKind.KindValueValuesEnum.SUM,
- MetricUpdateTranslators.translate_scalar_counter_float),
- cy_combiners.MinFloatFn: (
- dataflow.NameAndKind.KindValueValuesEnum.MIN,
- MetricUpdateTranslators.translate_scalar_counter_float),
- cy_combiners.MaxFloatFn: (
- dataflow.NameAndKind.KindValueValuesEnum.MAX,
- MetricUpdateTranslators.translate_scalar_counter_float),
- cy_combiners.MeanFloatFn: (
- dataflow.NameAndKind.KindValueValuesEnum.MEAN,
- MetricUpdateTranslators.translate_scalar_mean_float),
- cy_combiners.AllCombineFn: (
- dataflow.NameAndKind.KindValueValuesEnum.AND,
- MetricUpdateTranslators.translate_boolean),
- cy_combiners.AnyCombineFn: (
- dataflow.NameAndKind.KindValueValuesEnum.OR,
- MetricUpdateTranslators.translate_boolean),
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/internal/apiclient_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/apiclient_test.py b/sdks/python/apache_beam/internal/apiclient_test.py
deleted file mode 100644
index 86a771d..0000000
--- a/sdks/python/apache_beam/internal/apiclient_test.py
+++ /dev/null
@@ -1,87 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-"""Unit tests for the apiclient module."""
-
-import unittest
-
-from mock import Mock
-
-from apache_beam.internal import apiclient
-from apache_beam.internal.clients import dataflow
-from apache_beam.metrics.cells import DistributionData
-from apache_beam.runners.google_cloud_dataflow.dataflow_runner import DataflowRunner
-from apache_beam.utils.pipeline_options import PipelineOptions
-
-
-class UtilTest(unittest.TestCase):
-
- @unittest.skip("Enable once BEAM-1080 is fixed.")
- def test_create_application_client(self):
- pipeline_options = PipelineOptions()
- apiclient.DataflowApplicationClient(
- pipeline_options,
- DataflowRunner.BATCH_ENVIRONMENT_MAJOR_VERSION)
-
- def test_default_job_name(self):
- job_name = apiclient.Job.default_job_name(None)
- regexp = 'beamapp-.*-[0-9]{10}-[0-9]{6}'
- self.assertRegexpMatches(job_name, regexp)
-
- def test_split_int(self):
- number = 12345
- split_number = apiclient.to_split_int(number)
- self.assertEqual((split_number.lowBits, split_number.highBits),
- (number, 0))
- shift_number = number << 32
- split_number = apiclient.to_split_int(shift_number)
- self.assertEqual((split_number.lowBits, split_number.highBits),
- (0, number))
-
- def test_translate_distribution(self):
- metric_update = dataflow.CounterUpdate()
- distribution_update = DistributionData(16, 2, 1, 15)
- apiclient.translate_distribution(distribution_update, metric_update)
- self.assertEqual(metric_update.distribution.min.lowBits,
- distribution_update.min)
- self.assertEqual(metric_update.distribution.max.lowBits,
- distribution_update.max)
- self.assertEqual(metric_update.distribution.sum.lowBits,
- distribution_update.sum)
- self.assertEqual(metric_update.distribution.count.lowBits,
- distribution_update.count)
-
- def test_translate_means(self):
- metric_update = dataflow.CounterUpdate()
- accumulator = Mock()
- accumulator.sum = 16
- accumulator.count = 2
- apiclient.MetricUpdateTranslators.translate_scalar_mean_int(accumulator,
- metric_update)
- self.assertEqual(metric_update.integerMean.sum.lowBits, accumulator.sum)
- self.assertEqual(metric_update.integerMean.count.lowBits, accumulator.count)
-
- accumulator.sum = 16.0
- accumulator.count = 2
- apiclient.MetricUpdateTranslators.translate_scalar_mean_float(accumulator,
- metric_update)
- self.assertEqual(metric_update.floatingPointMean.sum, accumulator.sum)
- self.assertEqual(
- metric_update.floatingPointMean.count.lowBits, accumulator.count)
-
-
-if __name__ == '__main__':
- unittest.main()
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/internal/clients/dataflow/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/clients/dataflow/__init__.py b/sdks/python/apache_beam/internal/clients/dataflow/__init__.py
deleted file mode 100644
index 605aa17..0000000
--- a/sdks/python/apache_beam/internal/clients/dataflow/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Common imports for generated dataflow client library."""
-# pylint:disable=wildcard-import
-
-import pkgutil
-
-from apitools.base.py import *
-from apache_beam.internal.clients.dataflow.dataflow_v1b3_client import *
-from apache_beam.internal.clients.dataflow.dataflow_v1b3_messages import *
-
-__path__ = pkgutil.extend_path(__path__, __name__)
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_client.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_client.py b/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_client.py
deleted file mode 100644
index 6ae2b73..0000000
--- a/sdks/python/apache_beam/internal/clients/dataflow/dataflow_v1b3_client.py
+++ /dev/null
@@ -1,683 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Generated client library for dataflow version v1b3."""
-# NOTE: This file is autogenerated and should not be edited by hand.
-from apitools.base.py import base_api
-from apache_beam.internal.clients.dataflow import dataflow_v1b3_messages as messages
-
-
-class DataflowV1b3(base_api.BaseApiClient):
- """Generated client library for service dataflow version v1b3."""
-
- MESSAGES_MODULE = messages
- BASE_URL = u'https://dataflow.googleapis.com/'
-
- _PACKAGE = u'dataflow'
- _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/userinfo.email']
- _VERSION = u'v1b3'
- _CLIENT_ID = '1042881264118.apps.googleusercontent.com'
- _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b'
- _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b'
- _CLIENT_CLASS_NAME = u'DataflowV1b3'
- _URL_VERSION = u'v1b3'
- _API_KEY = None
-
- def __init__(self, url='', credentials=None,
- get_credentials=True, http=None, model=None,
- log_request=False, log_response=False,
- credentials_args=None, default_global_params=None,
- additional_http_headers=None):
- """Create a new dataflow handle."""
- url = url or self.BASE_URL
- super(DataflowV1b3, self).__init__(
- url, credentials=credentials,
- get_credentials=get_credentials, http=http, model=model,
- log_request=log_request, log_response=log_response,
- credentials_args=credentials_args,
- default_global_params=default_global_params,
- additional_http_headers=additional_http_headers)
- self.projects_jobs_debug = self.ProjectsJobsDebugService(self)
- self.projects_jobs_messages = self.ProjectsJobsMessagesService(self)
- self.projects_jobs_workItems = self.ProjectsJobsWorkItemsService(self)
- self.projects_jobs = self.ProjectsJobsService(self)
- self.projects_locations_jobs_messages = self.ProjectsLocationsJobsMessagesService(self)
- self.projects_locations_jobs_workItems = self.ProjectsLocationsJobsWorkItemsService(self)
- self.projects_locations_jobs = self.ProjectsLocationsJobsService(self)
- self.projects_locations = self.ProjectsLocationsService(self)
- self.projects_templates = self.ProjectsTemplatesService(self)
- self.projects = self.ProjectsService(self)
-
- class ProjectsJobsDebugService(base_api.BaseApiService):
- """Service class for the projects_jobs_debug resource."""
-
- _NAME = u'projects_jobs_debug'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsJobsDebugService, self).__init__(client)
- self._upload_configs = {
- }
-
- def GetConfig(self, request, global_params=None):
- """Get encoded debug configuration for component. Not cacheable.
-
- Args:
- request: (DataflowProjectsJobsDebugGetConfigRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (GetDebugConfigResponse) The response message.
- """
- config = self.GetMethodConfig('GetConfig')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- GetConfig.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.jobs.debug.getConfig',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/debug/getConfig',
- request_field=u'getDebugConfigRequest',
- request_type_name=u'DataflowProjectsJobsDebugGetConfigRequest',
- response_type_name=u'GetDebugConfigResponse',
- supports_download=False,
- )
-
- def SendCapture(self, request, global_params=None):
- """Send encoded debug capture data for component.
-
- Args:
- request: (DataflowProjectsJobsDebugSendCaptureRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (SendDebugCaptureResponse) The response message.
- """
- config = self.GetMethodConfig('SendCapture')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- SendCapture.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.jobs.debug.sendCapture',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/debug/sendCapture',
- request_field=u'sendDebugCaptureRequest',
- request_type_name=u'DataflowProjectsJobsDebugSendCaptureRequest',
- response_type_name=u'SendDebugCaptureResponse',
- supports_download=False,
- )
-
- class ProjectsJobsMessagesService(base_api.BaseApiService):
- """Service class for the projects_jobs_messages resource."""
-
- _NAME = u'projects_jobs_messages'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsJobsMessagesService, self).__init__(client)
- self._upload_configs = {
- }
-
- def List(self, request, global_params=None):
- """Request the job status.
-
- Args:
- request: (DataflowProjectsJobsMessagesListRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (ListJobMessagesResponse) The response message.
- """
- config = self.GetMethodConfig('List')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- List.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.jobs.messages.list',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[u'endTime', u'location', u'minimumImportance', u'pageSize', u'pageToken', u'startTime'],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/messages',
- request_field='',
- request_type_name=u'DataflowProjectsJobsMessagesListRequest',
- response_type_name=u'ListJobMessagesResponse',
- supports_download=False,
- )
-
- class ProjectsJobsWorkItemsService(base_api.BaseApiService):
- """Service class for the projects_jobs_workItems resource."""
-
- _NAME = u'projects_jobs_workItems'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsJobsWorkItemsService, self).__init__(client)
- self._upload_configs = {
- }
-
- def Lease(self, request, global_params=None):
- """Leases a dataflow WorkItem to run.
-
- Args:
- request: (DataflowProjectsJobsWorkItemsLeaseRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (LeaseWorkItemResponse) The response message.
- """
- config = self.GetMethodConfig('Lease')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Lease.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.jobs.workItems.lease',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/workItems:lease',
- request_field=u'leaseWorkItemRequest',
- request_type_name=u'DataflowProjectsJobsWorkItemsLeaseRequest',
- response_type_name=u'LeaseWorkItemResponse',
- supports_download=False,
- )
-
- def ReportStatus(self, request, global_params=None):
- """Reports the status of dataflow WorkItems leased by a worker.
-
- Args:
- request: (DataflowProjectsJobsWorkItemsReportStatusRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (ReportWorkItemStatusResponse) The response message.
- """
- config = self.GetMethodConfig('ReportStatus')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- ReportStatus.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.jobs.workItems.reportStatus',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/workItems:reportStatus',
- request_field=u'reportWorkItemStatusRequest',
- request_type_name=u'DataflowProjectsJobsWorkItemsReportStatusRequest',
- response_type_name=u'ReportWorkItemStatusResponse',
- supports_download=False,
- )
-
- class ProjectsJobsService(base_api.BaseApiService):
- """Service class for the projects_jobs resource."""
-
- _NAME = u'projects_jobs'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsJobsService, self).__init__(client)
- self._upload_configs = {
- }
-
- def Create(self, request, global_params=None):
- """Creates a Cloud Dataflow job.
-
- Args:
- request: (DataflowProjectsJobsCreateRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (Job) The response message.
- """
- config = self.GetMethodConfig('Create')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Create.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.jobs.create',
- ordered_params=[u'projectId'],
- path_params=[u'projectId'],
- query_params=[u'location', u'replaceJobId', u'view'],
- relative_path=u'v1b3/projects/{projectId}/jobs',
- request_field=u'job',
- request_type_name=u'DataflowProjectsJobsCreateRequest',
- response_type_name=u'Job',
- supports_download=False,
- )
-
- def Get(self, request, global_params=None):
- """Gets the state of the specified Cloud Dataflow job.
-
- Args:
- request: (DataflowProjectsJobsGetRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (Job) The response message.
- """
- config = self.GetMethodConfig('Get')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Get.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.jobs.get',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[u'location', u'view'],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}',
- request_field='',
- request_type_name=u'DataflowProjectsJobsGetRequest',
- response_type_name=u'Job',
- supports_download=False,
- )
-
- def GetMetrics(self, request, global_params=None):
- """Request the job status.
-
- Args:
- request: (DataflowProjectsJobsGetMetricsRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (JobMetrics) The response message.
- """
- config = self.GetMethodConfig('GetMetrics')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- GetMetrics.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.jobs.getMetrics',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[u'location', u'startTime'],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/metrics',
- request_field='',
- request_type_name=u'DataflowProjectsJobsGetMetricsRequest',
- response_type_name=u'JobMetrics',
- supports_download=False,
- )
-
- def List(self, request, global_params=None):
- """List the jobs of a project.
-
- Args:
- request: (DataflowProjectsJobsListRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (ListJobsResponse) The response message.
- """
- config = self.GetMethodConfig('List')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- List.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.jobs.list',
- ordered_params=[u'projectId'],
- path_params=[u'projectId'],
- query_params=[u'filter', u'location', u'pageSize', u'pageToken', u'view'],
- relative_path=u'v1b3/projects/{projectId}/jobs',
- request_field='',
- request_type_name=u'DataflowProjectsJobsListRequest',
- response_type_name=u'ListJobsResponse',
- supports_download=False,
- )
-
- def Update(self, request, global_params=None):
- """Updates the state of an existing Cloud Dataflow job.
-
- Args:
- request: (DataflowProjectsJobsUpdateRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (Job) The response message.
- """
- config = self.GetMethodConfig('Update')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Update.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'PUT',
- method_id=u'dataflow.projects.jobs.update',
- ordered_params=[u'projectId', u'jobId'],
- path_params=[u'jobId', u'projectId'],
- query_params=[u'location'],
- relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}',
- request_field=u'job',
- request_type_name=u'DataflowProjectsJobsUpdateRequest',
- response_type_name=u'Job',
- supports_download=False,
- )
-
- class ProjectsLocationsJobsMessagesService(base_api.BaseApiService):
- """Service class for the projects_locations_jobs_messages resource."""
-
- _NAME = u'projects_locations_jobs_messages'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsLocationsJobsMessagesService, self).__init__(client)
- self._upload_configs = {
- }
-
- def List(self, request, global_params=None):
- """Request the job status.
-
- Args:
- request: (DataflowProjectsLocationsJobsMessagesListRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (ListJobMessagesResponse) The response message.
- """
- config = self.GetMethodConfig('List')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- List.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.locations.jobs.messages.list',
- ordered_params=[u'projectId', u'location', u'jobId'],
- path_params=[u'jobId', u'location', u'projectId'],
- query_params=[u'endTime', u'minimumImportance', u'pageSize', u'pageToken', u'startTime'],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/messages',
- request_field='',
- request_type_name=u'DataflowProjectsLocationsJobsMessagesListRequest',
- response_type_name=u'ListJobMessagesResponse',
- supports_download=False,
- )
-
- class ProjectsLocationsJobsWorkItemsService(base_api.BaseApiService):
- """Service class for the projects_locations_jobs_workItems resource."""
-
- _NAME = u'projects_locations_jobs_workItems'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsLocationsJobsWorkItemsService, self).__init__(client)
- self._upload_configs = {
- }
-
- def Lease(self, request, global_params=None):
- """Leases a dataflow WorkItem to run.
-
- Args:
- request: (DataflowProjectsLocationsJobsWorkItemsLeaseRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (LeaseWorkItemResponse) The response message.
- """
- config = self.GetMethodConfig('Lease')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Lease.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.locations.jobs.workItems.lease',
- ordered_params=[u'projectId', u'location', u'jobId'],
- path_params=[u'jobId', u'location', u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/workItems:lease',
- request_field=u'leaseWorkItemRequest',
- request_type_name=u'DataflowProjectsLocationsJobsWorkItemsLeaseRequest',
- response_type_name=u'LeaseWorkItemResponse',
- supports_download=False,
- )
-
- def ReportStatus(self, request, global_params=None):
- """Reports the status of dataflow WorkItems leased by a worker.
-
- Args:
- request: (DataflowProjectsLocationsJobsWorkItemsReportStatusRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (ReportWorkItemStatusResponse) The response message.
- """
- config = self.GetMethodConfig('ReportStatus')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- ReportStatus.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.locations.jobs.workItems.reportStatus',
- ordered_params=[u'projectId', u'location', u'jobId'],
- path_params=[u'jobId', u'location', u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/workItems:reportStatus',
- request_field=u'reportWorkItemStatusRequest',
- request_type_name=u'DataflowProjectsLocationsJobsWorkItemsReportStatusRequest',
- response_type_name=u'ReportWorkItemStatusResponse',
- supports_download=False,
- )
-
- class ProjectsLocationsJobsService(base_api.BaseApiService):
- """Service class for the projects_locations_jobs resource."""
-
- _NAME = u'projects_locations_jobs'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsLocationsJobsService, self).__init__(client)
- self._upload_configs = {
- }
-
- def Create(self, request, global_params=None):
- """Creates a Cloud Dataflow job.
-
- Args:
- request: (DataflowProjectsLocationsJobsCreateRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (Job) The response message.
- """
- config = self.GetMethodConfig('Create')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Create.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.locations.jobs.create',
- ordered_params=[u'projectId', u'location'],
- path_params=[u'location', u'projectId'],
- query_params=[u'replaceJobId', u'view'],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs',
- request_field=u'job',
- request_type_name=u'DataflowProjectsLocationsJobsCreateRequest',
- response_type_name=u'Job',
- supports_download=False,
- )
-
- def Get(self, request, global_params=None):
- """Gets the state of the specified Cloud Dataflow job.
-
- Args:
- request: (DataflowProjectsLocationsJobsGetRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (Job) The response message.
- """
- config = self.GetMethodConfig('Get')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Get.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.locations.jobs.get',
- ordered_params=[u'projectId', u'location', u'jobId'],
- path_params=[u'jobId', u'location', u'projectId'],
- query_params=[u'view'],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}',
- request_field='',
- request_type_name=u'DataflowProjectsLocationsJobsGetRequest',
- response_type_name=u'Job',
- supports_download=False,
- )
-
- def GetMetrics(self, request, global_params=None):
- """Request the job status.
-
- Args:
- request: (DataflowProjectsLocationsJobsGetMetricsRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (JobMetrics) The response message.
- """
- config = self.GetMethodConfig('GetMetrics')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- GetMetrics.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.locations.jobs.getMetrics',
- ordered_params=[u'projectId', u'location', u'jobId'],
- path_params=[u'jobId', u'location', u'projectId'],
- query_params=[u'startTime'],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/metrics',
- request_field='',
- request_type_name=u'DataflowProjectsLocationsJobsGetMetricsRequest',
- response_type_name=u'JobMetrics',
- supports_download=False,
- )
-
- def List(self, request, global_params=None):
- """List the jobs of a project.
-
- Args:
- request: (DataflowProjectsLocationsJobsListRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (ListJobsResponse) The response message.
- """
- config = self.GetMethodConfig('List')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- List.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'GET',
- method_id=u'dataflow.projects.locations.jobs.list',
- ordered_params=[u'projectId', u'location'],
- path_params=[u'location', u'projectId'],
- query_params=[u'filter', u'pageSize', u'pageToken', u'view'],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs',
- request_field='',
- request_type_name=u'DataflowProjectsLocationsJobsListRequest',
- response_type_name=u'ListJobsResponse',
- supports_download=False,
- )
-
- def Update(self, request, global_params=None):
- """Updates the state of an existing Cloud Dataflow job.
-
- Args:
- request: (DataflowProjectsLocationsJobsUpdateRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (Job) The response message.
- """
- config = self.GetMethodConfig('Update')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Update.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'PUT',
- method_id=u'dataflow.projects.locations.jobs.update',
- ordered_params=[u'projectId', u'location', u'jobId'],
- path_params=[u'jobId', u'location', u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}',
- request_field=u'job',
- request_type_name=u'DataflowProjectsLocationsJobsUpdateRequest',
- response_type_name=u'Job',
- supports_download=False,
- )
-
- class ProjectsLocationsService(base_api.BaseApiService):
- """Service class for the projects_locations resource."""
-
- _NAME = u'projects_locations'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsLocationsService, self).__init__(client)
- self._upload_configs = {
- }
-
- class ProjectsTemplatesService(base_api.BaseApiService):
- """Service class for the projects_templates resource."""
-
- _NAME = u'projects_templates'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsTemplatesService, self).__init__(client)
- self._upload_configs = {
- }
-
- def Create(self, request, global_params=None):
- """Creates a Cloud Dataflow job from a template.
-
- Args:
- request: (DataflowProjectsTemplatesCreateRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (Job) The response message.
- """
- config = self.GetMethodConfig('Create')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- Create.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.templates.create',
- ordered_params=[u'projectId'],
- path_params=[u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/templates',
- request_field=u'createJobFromTemplateRequest',
- request_type_name=u'DataflowProjectsTemplatesCreateRequest',
- response_type_name=u'Job',
- supports_download=False,
- )
-
- class ProjectsService(base_api.BaseApiService):
- """Service class for the projects resource."""
-
- _NAME = u'projects'
-
- def __init__(self, client):
- super(DataflowV1b3.ProjectsService, self).__init__(client)
- self._upload_configs = {
- }
-
- def WorkerMessages(self, request, global_params=None):
- """Send a worker_message to the service.
-
- Args:
- request: (DataflowProjectsWorkerMessagesRequest) input message
- global_params: (StandardQueryParameters, default: None) global arguments
- Returns:
- (SendWorkerMessagesResponse) The response message.
- """
- config = self.GetMethodConfig('WorkerMessages')
- return self._RunMethod(
- config, request, global_params=global_params)
-
- WorkerMessages.method_config = lambda: base_api.ApiMethodInfo(
- http_method=u'POST',
- method_id=u'dataflow.projects.workerMessages',
- ordered_params=[u'projectId'],
- path_params=[u'projectId'],
- query_params=[],
- relative_path=u'v1b3/projects/{projectId}/WorkerMessages',
- request_field=u'sendWorkerMessagesRequest',
- request_type_name=u'DataflowProjectsWorkerMessagesRequest',
- response_type_name=u'SendWorkerMessagesResponse',
- supports_download=False,
- )
[6/6] beam git commit: This closes #2026
Posted by al...@apache.org.
This closes #2026
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/36ed3f6c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/36ed3f6c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/36ed3f6c
Branch: refs/heads/master
Commit: 36ed3f6c54d35ac48241789fd77e2657d54c5831
Parents: 9df1da4 4337c3e
Author: Ahmet Altay <al...@google.com>
Authored: Thu Feb 16 23:09:06 2017 -0800
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Feb 16 23:09:06 2017 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/internal/apiclient.py | 726 ---
.../apache_beam/internal/apiclient_test.py | 87 -
.../internal/clients/dataflow/__init__.py | 27 -
.../clients/dataflow/dataflow_v1b3_client.py | 683 ---
.../clients/dataflow/dataflow_v1b3_messages.py | 4173 ------------------
.../clients/dataflow/message_matchers.py | 124 -
.../clients/dataflow/message_matchers_test.py | 69 -
.../google_cloud_dataflow/dataflow_runner.py | 8 +-
.../dataflow_runner_test.py | 2 +-
.../google_cloud_dataflow/internal/__init__.py | 16 +
.../google_cloud_dataflow/internal/apiclient.py | 726 +++
.../internal/apiclient_test.py | 87 +
.../internal/clients/__init__.py | 16 +
.../internal/clients/dataflow/__init__.py | 27 +
.../clients/dataflow/dataflow_v1b3_client.py | 684 +++
.../clients/dataflow/dataflow_v1b3_messages.py | 4173 ++++++++++++++++++
.../clients/dataflow/message_matchers.py | 124 +
.../clients/dataflow/message_matchers_test.py | 69 +
.../template_runner_test.py | 88 +
sdks/python/apache_beam/runners/runner_test.py | 20 +-
.../apache_beam/runners/template_runner_test.py | 88 -
21 files changed, 6024 insertions(+), 5993 deletions(-)
----------------------------------------------------------------------
[3/6] beam git commit: Moving dataflow runner related code to vendor
directory
Posted by al...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/internal/clients/dataflow/message_matchers.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/clients/dataflow/message_matchers.py b/sdks/python/apache_beam/internal/clients/dataflow/message_matchers.py
deleted file mode 100644
index 4dda47a..0000000
--- a/sdks/python/apache_beam/internal/clients/dataflow/message_matchers.py
+++ /dev/null
@@ -1,124 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from hamcrest.core.base_matcher import BaseMatcher
-
-
-IGNORED = object()
-
-
-class MetricStructuredNameMatcher(BaseMatcher):
- """Matches a MetricStructuredName."""
- def __init__(self,
- name=IGNORED,
- origin=IGNORED,
- context=IGNORED):
- """Creates a MetricsStructuredNameMatcher.
-
- Any property not passed in to the constructor will be ignored when matching.
-
- Args:
- name: A string with the metric name.
- origin: A string with the metric namespace.
- context: A key:value dictionary that will be matched to the
- structured name.
- """
- if context != IGNORED and not isinstance(context, dict):
- raise ValueError('context must be a Python dictionary.')
-
- self.name = name
- self.origin = origin
- self.context = context
-
- def _matches(self, item):
- if self.name != IGNORED and item.name != self.name:
- return False
- if self.origin != IGNORED and item.origin != self.origin:
- return False
- if self.context != IGNORED:
- for key, name in self.context.iteritems():
- if key not in item.context:
- return False
- if name != IGNORED and item.context[key] != name:
- return False
- return True
-
- def describe_to(self, description):
- descriptors = []
- if self.name != IGNORED:
- descriptors.append('name is {}'.format(self.name))
- if self.origin != IGNORED:
- descriptors.append('origin is {}'.format(self.origin))
- if self.context != IGNORED:
- descriptors.append('context is ({})'.format(str(self.context)))
-
- item_description = ' and '.join(descriptors)
- description.append(item_description)
-
-
-class MetricUpdateMatcher(BaseMatcher):
- """Matches a metrics update protocol buffer."""
- def __init__(self,
- cumulative=IGNORED,
- name=IGNORED,
- scalar=IGNORED,
- kind=IGNORED):
- """Creates a MetricUpdateMatcher.
-
- Any property not passed in to the constructor will be ignored when matching.
-
- Args:
- cumulative: A boolean.
- name: A MetricStructuredNameMatcher object that matches the name.
- scalar: An integer with the metric update.
- kind: A string defining the kind of counter.
- """
- if name != IGNORED and not isinstance(name, MetricStructuredNameMatcher):
- raise ValueError('name must be a MetricStructuredNameMatcher.')
-
- self.cumulative = cumulative
- self.name = name
- self.scalar = scalar
- self.kind = kind
-
- def _matches(self, item):
- if self.cumulative != IGNORED and item.cumulative != self.cumulative:
- return False
- if self.name != IGNORED and not self.name._matches(item.name):
- return False
- if self.kind != IGNORED and item.kind != self.kind:
- return False
- if self.scalar != IGNORED:
- value_property = [p
- for p in item.scalar.object_value.properties
- if p.key == 'value']
- int_value = value_property[0].value.integer_value
- if self.scalar != int_value:
- return False
- return True
-
- def describe_to(self, description):
- descriptors = []
- if self.cumulative != IGNORED:
- descriptors.append('cumulative is {}'.format(self.cumulative))
- if self.name != IGNORED:
- descriptors.append('name is {}'.format(self.name))
- if self.scalar != IGNORED:
- descriptors.append('scalar is ({})'.format(str(self.scalar)))
-
- item_description = ' and '.join(descriptors)
- description.append(item_description)
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/internal/clients/dataflow/message_matchers_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/clients/dataflow/message_matchers_test.py b/sdks/python/apache_beam/internal/clients/dataflow/message_matchers_test.py
deleted file mode 100644
index ec63ce7..0000000
--- a/sdks/python/apache_beam/internal/clients/dataflow/message_matchers_test.py
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import unittest
-
-import hamcrest as hc
-
-import apache_beam.internal.clients.dataflow as dataflow
-from apache_beam.internal.clients.dataflow import message_matchers
-from apache_beam.internal.json_value import to_json_value
-
-
-class TestMatchers(unittest.TestCase):
-
- def test_structured_name_matcher_basic(self):
- metric_name = dataflow.MetricStructuredName()
- metric_name.name = 'metric1'
- metric_name.origin = 'origin2'
-
- matcher = message_matchers.MetricStructuredNameMatcher(
- name='metric1',
- origin='origin2')
- hc.assert_that(metric_name, hc.is_(matcher))
- with self.assertRaises(AssertionError):
- matcher = message_matchers.MetricStructuredNameMatcher(
- name='metric1',
- origin='origin1')
- hc.assert_that(metric_name, hc.is_(matcher))
-
- def test_metric_update_basic(self):
- metric_update = dataflow.MetricUpdate()
- metric_update.name = dataflow.MetricStructuredName()
- metric_update.name.name = 'metric1'
- metric_update.name.origin = 'origin1'
-
- metric_update.cumulative = False
- metric_update.kind = 'sum'
- metric_update.scalar = to_json_value(1, with_type=True)
-
- name_matcher = message_matchers.MetricStructuredNameMatcher(
- name='metric1',
- origin='origin1')
- matcher = message_matchers.MetricUpdateMatcher(
- name=name_matcher,
- kind='sum',
- scalar=1)
-
- hc.assert_that(metric_update, hc.is_(matcher))
-
- with self.assertRaises(AssertionError):
- matcher.kind = 'suma'
- hc.assert_that(metric_update, hc.is_(matcher))
-
-
-if __name__ == '__main__':
- unittest.main()
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner.py
index 1507d88..760935d 100644
--- a/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner.py
@@ -33,17 +33,17 @@ from apache_beam.internal import json_value
from apache_beam.internal import pickler
from apache_beam.pvalue import PCollectionView
from apache_beam.runners.google_cloud_dataflow.dataflow_metrics import DataflowMetrics
+from apache_beam.runners.google_cloud_dataflow.internal.clients import dataflow as dataflow_api
+from apache_beam.runners.runner import PValueCache
from apache_beam.runners.runner import PipelineResult
from apache_beam.runners.runner import PipelineRunner
from apache_beam.runners.runner import PipelineState
-from apache_beam.runners.runner import PValueCache
from apache_beam.transforms.display import DisplayData
from apache_beam.typehints import typehints
from apache_beam.utils import names
from apache_beam.utils.names import PropertyNames
from apache_beam.utils.names import TransformNames
from apache_beam.utils.pipeline_options import StandardOptions
-from apache_beam.internal.clients import dataflow as dataflow_api
class DataflowRunner(PipelineRunner):
@@ -154,7 +154,7 @@ class DataflowRunner(PipelineRunner):
"""Remotely executes entire pipeline or parts reachable from node."""
# Import here to avoid adding the dependency for local running scenarios.
# pylint: disable=wrong-import-order, wrong-import-position
- from apache_beam.internal import apiclient
+ from apache_beam.runners.google_cloud_dataflow.internal import apiclient
self.job = apiclient.Job(pipeline.options)
# The superclass's run will trigger a traversal of all reachable nodes.
@@ -235,7 +235,7 @@ class DataflowRunner(PipelineRunner):
"""Creates a Step object and adds it to the cache."""
# Import here to avoid adding the dependency for local running scenarios.
# pylint: disable=wrong-import-order, wrong-import-position
- from apache_beam.internal import apiclient
+ from apache_beam.runners.google_cloud_dataflow.internal import apiclient
step = apiclient.Step(step_kind, self._get_unique_step_name())
self.job.proto.steps.append(step.proto)
step.add_property(PropertyNames.USER_NAME, step_label)
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner_test.py
index e9b5aff..59bd2fc 100644
--- a/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/dataflow_runner_test.py
@@ -21,9 +21,9 @@ import unittest
import mock
-from apache_beam.internal.clients import dataflow as dataflow_api
from apache_beam.runners.google_cloud_dataflow.dataflow_runner import DataflowPipelineResult
from apache_beam.runners.google_cloud_dataflow.dataflow_runner import DataflowRuntimeException
+from apache_beam.runners.google_cloud_dataflow.internal.clients import dataflow as dataflow_api
class DataflowRunnerTest(unittest.TestCase):
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/__init__.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient.py
new file mode 100644
index 0000000..db761ea
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient.py
@@ -0,0 +1,726 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Dataflow client utility functions."""
+
+import codecs
+import getpass
+import json
+import logging
+import os
+import re
+import time
+from StringIO import StringIO
+from datetime import datetime
+
+from apitools.base.py import encoding
+from apitools.base.py import exceptions
+
+from apache_beam import utils
+from apache_beam.internal.auth import get_service_credentials
+from apache_beam.internal.clients import storage
+from apache_beam.internal.json_value import to_json_value
+from apache_beam.runners.google_cloud_dataflow.internal.clients import dataflow
+from apache_beam.transforms import cy_combiners
+from apache_beam.transforms.display import DisplayData
+from apache_beam.utils import dependency
+from apache_beam.utils import retry
+from apache_beam.utils.dependency import get_required_container_version
+from apache_beam.utils.dependency import get_sdk_name_and_version
+from apache_beam.utils.names import PropertyNames
+from apache_beam.utils.pipeline_options import DebugOptions
+from apache_beam.utils.pipeline_options import GoogleCloudOptions
+from apache_beam.utils.pipeline_options import StandardOptions
+from apache_beam.utils.pipeline_options import WorkerOptions
+
+
+class Step(object):
+ """Wrapper for a dataflow Step protobuf."""
+
+ def __init__(self, step_kind, step_name, additional_properties=None):
+ self.step_kind = step_kind
+ self.step_name = step_name
+ self.proto = dataflow.Step(kind=step_kind, name=step_name)
+ self.proto.properties = {}
+ self._additional_properties = []
+
+ if additional_properties is not None:
+ for (n, v, t) in additional_properties:
+ self.add_property(n, v, t)
+
+ def add_property(self, name, value, with_type=False):
+ self._additional_properties.append((name, value, with_type))
+ self.proto.properties.additionalProperties.append(
+ dataflow.Step.PropertiesValue.AdditionalProperty(
+ key=name, value=to_json_value(value, with_type=with_type)))
+
+ def _get_outputs(self):
+ """Returns a list of all output labels for a step."""
+ outputs = []
+ for p in self.proto.properties.additionalProperties:
+ if p.key == PropertyNames.OUTPUT_INFO:
+ for entry in p.value.array_value.entries:
+ for entry_prop in entry.object_value.properties:
+ if entry_prop.key == PropertyNames.OUTPUT_NAME:
+ outputs.append(entry_prop.value.string_value)
+ return outputs
+
+ def __reduce__(self):
+ """Reduce hook for pickling the Step class more easily."""
+ return (Step, (self.step_kind, self.step_name, self._additional_properties))
+
+ def get_output(self, tag=None):
+ """Returns name if it is one of the outputs or first output if name is None.
+
+ Args:
+ tag: tag of the output as a string or None if we want to get the
+ name of the first output.
+
+ Returns:
+ The name of the output associated with the tag or the first output
+ if tag was None.
+
+ Raises:
+ ValueError: if the tag does not exist within outputs.
+ """
+ outputs = self._get_outputs()
+ if tag is None:
+ return outputs[0]
+ else:
+ name = '%s_%s' % (PropertyNames.OUT, tag)
+ if name not in outputs:
+ raise ValueError(
+ 'Cannot find named output: %s in %s.' % (name, outputs))
+ return name
+
+
+class Environment(object):
+ """Wrapper for a dataflow Environment protobuf."""
+
+ def __init__(self, packages, options, environment_version):
+ self.standard_options = options.view_as(StandardOptions)
+ self.google_cloud_options = options.view_as(GoogleCloudOptions)
+ self.worker_options = options.view_as(WorkerOptions)
+ self.debug_options = options.view_as(DebugOptions)
+ self.proto = dataflow.Environment()
+ self.proto.clusterManagerApiService = GoogleCloudOptions.COMPUTE_API_SERVICE
+ self.proto.dataset = '{}/cloud_dataflow'.format(
+ GoogleCloudOptions.BIGQUERY_API_SERVICE)
+ self.proto.tempStoragePrefix = (
+ self.google_cloud_options.temp_location.replace(
+ 'gs:/',
+ GoogleCloudOptions.STORAGE_API_SERVICE))
+ # User agent information.
+ self.proto.userAgent = dataflow.Environment.UserAgentValue()
+ self.local = 'localhost' in self.google_cloud_options.dataflow_endpoint
+
+ if self.google_cloud_options.service_account_email:
+ self.proto.serviceAccountEmail = (
+ self.google_cloud_options.service_account_email)
+
+ sdk_name, version_string = get_sdk_name_and_version()
+
+ self.proto.userAgent.additionalProperties.extend([
+ dataflow.Environment.UserAgentValue.AdditionalProperty(
+ key='name',
+ value=to_json_value(sdk_name)),
+ dataflow.Environment.UserAgentValue.AdditionalProperty(
+ key='version', value=to_json_value(version_string))])
+ # Version information.
+ self.proto.version = dataflow.Environment.VersionValue()
+ if self.standard_options.streaming:
+ job_type = 'PYTHON_STREAMING'
+ else:
+ job_type = 'PYTHON_BATCH'
+ self.proto.version.additionalProperties.extend([
+ dataflow.Environment.VersionValue.AdditionalProperty(
+ key='job_type',
+ value=to_json_value(job_type)),
+ dataflow.Environment.VersionValue.AdditionalProperty(
+ key='major', value=to_json_value(environment_version))])
+ # Experiments
+ if self.debug_options.experiments:
+ for experiment in self.debug_options.experiments:
+ self.proto.experiments.append(experiment)
+ # Worker pool(s) information.
+ package_descriptors = []
+ for package in packages:
+ package_descriptors.append(
+ dataflow.Package(
+ location='%s/%s' % (
+ self.google_cloud_options.staging_location.replace(
+ 'gs:/', GoogleCloudOptions.STORAGE_API_SERVICE),
+ package),
+ name=package))
+
+ pool = dataflow.WorkerPool(
+ kind='local' if self.local else 'harness',
+ packages=package_descriptors,
+ taskrunnerSettings=dataflow.TaskRunnerSettings(
+ parallelWorkerSettings=dataflow.WorkerSettings(
+ baseUrl=GoogleCloudOptions.DATAFLOW_ENDPOINT,
+ servicePath=self.google_cloud_options.dataflow_endpoint)))
+ pool.autoscalingSettings = dataflow.AutoscalingSettings()
+ # Set worker pool options received through command line.
+ if self.worker_options.num_workers:
+ pool.numWorkers = self.worker_options.num_workers
+ if self.worker_options.max_num_workers:
+ pool.autoscalingSettings.maxNumWorkers = (
+ self.worker_options.max_num_workers)
+ if self.worker_options.autoscaling_algorithm:
+ values_enum = dataflow.AutoscalingSettings.AlgorithmValueValuesEnum
+ pool.autoscalingSettings.algorithm = {
+ 'NONE': values_enum.AUTOSCALING_ALGORITHM_NONE,
+ 'THROUGHPUT_BASED': values_enum.AUTOSCALING_ALGORITHM_BASIC,
+ }.get(self.worker_options.autoscaling_algorithm)
+ if self.worker_options.machine_type:
+ pool.machineType = self.worker_options.machine_type
+ if self.worker_options.disk_size_gb:
+ pool.diskSizeGb = self.worker_options.disk_size_gb
+ if self.worker_options.disk_type:
+ pool.diskType = self.worker_options.disk_type
+ if self.worker_options.zone:
+ pool.zone = self.worker_options.zone
+ if self.worker_options.network:
+ pool.network = self.worker_options.network
+ if self.worker_options.worker_harness_container_image:
+ pool.workerHarnessContainerImage = (
+ self.worker_options.worker_harness_container_image)
+ else:
+ # Default to using the worker harness container image for the current SDK
+ # version.
+ pool.workerHarnessContainerImage = (
+ 'dataflow.gcr.io/v1beta3/python:%s' %
+ get_required_container_version())
+ if self.worker_options.use_public_ips is not None:
+ if self.worker_options.use_public_ips:
+ pool.ipConfiguration = (
+ dataflow.WorkerPool
+ .IpConfigurationValueValuesEnum.WORKER_IP_PUBLIC)
+ else:
+ pool.ipConfiguration = (
+ dataflow.WorkerPool
+ .IpConfigurationValueValuesEnum.WORKER_IP_PRIVATE)
+
+ if self.standard_options.streaming:
+ # Use separate data disk for streaming.
+ disk = dataflow.Disk()
+ if self.local:
+ disk.diskType = 'local'
+ # TODO(ccy): allow customization of disk.
+ pool.dataDisks.append(disk)
+ self.proto.workerPools.append(pool)
+
+ sdk_pipeline_options = options.get_all_options()
+ if sdk_pipeline_options:
+ self.proto.sdkPipelineOptions = (
+ dataflow.Environment.SdkPipelineOptionsValue())
+
+ options_dict = {k: v
+ for k, v in sdk_pipeline_options.iteritems()
+ if v is not None}
+ self.proto.sdkPipelineOptions.additionalProperties.append(
+ dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(
+ key='options', value=to_json_value(options_dict)))
+
+ dd = DisplayData.create_from_options(options)
+ items = [item.get_dict() for item in dd.items]
+ self.proto.sdkPipelineOptions.additionalProperties.append(
+ dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(
+ key='display_data', value=to_json_value(items)))
+
+
+class Job(object):
+ """Wrapper for a dataflow Job protobuf."""
+
+ def __str__(self):
+ def encode_shortstrings(input_buffer, errors='strict'):
+ """Encoder (from Unicode) that suppresses long base64 strings."""
+ original_len = len(input_buffer)
+ if original_len > 150:
+ if self.base64_str_re.match(input_buffer):
+ input_buffer = '<string of %d bytes>' % original_len
+ input_buffer = input_buffer.encode('ascii', errors=errors)
+ else:
+ matched = self.coder_str_re.match(input_buffer)
+ if matched:
+ input_buffer = '%s<string of %d bytes>' % (
+ matched.group(1), matched.end(2) - matched.start(2))
+ input_buffer = input_buffer.encode('ascii', errors=errors)
+ return input_buffer, original_len
+
+ def decode_shortstrings(input_buffer, errors='strict'):
+ """Decoder (to Unicode) that suppresses long base64 strings."""
+ shortened, length = encode_shortstrings(input_buffer, errors)
+ return unicode(shortened), length
+
+ def shortstrings_registerer(encoding_name):
+ if encoding_name == 'shortstrings':
+ return codecs.CodecInfo(name='shortstrings',
+ encode=encode_shortstrings,
+ decode=decode_shortstrings)
+ return None
+
+ codecs.register(shortstrings_registerer)
+
+ # Use json "dump string" method to get readable formatting;
+ # further modify it to not output too-long strings, aimed at the
+ # 10,000+ character hex-encoded "serialized_fn" values.
+ return json.dumps(
+ json.loads(encoding.MessageToJson(self.proto), encoding='shortstrings'),
+ indent=2, sort_keys=True)
+
+ @staticmethod
+ def default_job_name(job_name):
+ if job_name is None:
+ user_name = getpass.getuser().lower()
+ date_component = datetime.utcnow().strftime('%m%d%H%M%S-%f')
+ app_name = 'beamapp'
+ job_name = '{}-{}-{}'.format(app_name, user_name, date_component)
+ return job_name
+
+ def __init__(self, options):
+ self.options = options
+ self.google_cloud_options = options.view_as(GoogleCloudOptions)
+ if not self.google_cloud_options.job_name:
+ self.google_cloud_options.job_name = self.default_job_name(
+ self.google_cloud_options.job_name)
+
+ required_google_cloud_options = ['project', 'job_name', 'temp_location']
+ missing = [
+ option for option in required_google_cloud_options
+ if not getattr(self.google_cloud_options, option)]
+ if missing:
+ raise ValueError(
+ 'Missing required configuration parameters: %s' % missing)
+
+ if not self.google_cloud_options.staging_location:
+ logging.info('Defaulting to the temp_location as staging_location: %s',
+ self.google_cloud_options.temp_location)
+ (self.google_cloud_options
+ .staging_location) = self.google_cloud_options.temp_location
+
+ # Make the staging and temp locations job name and time specific. This is
+ # needed to avoid clashes between job submissions using the same staging
+ # area or team members using same job names. This method is not entirely
+ # foolproof since two job submissions with same name can happen at exactly
+ # the same time. However the window is extremely small given that
+ # time.time() has at least microseconds granularity. We add the suffix only
+ # for GCS staging locations where the potential for such clashes is high.
+ if self.google_cloud_options.staging_location.startswith('gs://'):
+ path_suffix = '%s.%f' % (self.google_cloud_options.job_name, time.time())
+ self.google_cloud_options.staging_location = utils.path.join(
+ self.google_cloud_options.staging_location, path_suffix)
+ self.google_cloud_options.temp_location = utils.path.join(
+ self.google_cloud_options.temp_location, path_suffix)
+ self.proto = dataflow.Job(name=self.google_cloud_options.job_name)
+ if self.options.view_as(StandardOptions).streaming:
+ self.proto.type = dataflow.Job.TypeValueValuesEnum.JOB_TYPE_STREAMING
+ else:
+ self.proto.type = dataflow.Job.TypeValueValuesEnum.JOB_TYPE_BATCH
+ self.base64_str_re = re.compile(r'^[A-Za-z0-9+/]*=*$')
+ self.coder_str_re = re.compile(r'^([A-Za-z]+\$)([A-Za-z0-9+/]*=*)$')
+
+ def json(self):
+ return encoding.MessageToJson(self.proto)
+
+ def __reduce__(self):
+ """Reduce hook for pickling the Job class more easily."""
+ return (Job, (self.options,))
+
+
+class DataflowApplicationClient(object):
+ """A Dataflow API client used by application code to create and query jobs."""
+
+ def __init__(self, options, environment_version):
+ """Initializes a Dataflow API client object."""
+ self.standard_options = options.view_as(StandardOptions)
+ self.google_cloud_options = options.view_as(GoogleCloudOptions)
+ self.environment_version = environment_version
+ if self.google_cloud_options.no_auth:
+ credentials = None
+ else:
+ credentials = get_service_credentials()
+ self._client = dataflow.DataflowV1b3(
+ url=self.google_cloud_options.dataflow_endpoint,
+ credentials=credentials,
+ get_credentials=(not self.google_cloud_options.no_auth))
+ self._storage_client = storage.StorageV1(
+ url='https://www.googleapis.com/storage/v1',
+ credentials=credentials,
+ get_credentials=(not self.google_cloud_options.no_auth))
+
+ # TODO(silviuc): Refactor so that retry logic can be applied.
+ @retry.no_retries # Using no_retries marks this as an integration point.
+ def _gcs_file_copy(self, from_path, to_path):
+ to_folder, to_name = os.path.split(to_path)
+ with open(from_path, 'rb') as f:
+ self.stage_file(to_folder, to_name, f)
+
+ def stage_file(self, gcs_or_local_path, file_name, stream,
+ mime_type='application/octet-stream'):
+ """Stages a file at a GCS or local path with stream-supplied contents."""
+ if not gcs_or_local_path.startswith('gs://'):
+ local_path = os.path.join(gcs_or_local_path, file_name)
+ logging.info('Staging file locally to %s', local_path)
+ with open(local_path, 'wb') as f:
+ f.write(stream.read())
+ return
+ gcs_location = gcs_or_local_path + '/' + file_name
+ bucket, name = gcs_location[5:].split('/', 1)
+
+ request = storage.StorageObjectsInsertRequest(
+ bucket=bucket, name=name)
+ logging.info('Starting GCS upload to %s...', gcs_location)
+ upload = storage.Upload(stream, mime_type)
+ try:
+ response = self._storage_client.objects.Insert(request, upload=upload)
+ except exceptions.HttpError as e:
+ reportable_errors = {
+ 403: 'access denied',
+ 404: 'bucket not found',
+ }
+ if e.status_code in reportable_errors:
+ raise IOError(('Could not upload to GCS path %s: %s. Please verify '
+ 'that credentials are valid and that you have write '
+ 'access to the specified path. Stale credentials can be '
+ 'refreshed by executing "gcloud auth login".') %
+ (gcs_or_local_path, reportable_errors[e.status_code]))
+ raise
+ logging.info('Completed GCS upload to %s', gcs_location)
+ return response
+
+ # TODO(silviuc): Refactor so that retry logic can be applied.
+ @retry.no_retries # Using no_retries marks this as an integration point.
+ def create_job(self, job):
+ """Creates job description. May stage and/or submit for remote execution."""
+ self.create_job_description(job)
+
+ # Stage and submit the job when necessary
+ dataflow_job_file = job.options.view_as(DebugOptions).dataflow_job_file
+ template_location = (
+ job.options.view_as(GoogleCloudOptions).template_location)
+
+ job_location = template_location or dataflow_job_file
+ if job_location:
+ gcs_or_local_path = os.path.dirname(job_location)
+ file_name = os.path.basename(job_location)
+ self.stage_file(gcs_or_local_path, file_name, StringIO(job.json()))
+
+ if not template_location:
+ return self.submit_job_description(job)
+ else:
+ return None
+
+ def create_job_description(self, job):
+ """Creates a job described by the workflow proto."""
+ resources = dependency.stage_job_resources(
+ job.options, file_copy=self._gcs_file_copy)
+ job.proto.environment = Environment(
+ packages=resources, options=job.options,
+ environment_version=self.environment_version).proto
+ # TODO(silviuc): Remove the debug logging eventually.
+ logging.info('JOB: %s', job)
+
+ def submit_job_description(self, job):
+ """Creates and excutes a job request."""
+ request = dataflow.DataflowProjectsJobsCreateRequest()
+ request.projectId = self.google_cloud_options.project
+ request.job = job.proto
+
+ try:
+ response = self._client.projects_jobs.Create(request)
+ except exceptions.BadStatusCodeError as e:
+ logging.error('HTTP status %d trying to create job'
+ ' at dataflow service endpoint %s',
+ e.response.status,
+ self.google_cloud_options.dataflow_endpoint)
+ logging.fatal('details of server error: %s', e)
+ raise
+ logging.info('Create job: %s', response)
+ # The response is a Job proto with the id for the new job.
+ logging.info('Created job with id: [%s]', response.id)
+ logging.info(
+ 'To access the Dataflow monitoring console, please navigate to '
+ 'https://console.developers.google.com/project/%s/dataflow/job/%s',
+ self.google_cloud_options.project, response.id)
+
+ return response
+
+ @retry.with_exponential_backoff() # Using retry defaults from utils/retry.py
+ def modify_job_state(self, job_id, new_state):
+ """Modify the run state of the job.
+
+ Args:
+ job_id: The id of the job.
+ new_state: A string representing the new desired state. It could be set to
+ either 'JOB_STATE_DONE', 'JOB_STATE_CANCELLED' or 'JOB_STATE_DRAINING'.
+
+ Returns:
+ True if the job was modified successfully.
+ """
+ if new_state == 'JOB_STATE_DONE':
+ new_state = dataflow.Job.RequestedStateValueValuesEnum.JOB_STATE_DONE
+ elif new_state == 'JOB_STATE_CANCELLED':
+ new_state = dataflow.Job.RequestedStateValueValuesEnum.JOB_STATE_CANCELLED
+ elif new_state == 'JOB_STATE_DRAINING':
+ new_state = dataflow.Job.RequestedStateValueValuesEnum.JOB_STATE_DRAINING
+ else:
+ # Other states could only be set by the service.
+ return False
+
+ request = dataflow.DataflowProjectsJobsUpdateRequest()
+ request.jobId = job_id
+ request.projectId = self.google_cloud_options.project
+ request.job = dataflow.Job(requestedState=new_state)
+
+ self._client.projects_jobs.Update(request)
+ return True
+
+ @retry.with_exponential_backoff() # Using retry defaults from utils/retry.py
+ def get_job(self, job_id):
+ """Gets the job status for a submitted job.
+
+ Args:
+ job_id: A string representing the job_id for the workflow as returned
+ by the a create_job() request.
+
+ Returns:
+ A Job proto. See below for interesting fields.
+
+ The Job proto returned from a get_job() request contains some interesting
+ fields:
+ currentState: An object representing the current state of the job. The
+ string representation of the object (str() result) has the following
+ possible values: JOB_STATE_UNKNONW, JOB_STATE_STOPPED,
+ JOB_STATE_RUNNING, JOB_STATE_DONE, JOB_STATE_FAILED,
+ JOB_STATE_CANCELLED.
+ createTime: UTC time when the job was created
+ (e.g. '2015-03-10T00:01:53.074Z')
+ currentStateTime: UTC time for the current state of the job.
+ """
+ request = dataflow.DataflowProjectsJobsGetRequest()
+ request.jobId = job_id
+ request.projectId = self.google_cloud_options.project
+ response = self._client.projects_jobs.Get(request)
+ return response
+
+ @retry.with_exponential_backoff() # Using retry defaults from utils/retry.py
+ def list_messages(
+ self, job_id, start_time=None, end_time=None, page_token=None,
+ minimum_importance=None):
+ """List messages associated with the execution of a job.
+
+ Args:
+ job_id: A string representing the job_id for the workflow as returned
+ by the a create_job() request.
+ start_time: If specified, only messages generated after the start time
+ will be returned, otherwise all messages since job started will be
+ returned. The value is a string representing UTC time
+ (e.g., '2015-08-18T21:03:50.644Z')
+ end_time: If specified, only messages generated before the end time
+ will be returned, otherwise all messages up to current time will be
+ returned. The value is a string representing UTC time
+ (e.g., '2015-08-18T21:03:50.644Z')
+ page_token: A string to be used as next page token if the list call
+ returned paginated results.
+ minimum_importance: Filter for messages based on importance. The possible
+ string values in increasing order of importance are: JOB_MESSAGE_DEBUG,
+ JOB_MESSAGE_DETAILED, JOB_MESSAGE_BASIC, JOB_MESSAGE_WARNING,
+ JOB_MESSAGE_ERROR. For example, a filter set on warning will allow only
+ warnings and errors and exclude all others.
+
+ Returns:
+ A tuple consisting of a list of JobMessage instances and a
+ next page token string.
+
+ Raises:
+ RuntimeError: if an unexpected value for the message_importance argument
+ is used.
+
+ The JobMessage objects returned by the call contain the following fields:
+ id: A unique string identifier for the message.
+ time: A string representing the UTC time of the message
+ (e.g., '2015-08-18T21:03:50.644Z')
+ messageImportance: An enumeration value for the message importance. The
+ value if converted to string will have the following possible values:
+ JOB_MESSAGE_DEBUG, JOB_MESSAGE_DETAILED, JOB_MESSAGE_BASIC,
+ JOB_MESSAGE_WARNING, JOB_MESSAGE_ERROR.
+ messageText: A message string.
+ """
+ request = dataflow.DataflowProjectsJobsMessagesListRequest(
+ jobId=job_id, projectId=self.google_cloud_options.project)
+ if page_token is not None:
+ request.pageToken = page_token
+ if start_time is not None:
+ request.startTime = start_time
+ if end_time is not None:
+ request.endTime = end_time
+ if minimum_importance is not None:
+ if minimum_importance == 'JOB_MESSAGE_DEBUG':
+ request.minimumImportance = (
+ dataflow.DataflowProjectsJobsMessagesListRequest
+ .MinimumImportanceValueValuesEnum
+ .JOB_MESSAGE_DEBUG)
+ elif minimum_importance == 'JOB_MESSAGE_DETAILED':
+ request.minimumImportance = (
+ dataflow.DataflowProjectsJobsMessagesListRequest
+ .MinimumImportanceValueValuesEnum
+ .JOB_MESSAGE_DETAILED)
+ elif minimum_importance == 'JOB_MESSAGE_BASIC':
+ request.minimumImportance = (
+ dataflow.DataflowProjectsJobsMessagesListRequest
+ .MinimumImportanceValueValuesEnum
+ .JOB_MESSAGE_BASIC)
+ elif minimum_importance == 'JOB_MESSAGE_WARNING':
+ request.minimumImportance = (
+ dataflow.DataflowProjectsJobsMessagesListRequest
+ .MinimumImportanceValueValuesEnum
+ .JOB_MESSAGE_WARNING)
+ elif minimum_importance == 'JOB_MESSAGE_ERROR':
+ request.minimumImportance = (
+ dataflow.DataflowProjectsJobsMessagesListRequest
+ .MinimumImportanceValueValuesEnum
+ .JOB_MESSAGE_ERROR)
+ else:
+ raise RuntimeError(
+ 'Unexpected value for minimum_importance argument: %r',
+ minimum_importance)
+ response = self._client.projects_jobs_messages.List(request)
+ return response.jobMessages, response.nextPageToken
+
+
+class MetricUpdateTranslators(object):
+ """Translators between accumulators and dataflow metric updates."""
+
+ @staticmethod
+ def translate_boolean(accumulator, metric_update_proto):
+ metric_update_proto.boolean = accumulator.value
+
+ @staticmethod
+ def translate_scalar_mean_int(accumulator, metric_update_proto):
+ if accumulator.count:
+ metric_update_proto.integerMean = dataflow.IntegerMean()
+ metric_update_proto.integerMean.sum = to_split_int(accumulator.sum)
+ metric_update_proto.integerMean.count = to_split_int(accumulator.count)
+ else:
+ metric_update_proto.nameAndKind.kind = None
+
+ @staticmethod
+ def translate_scalar_mean_float(accumulator, metric_update_proto):
+ if accumulator.count:
+ metric_update_proto.floatingPointMean = dataflow.FloatingPointMean()
+ metric_update_proto.floatingPointMean.sum = accumulator.sum
+ metric_update_proto.floatingPointMean.count = to_split_int(
+ accumulator.count)
+ else:
+ metric_update_proto.nameAndKind.kind = None
+
+ @staticmethod
+ def translate_scalar_counter_int(accumulator, metric_update_proto):
+ metric_update_proto.integer = to_split_int(accumulator.value)
+
+ @staticmethod
+ def translate_scalar_counter_float(accumulator, metric_update_proto):
+ metric_update_proto.floatingPoint = accumulator.value
+
+
+def to_split_int(n):
+ res = dataflow.SplitInt64()
+ res.lowBits = n & 0xffffffff
+ res.highBits = n >> 32
+ return res
+
+
+def translate_distribution(distribution_update, metric_update_proto):
+ """Translate metrics DistributionUpdate to dataflow distribution update."""
+ dist_update_proto = dataflow.DistributionUpdate()
+ dist_update_proto.min = to_split_int(distribution_update.min)
+ dist_update_proto.max = to_split_int(distribution_update.max)
+ dist_update_proto.count = to_split_int(distribution_update.count)
+ dist_update_proto.sum = to_split_int(distribution_update.sum)
+ metric_update_proto.distribution = dist_update_proto
+
+
+def translate_value(value, metric_update_proto):
+ metric_update_proto.integer = to_split_int(value)
+
+
+def translate_scalar(accumulator, metric_update):
+ metric_update.scalar = to_json_value(accumulator.value, with_type=True)
+
+
+def translate_mean(accumulator, metric_update):
+ if accumulator.count:
+ metric_update.meanSum = to_json_value(accumulator.sum, with_type=True)
+ metric_update.meanCount = to_json_value(accumulator.count, with_type=True)
+ else:
+ # A denominator of 0 will raise an error in the service.
+ # What it means is we have nothing to report yet, so don't.
+ metric_update.kind = None
+
+
+# To enable a counter on the service, add it to this dictionary.
+metric_translations = {
+ cy_combiners.CountCombineFn: ('sum', translate_scalar),
+ cy_combiners.SumInt64Fn: ('sum', translate_scalar),
+ cy_combiners.MinInt64Fn: ('min', translate_scalar),
+ cy_combiners.MaxInt64Fn: ('max', translate_scalar),
+ cy_combiners.MeanInt64Fn: ('mean', translate_mean),
+ cy_combiners.SumFloatFn: ('sum', translate_scalar),
+ cy_combiners.MinFloatFn: ('min', translate_scalar),
+ cy_combiners.MaxFloatFn: ('max', translate_scalar),
+ cy_combiners.MeanFloatFn: ('mean', translate_mean),
+ cy_combiners.AllCombineFn: ('and', translate_scalar),
+ cy_combiners.AnyCombineFn: ('or', translate_scalar),
+}
+
+counter_translations = {
+ cy_combiners.CountCombineFn: (
+ dataflow.NameAndKind.KindValueValuesEnum.SUM,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.SumInt64Fn: (
+ dataflow.NameAndKind.KindValueValuesEnum.SUM,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.MinInt64Fn: (
+ dataflow.NameAndKind.KindValueValuesEnum.MIN,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.MaxInt64Fn: (
+ dataflow.NameAndKind.KindValueValuesEnum.MAX,
+ MetricUpdateTranslators.translate_scalar_counter_int),
+ cy_combiners.MeanInt64Fn: (
+ dataflow.NameAndKind.KindValueValuesEnum.MEAN,
+ MetricUpdateTranslators.translate_scalar_mean_int),
+ cy_combiners.SumFloatFn: (
+ dataflow.NameAndKind.KindValueValuesEnum.SUM,
+ MetricUpdateTranslators.translate_scalar_counter_float),
+ cy_combiners.MinFloatFn: (
+ dataflow.NameAndKind.KindValueValuesEnum.MIN,
+ MetricUpdateTranslators.translate_scalar_counter_float),
+ cy_combiners.MaxFloatFn: (
+ dataflow.NameAndKind.KindValueValuesEnum.MAX,
+ MetricUpdateTranslators.translate_scalar_counter_float),
+ cy_combiners.MeanFloatFn: (
+ dataflow.NameAndKind.KindValueValuesEnum.MEAN,
+ MetricUpdateTranslators.translate_scalar_mean_float),
+ cy_combiners.AllCombineFn: (
+ dataflow.NameAndKind.KindValueValuesEnum.AND,
+ MetricUpdateTranslators.translate_boolean),
+ cy_combiners.AnyCombineFn: (
+ dataflow.NameAndKind.KindValueValuesEnum.OR,
+ MetricUpdateTranslators.translate_boolean),
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient_test.py
new file mode 100644
index 0000000..bc57211
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/apiclient_test.py
@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Unit tests for the apiclient module."""
+
+import unittest
+
+from mock import Mock
+
+from apache_beam.metrics.cells import DistributionData
+from apache_beam.runners.google_cloud_dataflow.dataflow_runner import DataflowRunner
+from apache_beam.runners.google_cloud_dataflow.internal import apiclient
+from apache_beam.runners.google_cloud_dataflow.internal.clients import dataflow
+from apache_beam.utils.pipeline_options import PipelineOptions
+
+
+class UtilTest(unittest.TestCase):
+
+ @unittest.skip("Enable once BEAM-1080 is fixed.")
+ def test_create_application_client(self):
+ pipeline_options = PipelineOptions()
+ apiclient.DataflowApplicationClient(
+ pipeline_options,
+ DataflowRunner.BATCH_ENVIRONMENT_MAJOR_VERSION)
+
+ def test_default_job_name(self):
+ job_name = apiclient.Job.default_job_name(None)
+ regexp = 'beamapp-.*-[0-9]{10}-[0-9]{6}'
+ self.assertRegexpMatches(job_name, regexp)
+
+ def test_split_int(self):
+ number = 12345
+ split_number = apiclient.to_split_int(number)
+ self.assertEqual((split_number.lowBits, split_number.highBits),
+ (number, 0))
+ shift_number = number << 32
+ split_number = apiclient.to_split_int(shift_number)
+ self.assertEqual((split_number.lowBits, split_number.highBits),
+ (0, number))
+
+ def test_translate_distribution(self):
+ metric_update = dataflow.CounterUpdate()
+ distribution_update = DistributionData(16, 2, 1, 15)
+ apiclient.translate_distribution(distribution_update, metric_update)
+ self.assertEqual(metric_update.distribution.min.lowBits,
+ distribution_update.min)
+ self.assertEqual(metric_update.distribution.max.lowBits,
+ distribution_update.max)
+ self.assertEqual(metric_update.distribution.sum.lowBits,
+ distribution_update.sum)
+ self.assertEqual(metric_update.distribution.count.lowBits,
+ distribution_update.count)
+
+ def test_translate_means(self):
+ metric_update = dataflow.CounterUpdate()
+ accumulator = Mock()
+ accumulator.sum = 16
+ accumulator.count = 2
+ apiclient.MetricUpdateTranslators.translate_scalar_mean_int(accumulator,
+ metric_update)
+ self.assertEqual(metric_update.integerMean.sum.lowBits, accumulator.sum)
+ self.assertEqual(metric_update.integerMean.count.lowBits, accumulator.count)
+
+ accumulator.sum = 16.0
+ accumulator.count = 2
+ apiclient.MetricUpdateTranslators.translate_scalar_mean_float(accumulator,
+ metric_update)
+ self.assertEqual(metric_update.floatingPointMean.sum, accumulator.sum)
+ self.assertEqual(
+ metric_update.floatingPointMean.count.lowBits, accumulator.count)
+
+
+if __name__ == '__main__':
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/__init__.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/__init__.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/__init__.py
new file mode 100644
index 0000000..a6857f8
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/__init__.py
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Common imports for generated dataflow client library."""
+# pylint:disable=wildcard-import
+
+import pkgutil
+
+from apache_beam.runners.google_cloud_dataflow.internal.clients.dataflow.dataflow_v1b3_messages import *
+
+from apache_beam.runners.google_cloud_dataflow.internal.clients.dataflow.dataflow_v1b3_client import *
+
+__path__ = pkgutil.extend_path(__path__, __name__)
http://git-wip-us.apache.org/repos/asf/beam/blob/4337c3ec/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_client.py b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
new file mode 100644
index 0000000..4d3d525
--- /dev/null
+++ b/sdks/python/apache_beam/runners/google_cloud_dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
@@ -0,0 +1,684 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Generated client library for dataflow version v1b3."""
+# NOTE: This file is autogenerated and should not be edited by hand.
+from apitools.base.py import base_api
+
+from apache_beam.runners.google_cloud_dataflow.internal.clients.dataflow import dataflow_v1b3_messages as messages
+
+
+class DataflowV1b3(base_api.BaseApiClient):
+ """Generated client library for service dataflow version v1b3."""
+
+ MESSAGES_MODULE = messages
+ BASE_URL = u'https://dataflow.googleapis.com/'
+
+ _PACKAGE = u'dataflow'
+ _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/userinfo.email']
+ _VERSION = u'v1b3'
+ _CLIENT_ID = '1042881264118.apps.googleusercontent.com'
+ _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b'
+ _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b'
+ _CLIENT_CLASS_NAME = u'DataflowV1b3'
+ _URL_VERSION = u'v1b3'
+ _API_KEY = None
+
+ def __init__(self, url='', credentials=None,
+ get_credentials=True, http=None, model=None,
+ log_request=False, log_response=False,
+ credentials_args=None, default_global_params=None,
+ additional_http_headers=None):
+ """Create a new dataflow handle."""
+ url = url or self.BASE_URL
+ super(DataflowV1b3, self).__init__(
+ url, credentials=credentials,
+ get_credentials=get_credentials, http=http, model=model,
+ log_request=log_request, log_response=log_response,
+ credentials_args=credentials_args,
+ default_global_params=default_global_params,
+ additional_http_headers=additional_http_headers)
+ self.projects_jobs_debug = self.ProjectsJobsDebugService(self)
+ self.projects_jobs_messages = self.ProjectsJobsMessagesService(self)
+ self.projects_jobs_workItems = self.ProjectsJobsWorkItemsService(self)
+ self.projects_jobs = self.ProjectsJobsService(self)
+ self.projects_locations_jobs_messages = self.ProjectsLocationsJobsMessagesService(self)
+ self.projects_locations_jobs_workItems = self.ProjectsLocationsJobsWorkItemsService(self)
+ self.projects_locations_jobs = self.ProjectsLocationsJobsService(self)
+ self.projects_locations = self.ProjectsLocationsService(self)
+ self.projects_templates = self.ProjectsTemplatesService(self)
+ self.projects = self.ProjectsService(self)
+
+ class ProjectsJobsDebugService(base_api.BaseApiService):
+ """Service class for the projects_jobs_debug resource."""
+
+ _NAME = u'projects_jobs_debug'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsJobsDebugService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def GetConfig(self, request, global_params=None):
+ """Get encoded debug configuration for component. Not cacheable.
+
+ Args:
+ request: (DataflowProjectsJobsDebugGetConfigRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (GetDebugConfigResponse) The response message.
+ """
+ config = self.GetMethodConfig('GetConfig')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ GetConfig.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.jobs.debug.getConfig',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/debug/getConfig',
+ request_field=u'getDebugConfigRequest',
+ request_type_name=u'DataflowProjectsJobsDebugGetConfigRequest',
+ response_type_name=u'GetDebugConfigResponse',
+ supports_download=False,
+ )
+
+ def SendCapture(self, request, global_params=None):
+ """Send encoded debug capture data for component.
+
+ Args:
+ request: (DataflowProjectsJobsDebugSendCaptureRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (SendDebugCaptureResponse) The response message.
+ """
+ config = self.GetMethodConfig('SendCapture')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ SendCapture.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.jobs.debug.sendCapture',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/debug/sendCapture',
+ request_field=u'sendDebugCaptureRequest',
+ request_type_name=u'DataflowProjectsJobsDebugSendCaptureRequest',
+ response_type_name=u'SendDebugCaptureResponse',
+ supports_download=False,
+ )
+
+ class ProjectsJobsMessagesService(base_api.BaseApiService):
+ """Service class for the projects_jobs_messages resource."""
+
+ _NAME = u'projects_jobs_messages'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsJobsMessagesService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def List(self, request, global_params=None):
+ """Request the job status.
+
+ Args:
+ request: (DataflowProjectsJobsMessagesListRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ListJobMessagesResponse) The response message.
+ """
+ config = self.GetMethodConfig('List')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ List.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.jobs.messages.list',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[u'endTime', u'location', u'minimumImportance', u'pageSize', u'pageToken', u'startTime'],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/messages',
+ request_field='',
+ request_type_name=u'DataflowProjectsJobsMessagesListRequest',
+ response_type_name=u'ListJobMessagesResponse',
+ supports_download=False,
+ )
+
+ class ProjectsJobsWorkItemsService(base_api.BaseApiService):
+ """Service class for the projects_jobs_workItems resource."""
+
+ _NAME = u'projects_jobs_workItems'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsJobsWorkItemsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Lease(self, request, global_params=None):
+ """Leases a dataflow WorkItem to run.
+
+ Args:
+ request: (DataflowProjectsJobsWorkItemsLeaseRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (LeaseWorkItemResponse) The response message.
+ """
+ config = self.GetMethodConfig('Lease')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Lease.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.jobs.workItems.lease',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/workItems:lease',
+ request_field=u'leaseWorkItemRequest',
+ request_type_name=u'DataflowProjectsJobsWorkItemsLeaseRequest',
+ response_type_name=u'LeaseWorkItemResponse',
+ supports_download=False,
+ )
+
+ def ReportStatus(self, request, global_params=None):
+ """Reports the status of dataflow WorkItems leased by a worker.
+
+ Args:
+ request: (DataflowProjectsJobsWorkItemsReportStatusRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ReportWorkItemStatusResponse) The response message.
+ """
+ config = self.GetMethodConfig('ReportStatus')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ ReportStatus.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.jobs.workItems.reportStatus',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/workItems:reportStatus',
+ request_field=u'reportWorkItemStatusRequest',
+ request_type_name=u'DataflowProjectsJobsWorkItemsReportStatusRequest',
+ response_type_name=u'ReportWorkItemStatusResponse',
+ supports_download=False,
+ )
+
+ class ProjectsJobsService(base_api.BaseApiService):
+ """Service class for the projects_jobs resource."""
+
+ _NAME = u'projects_jobs'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsJobsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Create(self, request, global_params=None):
+ """Creates a Cloud Dataflow job.
+
+ Args:
+ request: (DataflowProjectsJobsCreateRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Create')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Create.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.jobs.create',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[u'location', u'replaceJobId', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/jobs',
+ request_field=u'job',
+ request_type_name=u'DataflowProjectsJobsCreateRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ )
+
+ def Get(self, request, global_params=None):
+ """Gets the state of the specified Cloud Dataflow job.
+
+ Args:
+ request: (DataflowProjectsJobsGetRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Get')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Get.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.jobs.get',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[u'location', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}',
+ request_field='',
+ request_type_name=u'DataflowProjectsJobsGetRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ )
+
+ def GetMetrics(self, request, global_params=None):
+ """Request the job status.
+
+ Args:
+ request: (DataflowProjectsJobsGetMetricsRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (JobMetrics) The response message.
+ """
+ config = self.GetMethodConfig('GetMetrics')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ GetMetrics.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.jobs.getMetrics',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[u'location', u'startTime'],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}/metrics',
+ request_field='',
+ request_type_name=u'DataflowProjectsJobsGetMetricsRequest',
+ response_type_name=u'JobMetrics',
+ supports_download=False,
+ )
+
+ def List(self, request, global_params=None):
+ """List the jobs of a project.
+
+ Args:
+ request: (DataflowProjectsJobsListRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ListJobsResponse) The response message.
+ """
+ config = self.GetMethodConfig('List')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ List.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.jobs.list',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[u'filter', u'location', u'pageSize', u'pageToken', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/jobs',
+ request_field='',
+ request_type_name=u'DataflowProjectsJobsListRequest',
+ response_type_name=u'ListJobsResponse',
+ supports_download=False,
+ )
+
+ def Update(self, request, global_params=None):
+ """Updates the state of an existing Cloud Dataflow job.
+
+ Args:
+ request: (DataflowProjectsJobsUpdateRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Update')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Update.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'PUT',
+ method_id=u'dataflow.projects.jobs.update',
+ ordered_params=[u'projectId', u'jobId'],
+ path_params=[u'jobId', u'projectId'],
+ query_params=[u'location'],
+ relative_path=u'v1b3/projects/{projectId}/jobs/{jobId}',
+ request_field=u'job',
+ request_type_name=u'DataflowProjectsJobsUpdateRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ )
+
+ class ProjectsLocationsJobsMessagesService(base_api.BaseApiService):
+ """Service class for the projects_locations_jobs_messages resource."""
+
+ _NAME = u'projects_locations_jobs_messages'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsJobsMessagesService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def List(self, request, global_params=None):
+ """Request the job status.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsMessagesListRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ListJobMessagesResponse) The response message.
+ """
+ config = self.GetMethodConfig('List')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ List.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.jobs.messages.list',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[u'endTime', u'minimumImportance', u'pageSize', u'pageToken', u'startTime'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/messages',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsJobsMessagesListRequest',
+ response_type_name=u'ListJobMessagesResponse',
+ supports_download=False,
+ )
+
+ class ProjectsLocationsJobsWorkItemsService(base_api.BaseApiService):
+ """Service class for the projects_locations_jobs_workItems resource."""
+
+ _NAME = u'projects_locations_jobs_workItems'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsJobsWorkItemsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Lease(self, request, global_params=None):
+ """Leases a dataflow WorkItem to run.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsWorkItemsLeaseRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (LeaseWorkItemResponse) The response message.
+ """
+ config = self.GetMethodConfig('Lease')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Lease.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.jobs.workItems.lease',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/workItems:lease',
+ request_field=u'leaseWorkItemRequest',
+ request_type_name=u'DataflowProjectsLocationsJobsWorkItemsLeaseRequest',
+ response_type_name=u'LeaseWorkItemResponse',
+ supports_download=False,
+ )
+
+ def ReportStatus(self, request, global_params=None):
+ """Reports the status of dataflow WorkItems leased by a worker.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsWorkItemsReportStatusRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ReportWorkItemStatusResponse) The response message.
+ """
+ config = self.GetMethodConfig('ReportStatus')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ ReportStatus.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.jobs.workItems.reportStatus',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/workItems:reportStatus',
+ request_field=u'reportWorkItemStatusRequest',
+ request_type_name=u'DataflowProjectsLocationsJobsWorkItemsReportStatusRequest',
+ response_type_name=u'ReportWorkItemStatusResponse',
+ supports_download=False,
+ )
+
+ class ProjectsLocationsJobsService(base_api.BaseApiService):
+ """Service class for the projects_locations_jobs resource."""
+
+ _NAME = u'projects_locations_jobs'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsJobsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Create(self, request, global_params=None):
+ """Creates a Cloud Dataflow job.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsCreateRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Create')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Create.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.locations.jobs.create',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[u'replaceJobId', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs',
+ request_field=u'job',
+ request_type_name=u'DataflowProjectsLocationsJobsCreateRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ )
+
+ def Get(self, request, global_params=None):
+ """Gets the state of the specified Cloud Dataflow job.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsGetRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Get')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Get.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.jobs.get',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[u'view'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsJobsGetRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ )
+
+ def GetMetrics(self, request, global_params=None):
+ """Request the job status.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsGetMetricsRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (JobMetrics) The response message.
+ """
+ config = self.GetMethodConfig('GetMetrics')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ GetMetrics.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.jobs.getMetrics',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[u'startTime'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/metrics',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsJobsGetMetricsRequest',
+ response_type_name=u'JobMetrics',
+ supports_download=False,
+ )
+
+ def List(self, request, global_params=None):
+ """List the jobs of a project.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsListRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (ListJobsResponse) The response message.
+ """
+ config = self.GetMethodConfig('List')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ List.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'GET',
+ method_id=u'dataflow.projects.locations.jobs.list',
+ ordered_params=[u'projectId', u'location'],
+ path_params=[u'location', u'projectId'],
+ query_params=[u'filter', u'pageSize', u'pageToken', u'view'],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs',
+ request_field='',
+ request_type_name=u'DataflowProjectsLocationsJobsListRequest',
+ response_type_name=u'ListJobsResponse',
+ supports_download=False,
+ )
+
+ def Update(self, request, global_params=None):
+ """Updates the state of an existing Cloud Dataflow job.
+
+ Args:
+ request: (DataflowProjectsLocationsJobsUpdateRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Update')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Update.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'PUT',
+ method_id=u'dataflow.projects.locations.jobs.update',
+ ordered_params=[u'projectId', u'location', u'jobId'],
+ path_params=[u'jobId', u'location', u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}',
+ request_field=u'job',
+ request_type_name=u'DataflowProjectsLocationsJobsUpdateRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ )
+
+ class ProjectsLocationsService(base_api.BaseApiService):
+ """Service class for the projects_locations resource."""
+
+ _NAME = u'projects_locations'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsLocationsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ class ProjectsTemplatesService(base_api.BaseApiService):
+ """Service class for the projects_templates resource."""
+
+ _NAME = u'projects_templates'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsTemplatesService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def Create(self, request, global_params=None):
+ """Creates a Cloud Dataflow job from a template.
+
+ Args:
+ request: (DataflowProjectsTemplatesCreateRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (Job) The response message.
+ """
+ config = self.GetMethodConfig('Create')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ Create.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.templates.create',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/templates',
+ request_field=u'createJobFromTemplateRequest',
+ request_type_name=u'DataflowProjectsTemplatesCreateRequest',
+ response_type_name=u'Job',
+ supports_download=False,
+ )
+
+ class ProjectsService(base_api.BaseApiService):
+ """Service class for the projects resource."""
+
+ _NAME = u'projects'
+
+ def __init__(self, client):
+ super(DataflowV1b3.ProjectsService, self).__init__(client)
+ self._upload_configs = {
+ }
+
+ def WorkerMessages(self, request, global_params=None):
+ """Send a worker_message to the service.
+
+ Args:
+ request: (DataflowProjectsWorkerMessagesRequest) input message
+ global_params: (StandardQueryParameters, default: None) global arguments
+ Returns:
+ (SendWorkerMessagesResponse) The response message.
+ """
+ config = self.GetMethodConfig('WorkerMessages')
+ return self._RunMethod(
+ config, request, global_params=global_params)
+
+ WorkerMessages.method_config = lambda: base_api.ApiMethodInfo(
+ http_method=u'POST',
+ method_id=u'dataflow.projects.workerMessages',
+ ordered_params=[u'projectId'],
+ path_params=[u'projectId'],
+ query_params=[],
+ relative_path=u'v1b3/projects/{projectId}/WorkerMessages',
+ request_field=u'sendWorkerMessagesRequest',
+ request_type_name=u'DataflowProjectsWorkerMessagesRequest',
+ response_type_name=u'SendWorkerMessagesResponse',
+ supports_download=False,
+ )