You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@beam.apache.org by GitBox <gi...@apache.org> on 2021/06/18 21:06:26 UTC
[GitHub] [beam] TheNeuralBit commented on a change in pull request #14770: [BEAM-11984] Request count metrics for GCSIO
TheNeuralBit commented on a change in pull request #14770:
URL: https://github.com/apache/beam/pull/14770#discussion_r653965379
##########
File path: sdks/python/apache_beam/io/gcp/gcsio_test.py
##########
@@ -751,6 +768,53 @@ def test_mime_binary_encoding(self):
generator._handle_text(message)
self.assertEqual(test_msg.encode('ascii'), output_buffer.getvalue())
+ def test_downloader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ self.gcs.open(file_name, 'r')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.get',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+
+ metric_name = MetricName(
+ None, None, urn=monitoring_infos.API_REQUEST_COUNT_URN, labels=labels)
+ metric_value = MetricsEnvironment.process_wide_container().get_counter(
+ metric_name).get_cumulative()
+
+ self.assertEqual(metric_value, 2)
+
+ def test_uploader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ f = self.gcs.open(file_name, 'w')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.insert',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
Review comment:
Are these labels supposed to map to [`MonitoringInfo.labels`](https://github.com/apache/beam/blob/2a86ae963a748328dd671f5cfc8eb9c8399664ac/model/pipeline/src/main/proto/metrics.proto#L432)? That's a map<string, string> but this value is an integer.
##########
File path: sdks/python/apache_beam/io/gcp/gcsio_test.py
##########
@@ -751,6 +768,53 @@ def test_mime_binary_encoding(self):
generator._handle_text(message)
self.assertEqual(test_msg.encode('ascii'), output_buffer.getvalue())
+ def test_downloader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ self.gcs.open(file_name, 'r')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.get',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+
+ metric_name = MetricName(
+ None, None, urn=monitoring_infos.API_REQUEST_COUNT_URN, labels=labels)
+ metric_value = MetricsEnvironment.process_wide_container().get_counter(
+ metric_name).get_cumulative()
+
+ self.assertEqual(metric_value, 2)
+
+ def test_uploader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ f = self.gcs.open(file_name, 'w')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.insert',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+
+ f.close()
+ metric_name = MetricName(
+ None, None, urn=monitoring_infos.API_REQUEST_COUNT_URN, labels=labels)
+ metric_value = MetricsEnvironment.process_wide_container().get_counter(
Review comment:
I'm assuming this process_wide_container is intended to be a global container that ends up being shared between tests, because we often run many tests in one process.
##########
File path: sdks/python/apache_beam/io/gcp/gcsio_test.py
##########
@@ -751,6 +768,53 @@ def test_mime_binary_encoding(self):
generator._handle_text(message)
self.assertEqual(test_msg.encode('ascii'), output_buffer.getvalue())
+ def test_downloader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ self.gcs.open(file_name, 'r')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.get',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+
+ metric_name = MetricName(
+ None, None, urn=monitoring_infos.API_REQUEST_COUNT_URN, labels=labels)
+ metric_value = MetricsEnvironment.process_wide_container().get_counter(
+ metric_name).get_cumulative()
+
+ self.assertEqual(metric_value, 2)
+
+ def test_uploader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ f = self.gcs.open(file_name, 'w')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.insert',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+
+ f.close()
+ metric_name = MetricName(
+ None, None, urn=monitoring_infos.API_REQUEST_COUNT_URN, labels=labels)
+ metric_value = MetricsEnvironment.process_wide_container().get_counter(
Review comment:
I'm assuming this process_wide_container is a global container that ends up being shared between tests, because we often run many tests in one process.
##########
File path: sdks/python/apache_beam/io/gcp/gcsio_test.py
##########
@@ -751,6 +768,53 @@ def test_mime_binary_encoding(self):
generator._handle_text(message)
self.assertEqual(test_msg.encode('ascii'), output_buffer.getvalue())
+ def test_downloader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ self.gcs.open(file_name, 'r')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.get',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+
+ metric_name = MetricName(
+ None, None, urn=monitoring_infos.API_REQUEST_COUNT_URN, labels=labels)
+ metric_value = MetricsEnvironment.process_wide_container().get_counter(
+ metric_name).get_cumulative()
+
+ self.assertEqual(metric_value, 2)
+
+ def test_uploader_monitoring_info(self):
+ file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
+ file_size = 5 * 1024 * 1024 + 100
+ random_file = self._insert_random_file(self.client, file_name, file_size)
+ f = self.gcs.open(file_name, 'w')
+
+ resource = resource_identifiers.GoogleCloudStorageBucket(random_file.bucket)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Storage',
+ monitoring_infos.METHOD_LABEL: 'Objects.insert',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
+ monitoring_infos.GCS_PROJECT_ID_LABEL: DEFAULT_PROJECT_NUMBER,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+
+ f.close()
+ metric_name = MetricName(
+ None, None, urn=monitoring_infos.API_REQUEST_COUNT_URN, labels=labels)
+ metric_value = MetricsEnvironment.process_wide_container().get_counter(
Review comment:
The failing test does have a reset() call: https://github.com/apache/beam/blob/f053948ccea7c26588e17435842ad02b6b23d975/sdks/python/apache_beam/runners/worker/sdk_worker_test.py#L225
Shouldn't that have prevented this flake?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org