You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2022/08/02 17:55:43 UTC
[beam] branch master updated: Downgrade less informative logs during write to files (#22273)
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 649aba43343 Downgrade less informative logs during write to files (#22273)
649aba43343 is described below
commit 649aba4334347a5f011cc6b46de6444a4aaa73c3
Author: Yi Hu <ya...@google.com>
AuthorDate: Tue Aug 2 13:55:35 2022 -0400
Downgrade less informative logs during write to files (#22273)
---
sdks/python/apache_beam/io/aws/s3io.py | 8 +++++---
sdks/python/apache_beam/io/azure/blobstorageio.py | 8 +++++---
sdks/python/apache_beam/io/fileio.py | 12 ++++++------
sdks/python/apache_beam/io/gcp/gcsio.py | 8 +++++---
4 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/sdks/python/apache_beam/io/aws/s3io.py b/sdks/python/apache_beam/io/aws/s3io.py
index d8bbfe164be..66bdaa52e90 100644
--- a/sdks/python/apache_beam/io/aws/s3io.py
+++ b/sdks/python/apache_beam/io/aws/s3io.py
@@ -120,9 +120,9 @@ class S3IO(object):
start_time = time.time()
if with_metadata:
- logging.info("Starting the file information of the input")
+ logging.debug("Starting the file information of the input")
else:
- logging.info("Starting the size estimation of the input")
+ logging.debug("Starting the size estimation of the input")
while True:
#The list operation will raise an exception
@@ -157,7 +157,9 @@ class S3IO(object):
else:
break
- logging.info(
+ logging.log(
+ # do not spam logs when list_prefix is likely used to check empty folder
+ logging.INFO if counter > 0 else logging.DEBUG,
"Finished listing %s files in %s seconds.",
counter,
time.time() - start_time)
diff --git a/sdks/python/apache_beam/io/azure/blobstorageio.py b/sdks/python/apache_beam/io/azure/blobstorageio.py
index 290b5228198..ae0a4944d52 100644
--- a/sdks/python/apache_beam/io/azure/blobstorageio.py
+++ b/sdks/python/apache_beam/io/azure/blobstorageio.py
@@ -588,9 +588,9 @@ class BlobStorageIO(object):
start_time = time.time()
if with_metadata:
- logging.info("Starting the file information of the input")
+ logging.debug("Starting the file information of the input")
else:
- logging.info("Starting the size estimation of the input")
+ logging.debug("Starting the size estimation of the input")
container_client = self.client.get_container_client(container)
while True:
@@ -612,7 +612,9 @@ class BlobStorageIO(object):
logging.info("Finished computing size of: %s files", len(file_info))
break
- logging.info(
+ logging.log(
+ # do not spam logs when list_prefix is likely used to check empty folder
+ logging.INFO if counter > 0 else logging.DEBUG,
"Finished listing %s files in %s seconds.",
counter,
time.time() - start_time)
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 38881e81d90..e6352804963 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -689,9 +689,8 @@ class _MoveTempFilesIntoFinalDestinationFn(beam.DoFn):
yield FileResult(
final_file_name, i, len(file_results), r.window, r.pane, destination)
- _LOGGER.info(
- 'Checking orphaned temporary files for'
- ' destination %s and window %s',
+ _LOGGER.debug(
+ 'Checking orphaned temporary files for destination %s and window %s',
destination,
w)
writer_key = (destination, w)
@@ -704,9 +703,10 @@ class _MoveTempFilesIntoFinalDestinationFn(beam.DoFn):
match_result = filesystems.FileSystems.match(['%s*' % prefix])
orphaned_files = [m.path for m in match_result[0].metadata_list]
- _LOGGER.info(
- 'Some files may be left orphaned in the temporary folder: %s',
- orphaned_files)
+ if len(orphaned_files) > 0:
+ _LOGGER.info(
+ 'Some files may be left orphaned in the temporary folder: %s',
+ orphaned_files)
except BeamIOError as e:
_LOGGER.info('Exceptions when checking orphaned files: %s', e)
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index bf41ae64610..d4ceeda4bd9 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -580,9 +580,9 @@ class GcsIO(object):
counter = 0
start_time = time.time()
if with_metadata:
- _LOGGER.info("Starting the file information of the input")
+ _LOGGER.debug("Starting the file information of the input")
else:
- _LOGGER.info("Starting the size estimation of the input")
+ _LOGGER.debug("Starting the size estimation of the input")
while True:
response = self.client.objects.List(request)
for item in response.items:
@@ -604,7 +604,9 @@ class GcsIO(object):
request.pageToken = response.nextPageToken
else:
break
- _LOGGER.info(
+ _LOGGER.log(
+ # do not spam logs when list_prefix is likely used to check empty folder
+ logging.INFO if counter > 0 else logging.DEBUG,
"Finished listing %s files in %s seconds.",
counter,
time.time() - start_time)