You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2022/08/02 17:55:43 UTC

[beam] branch master updated: Downgrade less informative logs during write to files (#22273)

This is an automated email from the ASF dual-hosted git repository.

tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 649aba43343 Downgrade less informative logs during write to files (#22273)
649aba43343 is described below

commit 649aba4334347a5f011cc6b46de6444a4aaa73c3
Author: Yi Hu <ya...@google.com>
AuthorDate: Tue Aug 2 13:55:35 2022 -0400

    Downgrade less informative logs during write to files (#22273)
---
 sdks/python/apache_beam/io/aws/s3io.py            |  8 +++++---
 sdks/python/apache_beam/io/azure/blobstorageio.py |  8 +++++---
 sdks/python/apache_beam/io/fileio.py              | 12 ++++++------
 sdks/python/apache_beam/io/gcp/gcsio.py           |  8 +++++---
 4 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/sdks/python/apache_beam/io/aws/s3io.py b/sdks/python/apache_beam/io/aws/s3io.py
index d8bbfe164be..66bdaa52e90 100644
--- a/sdks/python/apache_beam/io/aws/s3io.py
+++ b/sdks/python/apache_beam/io/aws/s3io.py
@@ -120,9 +120,9 @@ class S3IO(object):
     start_time = time.time()
 
     if with_metadata:
-      logging.info("Starting the file information of the input")
+      logging.debug("Starting the file information of the input")
     else:
-      logging.info("Starting the size estimation of the input")
+      logging.debug("Starting the size estimation of the input")
 
     while True:
       #The list operation will raise an exception
@@ -157,7 +157,9 @@ class S3IO(object):
       else:
         break
 
-    logging.info(
+    logging.log(
+        # do not spam logs when list_prefix is likely used to check empty folder
+        logging.INFO if counter > 0 else logging.DEBUG,
         "Finished listing %s files in %s seconds.",
         counter,
         time.time() - start_time)
diff --git a/sdks/python/apache_beam/io/azure/blobstorageio.py b/sdks/python/apache_beam/io/azure/blobstorageio.py
index 290b5228198..ae0a4944d52 100644
--- a/sdks/python/apache_beam/io/azure/blobstorageio.py
+++ b/sdks/python/apache_beam/io/azure/blobstorageio.py
@@ -588,9 +588,9 @@ class BlobStorageIO(object):
     start_time = time.time()
 
     if with_metadata:
-      logging.info("Starting the file information of the input")
+      logging.debug("Starting the file information of the input")
     else:
-      logging.info("Starting the size estimation of the input")
+      logging.debug("Starting the size estimation of the input")
     container_client = self.client.get_container_client(container)
 
     while True:
@@ -612,7 +612,9 @@ class BlobStorageIO(object):
             logging.info("Finished computing size of: %s files", len(file_info))
       break
 
-    logging.info(
+    logging.log(
+        # do not spam logs when list_prefix is likely used to check empty folder
+        logging.INFO if counter > 0 else logging.DEBUG,
         "Finished listing %s files in %s seconds.",
         counter,
         time.time() - start_time)
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 38881e81d90..e6352804963 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -689,9 +689,8 @@ class _MoveTempFilesIntoFinalDestinationFn(beam.DoFn):
       yield FileResult(
           final_file_name, i, len(file_results), r.window, r.pane, destination)
 
-    _LOGGER.info(
-        'Checking orphaned temporary files for'
-        ' destination %s and window %s',
+    _LOGGER.debug(
+        'Checking orphaned temporary files for destination %s and window %s',
         destination,
         w)
     writer_key = (destination, w)
@@ -704,9 +703,10 @@ class _MoveTempFilesIntoFinalDestinationFn(beam.DoFn):
       match_result = filesystems.FileSystems.match(['%s*' % prefix])
       orphaned_files = [m.path for m in match_result[0].metadata_list]
 
-      _LOGGER.info(
-          'Some files may be left orphaned in the temporary folder: %s',
-          orphaned_files)
+      if len(orphaned_files) > 0:
+        _LOGGER.info(
+            'Some files may be left orphaned in the temporary folder: %s',
+            orphaned_files)
     except BeamIOError as e:
       _LOGGER.info('Exceptions when checking orphaned files: %s', e)
 
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index bf41ae64610..d4ceeda4bd9 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -580,9 +580,9 @@ class GcsIO(object):
     counter = 0
     start_time = time.time()
     if with_metadata:
-      _LOGGER.info("Starting the file information of the input")
+      _LOGGER.debug("Starting the file information of the input")
     else:
-      _LOGGER.info("Starting the size estimation of the input")
+      _LOGGER.debug("Starting the size estimation of the input")
     while True:
       response = self.client.objects.List(request)
       for item in response.items:
@@ -604,7 +604,9 @@ class GcsIO(object):
         request.pageToken = response.nextPageToken
       else:
         break
-    _LOGGER.info(
+    _LOGGER.log(
+        # do not spam logs when list_prefix is likely used to check empty folder
+        logging.INFO if counter > 0 else logging.DEBUG,
         "Finished listing %s files in %s seconds.",
         counter,
         time.time() - start_time)