[beam] branch release-2.46.0 updated: Cherry pick some changes to fix the python postcommits (#25667)

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

damccorm pushed a commit to branch release-2.46.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.46.0 by this push:
     new c528eab18b3 Cherry pick some changes to fix the python postcommits (#25667)
c528eab18b3 is described below

commit c528eab18b32342daed53b750fe330d30c7e5224
Author: Danny McCormick <da...@google.com>
AuthorDate: Tue Feb 28 16:43:42 2023 -0500

    Cherry pick some changes to fix the python postcommits (#25667)
    
    * Fix tensorflowhub caching issue (#25661)
    
    * Fix tensorflowhub caching issue
    
    * Comment
    
    * lint
    
    * Fix Python3.7 PostCommit Azure integration test (#25654)
    
    ---------
    
    Co-authored-by: Yi Hu <ya...@google.com>
---
 .../io/azure/integration_test/docker-compose.yml   |  2 +-
 .../ml/inference/tensorflow_inference_it_test.py   | 24 ++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/io/azure/integration_test/docker-compose.yml b/sdks/python/apache_beam/io/azure/integration_test/docker-compose.yml
index 963b4a94c56..26d382b5191 100644
--- a/sdks/python/apache_beam/io/azure/integration_test/docker-compose.yml
+++ b/sdks/python/apache_beam/io/azure/integration_test/docker-compose.yml
@@ -19,7 +19,7 @@ version: '3'
 
 services:
   azurite:
-    image: mcr.microsoft.com/azure-storage/azurite:3.20.1
+    image: mcr.microsoft.com/azure-storage/azurite:3.22.0
     command:
       - "azurite-blob"
       - "--blobHost"
diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py
index fb1a2964841..bdc0291dd1e 100644
--- a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py
+++ b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py
@@ -20,6 +20,7 @@
 import logging
 import unittest
 import uuid
+from pathlib import Path
 
 import pytest
 
@@ -29,6 +30,7 @@ from apache_beam.testing.test_pipeline import TestPipeline
 # pylint: disable=ungrouped-imports
 try:
   import tensorflow as tf
+  import tensorflow_hub as hub
   from apache_beam.examples.inference import tensorflow_imagenet_segmentation
   from apache_beam.examples.inference import tensorflow_mnist_classification
   from apache_beam.examples.inference import tensorflow_mnist_with_weights
@@ -43,6 +45,27 @@ def process_outputs(filepath):
   return lines
 
 
+def rmdir(directory):
+  directory = Path(directory)
+  for item in directory.iterdir():
+    if item.is_dir():
+      rmdir(item)
+    else:
+      item.unlink()
+  directory.rmdir()
+
+
+def clear_tf_hub_temp_dir(model_path):
+  # When loading from tensorflow hub using tfhub.resolve, the model is saved in
+  # a temporary directory. That file can be persisted between test runs, in
+  # which case tfhub.resolve will no-op. If the model is deleted and the file
+  # isn't, tfhub.resolve will still no-op and tf.keras.models.load_model will
+  # throw. To avoid this (and test more robustly) we delete the temporary
+  # directory entirely between runs.
+  local_path = hub.resolve(model_path)
+  rmdir(local_path)
+
+
 @unittest.skipIf(
     tf is None, 'Missing dependencies. '
     'Test depends on tensorflow')
@@ -90,6 +113,7 @@ class TensorflowInference(unittest.TestCase):
     output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt'])
     model_path = (
         'https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/4')
+    clear_tf_hub_temp_dir(model_path)
     extra_opts = {
         'input': input_file,
         'output': output_file,