[beam] branch master updated: [BEAM-14390] Set user-agent when pulling licenses to avoid 403s (#17521)

[tv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new c52a5b87661 [BEAM-14390] Set user-agent when pulling licenses to avoid 403s (#17521)
c52a5b87661 is described below

commit c52a5b8766146fc0e39298ac66dc9319ac7eb99e
Author: tvalentyn <tv...@users.noreply.github.com>
AuthorDate: Mon May 2 16:38:12 2022 +0200

    [BEAM-14390] Set user-agent when pulling licenses to avoid 403s (#17521)
---
 sdks/java/container/license_scripts/pull_licenses_java.py | 4 ++--
 sdks/python/container/license_scripts/pull_licenses_py.py | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/sdks/java/container/license_scripts/pull_licenses_java.py b/sdks/java/container/license_scripts/pull_licenses_java.py
index 8c81556d7dc..cfa23f8f352 100644
--- a/sdks/java/container/license_scripts/pull_licenses_java.py
+++ b/sdks/java/container/license_scripts/pull_licenses_java.py
@@ -36,7 +36,7 @@ from queue import Queue
 from tenacity import retry
 from tenacity import stop_after_attempt
 from tenacity import wait_fixed
-from urllib.request import urlopen, URLError, HTTPError
+from urllib.request import urlopen, Request, URLError, HTTPError
 
 SOURCE_CODE_REQUIRED_LICENSES = ['lgpl', 'gpl', 'cddl', 'mpl', 'gnu', 'mozilla public license']
 RETRY_NUM = 9
@@ -55,7 +55,7 @@ def pull_from_url(file_name, url, dep, no_list):
         logging.info('Replaced local file URL with {url} for {dep}'.format(url=url, dep=dep))
 
     try:
-        url_read = urlopen(url)
+        url_read = urlopen(Request(url, headers={'User-Agent': 'Apache Beam'}))
         with open(file_name, 'wb') as temp_write:
             shutil.copyfileobj(url_read, temp_write)
         logging.debug(
diff --git a/sdks/python/container/license_scripts/pull_licenses_py.py b/sdks/python/container/license_scripts/pull_licenses_py.py
index 4754da4d105..fc37cda3631 100644
--- a/sdks/python/container/license_scripts/pull_licenses_py.py
+++ b/sdks/python/container/license_scripts/pull_licenses_py.py
@@ -30,7 +30,7 @@ import tempfile
 import traceback
 import yaml
 
-from urllib.request import urlopen
+from urllib.request import urlopen, Request
 from urllib.parse import urlparse
 from urllib.parse import urljoin
 from tenacity import retry
@@ -94,7 +94,8 @@ def pull_from_url(dep, configs):
       if config['license'] == 'skip':
         print('Skip pulling license for ', dep)
       else:
-        url_read = urlopen(config['license'])
+        url_read = urlopen(Request(config['license'],
+                                   headers={'User-Agent': 'Apache Beam'}))
         with open(cur_temp_dir + '/LICENSE', 'wb') as temp_write:
           shutil.copyfileobj(url_read, temp_write)
         logging.debug(