[GitHub] [beam] alanmyrvold commented on a change in pull request #13117: [BEAM-11067] Update java license script to include source for GNU and Mozilla dependencies

[GitBox <gi...@apache.org>]

alanmyrvold commented on a change in pull request #13117:
URL: https://github.com/apache/beam/pull/13117#discussion_r505680858



##########
File path: sdks/java/container/license_scripts/pull_licenses_java.py
##########
@@ -140,17 +149,22 @@ def execute(dep):
     }
     '''
 
-    name = dep['moduleName'].split(':')[1].lower()
+    name = dep['moduleName'].split(':')[1]
     version = dep['moduleVersion']
     name_version = name + '-' + version
+    # javac is not a runtime dependency
+    if name == 'javac':
+      logging.debug('Skipping', name_version)
+      return
     dir_name = '{license_dir}/{name_version}.jar'.format(
         license_dir=license_dir, name_version=name_version)
 
     # if auto pulled, directory is existing at {license_dir}
     if not os.path.isdir(dir_name):
         # skip self dependencies
         if dep['moduleName'].startswith('beam'):
-            logging.debug('Skippig', name_version)
+            logging.debug('Skipping', name_version)

Review comment:
       Moved to the top.

##########
File path: sdks/java/container/license_scripts/pull_licenses_java.py
##########
@@ -101,13 +101,22 @@ def pull_from_url(file_name, url, dep, no_list):
 
 def pull_source_code(base_url, dir_name, dep):
     # base_url example: https://repo1.maven.org/maven2/org/mortbay/jetty/jsp-2.1/6.1.14/
-    soup = BeautifulSoup(urlopen(base_url).read(), "html.parser")
+    try:
+      soup = BeautifulSoup(urlopen(base_url).read(), "html.parser")
+    except:
+      logging.error('Error reading source base from {base_url}'.format(base_url=base_url))
+      raise
+    source_count = 0
     for href in (a["href"] for a in soup.select("a[href]")):
         if href.endswith(
-                '.jar') and not 'javadoc' in href:  # download jar file only
+                '.jar') and 'sources.jar' in href:  # download sources jar file only
             file_name = dir_name + '/' + href
             url = base_url + '/' + href
+            logging.info('Pulling source from {url}'.format(url=url))

Review comment:
       done

##########
File path: sdks/java/container/license_scripts/pull_licenses_java.py
##########
@@ -101,13 +101,22 @@ def pull_from_url(file_name, url, dep, no_list):
 
 def pull_source_code(base_url, dir_name, dep):
     # base_url example: https://repo1.maven.org/maven2/org/mortbay/jetty/jsp-2.1/6.1.14/
-    soup = BeautifulSoup(urlopen(base_url).read(), "html.parser")
+    try:
+      soup = BeautifulSoup(urlopen(base_url).read(), "html.parser")
+    except:
+      logging.error('Error reading source base from {base_url}'.format(base_url=base_url))
+      raise
+    source_count = 0
     for href in (a["href"] for a in soup.select("a[href]")):
         if href.endswith(
-                '.jar') and not 'javadoc' in href:  # download jar file only
+                '.jar') and 'sources.jar' in href:  # download sources jar file only

Review comment:
       Yes, and this is guarded by the check that at least one file is found matching.
   
       if source_count == 0:
         raise RuntimeError('No source found at {base_url}'.format(base_url=base_url))




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org