svn commit: r1727050 - /comdev/projects.apache.org/scripts/cronjobs/urlutils.py

[se...@apache.org]

Author: sebb
Date: Wed Jan 27 13:41:37 2016
New Revision: 1727050

URL: http://svn.apache.org/viewvc?rev=1727050&view=rev
Log:
Auto-generate the file name from the last part of the URL

Modified:
    comdev/projects.apache.org/scripts/cronjobs/urlutils.py

Modified: comdev/projects.apache.org/scripts/cronjobs/urlutils.py
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/scripts/cronjobs/urlutils.py?rev=1727050&r1=1727049&r2=1727050&view=diff
==============================================================================
--- comdev/projects.apache.org/scripts/cronjobs/urlutils.py (original)
+++ comdev/projects.apache.org/scripts/cronjobs/urlutils.py Wed Jan 27 13:41:37 2016
@@ -4,17 +4,19 @@
 """
 
 import os
-from os.path import dirname, abspath, join, getmtime
+from os.path import dirname, abspath, join, getmtime, basename
 import shutil
 import io
 # Allow for Python2/3 differences
 try:
     from urllib.request import urlopen, Request
     from urllib.error import HTTPError
+    from urllib.parse import urlparse
     _PY3 = True
 except:
     from urllib2 import urlopen, Request
     from urllib2 import HTTPError
+    from urlparse import urlparse
     from io import open # needed for encoding
     _PY3 = False
 
@@ -91,13 +93,14 @@ class UrlCache(object):
     """
         Creates a cache for URLs.
         The file modification time is set to the Last-Modified header of the URL (if any)
-        A hidden marker file is used to record the last check time (unless useFileModTime==True)
+        If a check interval is specified (>0),
+        a hidden marker file is used to record the last check time (unless useFileModTime==True)
 
         @param cachedir: the cache directory to use 
             (default data/cache; this is assumed to be at the current directory, its parent or grandparent)
         @param interval: minimum interval between checks for updates to the URL (default 300 secs)
             if set to -1, never checks (intended for testing only)  
-            if set to 0, always checks (primarily intended for testing)
+            if set to 0, always checks (primarily intended for testing, also useful where URLs support If-Modified-Since)
         @return: the instance to use with the get() method
     """
     # get file mod_date
@@ -158,6 +161,8 @@ class UrlCache(object):
             provide a Last-Modified header
             @return: the opened stream, using the encoding if specified. Otherwise opened in binary mode. 
         """
+        if name == None:
+            name = basename(urlparse(url).path)
         target=self.__getname(name)
         fileTime = self.__file_mtime(target)
         if useFileModTime: