You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:28:06 UTC
[buildstream] branch juerg/tar-tracking created (now c4ea15f)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch juerg/tar-tracking
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at c4ea15f _downloadablefilesource.py: Support version tracking
This branch includes the following new commits:
new c4ea15f _downloadablefilesource.py: Support version tracking
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: _downloadablefilesource.py: Support version
tracking
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch juerg/tar-tracking
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c4ea15f7caa1d48efbd2e5383339263671c9444d
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Mon Feb 26 06:47:27 2018 +0100
_downloadablefilesource.py: Support version tracking
---
.../plugins/sources/_downloadablefilesource.py | 69 ++++++++++++++++++----
buildstream/plugins/sources/tar.py | 4 +-
2 files changed, 61 insertions(+), 12 deletions(-)
diff --git a/buildstream/plugins/sources/_downloadablefilesource.py b/buildstream/plugins/sources/_downloadablefilesource.py
index dba2a73..c7a6229 100644
--- a/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/buildstream/plugins/sources/_downloadablefilesource.py
@@ -4,30 +4,46 @@ import os
import urllib.request
import urllib.error
import contextlib
+import re
import shutil
+from distutils.version import LooseVersion
+
from buildstream import Source, SourceError, Consistency
from buildstream import utils
class DownloadableFileSource(Source):
- COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
+ COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag', 'version']
def configure(self, node):
self.original_url = self.node_get_member(node, str, 'url')
+ self.version = self.node_get_member(node, str, 'version', '') or None
self.ref = self.node_get_member(node, str, 'ref', '') or None
self.etag = self.node_get_member(node, str, 'etag', '') or None
self.url = self.translate_url(self.original_url)
+ if '{version}' in os.path.basename(self.url):
+ self.url_format = self.url
+ if self.version:
+ self.alias_url = self.original_url.format(version=self.version)
+ self.url = self.url_format.format(version=self.version)
+ else:
+ self.alias_url = None
+ self.url = None
+ else:
+ self.alias_url = self.original_url
+ self.url_format = None
+
def preflight(self):
return
def get_unique_key(self):
- return [self.original_url, self.ref]
+ return [self.alias_url, self.ref]
def get_consistency(self):
- if self.ref is None:
+ if self.alias_url is None or self.ref is None:
return Consistency.INCONSISTENT
if os.path.isfile(self._get_mirror_file()):
@@ -37,31 +53,64 @@ class DownloadableFileSource(Source):
return Consistency.RESOLVED
def get_ref(self):
- return (self.ref, self.etag)
+ return (self.ref, self.etag, self.version)
def set_ref(self, ref, node):
- self.ref, self.etag = ref
+ self.ref, self.etag, self.version = ref
- node['ref'] = self.ref
+ if self.version:
+ node['version'] = self.version
if self.etag:
node['etag'] = self.etag
+ node['ref'] = self.ref
def track(self):
# there is no 'track' field in the source to determine what/whether
# or not to update refs, because tracking a ref is always a conscious
# decision by the user.
- with self.timed_activity("Tracking {}".format(self.url),
+ with self.timed_activity("Tracking {}".format(self.url_format or self.url),
silent_nested=True):
+ if self.url_format:
+ try:
+ filename_format = os.path.basename(self.url_format)
+ index = filename_format.find('{version}')
+ escaped_prefix = re.escape('href="' + filename_format[:index])
+ escaped_suffix = re.escape(filename_format[index + len('{version}'):] + '"')
+ pattern = re.compile(escaped_prefix + r'([0-9.]+)' + escaped_suffix)
+ request = urllib.request.Request(self.url_format[:-len(filename_format)])
+ with contextlib.closing(urllib.request.urlopen(request)) as response:
+ info = response.info()
+ charset = info.get_content_charset()
+ listing = response.read().decode(charset or 'utf-8')
+
+ new_version = None
+ for match in re.findall(pattern, listing):
+ if new_version is None or LooseVersion(match) > LooseVersion(new_version):
+ new_version = match
+
+ except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError) as e:
+ raise SourceError("{}: Error tracking {}: {}"
+ .format(self, self.url_format, e)) from e
+
+ if new_version is None:
+ raise SourceError("{}: Error tracking {}: Pattern not found"
+ .format(self, self.url_format))
+
+ self.alias_url = self.original_url.format(version=new_version)
+ self.url = self.url_format.format(version=new_version)
+ else:
+ new_version = None
+
new_ref, new_etag = self._ensure_mirror()
- if self.ref and self.ref != new_ref:
+ if self.ref and self.ref != new_ref and self.version == new_version:
detail = "When tracking, new ref differs from current ref:\n" \
+ " Tracked URL: {}\n".format(self.url) \
+ " Current ref: {}\n".format(self.ref) \
+ " New ref: {}\n".format(new_ref)
self.warn("Potential man-in-the-middle attack!", detail=detail)
- return (new_ref, new_etag)
+ return (new_ref, new_etag, new_version)
def fetch(self):
@@ -125,7 +174,7 @@ class DownloadableFileSource(Source):
def _get_mirror_dir(self):
return os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url))
+ utils.url_directory_name(self.alias_url))
def _get_mirror_file(self, sha=None):
return os.path.join(self._get_mirror_dir(), sha or self.ref)
diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py
index 284554f..3f020ae 100644
--- a/buildstream/plugins/sources/tar.py
+++ b/buildstream/plugins/sources/tar.py
@@ -71,7 +71,7 @@ class TarSource(DownloadableFileSource):
def preflight(self):
self.host_lzip = None
- if self.url.endswith('.lz'):
+ if self.original_url.endswith('.lz'):
self.host_lzip = utils.get_host_tool('lzip')
def get_unique_key(self):
@@ -92,7 +92,7 @@ class TarSource(DownloadableFileSource):
@contextmanager
def _get_tar(self):
- if self.url.endswith('.lz'):
+ if self.original_url.endswith('.lz'):
with self._run_lzip() as lzip_dec:
with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
yield tar