You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:16:41 UTC
[buildstream] 02/04: Add dep_transform example
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gokcen/source_transform
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ddcc06456b7814afe7efcdca37f151c39b5a072e
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Mon Jun 18 11:43:07 2018 +0100
Add dep_transform example
---
buildstream/plugins/sources/dep_transform.py | 192 +++++++++++++++++++++++++++
1 file changed, 192 insertions(+)
diff --git a/buildstream/plugins/sources/dep_transform.py b/buildstream/plugins/sources/dep_transform.py
new file mode 100644
index 0000000..f0d0250
--- /dev/null
+++ b/buildstream/plugins/sources/dep_transform.py
@@ -0,0 +1,192 @@
+#
+# Copyright Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Chandan Singh <cs...@bloomberg.net>
+# Gokcen Nurlu <gn...@bloomberg.net>
+"""A SourceTransform implementation for staging Go dependencies via `dep`
+
+**Usage**
+
+.. code: yaml
+ # Specify the godep source kind
+ kind: dep_transform
+
+ # Optionally specify a relative staging directory
+ # directory: path/to/stage
+
+ # Specify the lock file as the ref.
+ # If no ref is provided, the new ref will be the contents of the lock file
+ # generated by running `dep ensure -no-vendor`.
+ ref: "<lock_file content>"
+..
+"""
+
+import errno
+import os
+import shutil
+from hashlib import md5
+
+from buildstream import SourceTransform, Source, SourceError
+from buildstream import utils
+from buildstream import Consistency
+
+GOPKG_LOCK_FILE = 'Gopkg.lock'
+GOPKG_VENDOR_DIR = 'vendor'
+
+
+class DepTransform(SourceTransform):
+ # pylint: disable=attribute-defined-outside-init
+
+ def configure(self, node):
+ self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
+ self.ref = self.node_get_member(node, str, 'ref', '').strip()
+
+ @property
+ def mirror(self):
+ path = os.path.join(
+ self.get_mirror_directory(),
+ self.name,
+ md5(self.ref.encode('utf-8')).hexdigest()
+ )
+ os.makedirs(path, exist_ok=True)
+ return path
+
+ def preflight(self):
+ # Check if dep is installed, get the binaries at the same time
+ self.host_dep = utils.get_host_tool('dep')
+
+ def get_unique_key(self):
+ # TODO: This plugin's inputs are actually previous source. What should
+ # we do here?
+ return (self.ref,)
+
+ def get_consistency(self):
+ if self.ref == '':
+ return Consistency.INCONSISTENT
+ for dest in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR):
+ dest_path = os.path.join(self.mirror, dest)
+ if not os.path.exists(dest_path):
+ return Consistency.RESOLVED
+ lock_file_path = os.path.join(self.mirror, GOPKG_LOCK_FILE)
+ with open(lock_file_path, encoding='utf-8') as lock_file:
+ if lock_file.read().strip() == self.ref:
+ return Consistency.CACHED
+ return Consistency.RESOLVED
+
+ def get_ref(self):
+ return self.ref
+
+ def set_ref(self, ref, node):
+ self.ref = node['ref'] = ref
+
+ def track(self, previous_staging_dir):
+ with self.timed_activity('Tracking DepTransform source based on previous sources'):
+ with self.tempdir() as goroot:
+ # dep refuses to work on sources that are not under
+ # GOPATH/src so we need to artificially create that directory
+ # stucture.
+ go_sources_root = os.path.join(goroot, 'src', 'project')
+ os.makedirs(os.path.dirname(go_sources_root), exist_ok=True)
+
+ shutil.move(previous_staging_dir, go_sources_root)
+ gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE)
+ # Check if the repo has a lock file already
+ if not os.path.isfile(gopkg_lock_path):
+ # If doesn't, let's create a new lock file
+ self.call([self.host_dep, 'ensure', '-no-vendor'],
+ cwd=go_sources_root,
+ env=dict(os.environ, GOPATH=goroot),
+ fail='Failed to update the go dep lock file with the new references')
+
+ with open(gopkg_lock_path, encoding='utf-8') as lock_file:
+ return lock_file.read()
+
+ def fetch(self, previous_staging_dir):
+ with self.timed_activity('Fetching DepTransform source dependencies based on Gopkg.lock'):
+ with self.tempdir() as goroot:
+ # dep refuses to work on sources that are not under
+ # GOPATH/src so we need to artificially create that directory
+ # stucture.
+ go_sources_root = os.path.join(goroot, 'src', 'project')
+ os.makedirs(os.path.dirname(go_sources_root), exist_ok=True)
+
+ shutil.move(previous_staging_dir, go_sources_root)
+
+ gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE)
+
+ def write_lock_file():
+ with open(gopkg_lock_path, mode='w', encoding='utf-8') as lock_file:
+ lock_file.write(self.get_ref())
+
+ # Is there a lock file present?
+ if os.path.isfile(gopkg_lock_path):
+ # There is! We should ensure it is the same as the ref
+ with open(gopkg_lock_path, encoding='utf-8') as lock_file:
+ stripped_lock_file = lock_file.read().strip()
+ # The current yaml parser strips this already, but let's not rely on it to do so
+ stripped_track_ref = self.get_ref().strip()
+
+ if stripped_lock_file != stripped_track_ref:
+ self.warn("Tracking ref and lock file in source differ. Using tracking ref.")
+ write_lock_file()
+ else:
+ write_lock_file()
+
+ self.call([self.host_dep, 'ensure', '-vendor-only'],
+ cwd=go_sources_root,
+ env=dict(os.environ, GOPATH=goroot),
+ fail='Failed to populate the vendor directory from the lock file')
+
+ # Copy the stuff that we really need
+ for source in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR):
+ source_path = os.path.join(go_sources_root, source)
+ dest_path = os.path.join(self.mirror, source)
+ try:
+ os.replace(source_path, dest_path)
+ except OSError as e:
+ # To avoid race condition between two concurrently
+ # running `fetch()` processes.
+ if e.errno not in (errno.ENOTEMPTY, errno.EEXIST):
+ raise
+ self.warn(
+ "{} seems already fetched. ".format(dest_path) +
+ "Continuing with the existing source."
+ )
+
+ def stage(self, directory):
+ with self.timed_activity("Staging DepTransform source from based on Gopkg.lock", silent_nested=True):
+ shutil.copy(os.path.join(self.mirror, GOPKG_LOCK_FILE),
+ os.path.join(directory, GOPKG_LOCK_FILE))
+ # FIXME: Should we ever need to support one, this may
+ # break on platforms that don't support symlinks if
+ # a project contains broken symlinks.
+ target_vendor_dir = os.path.join(directory, GOPKG_VENDOR_DIR)
+ try:
+ shutil.copytree(os.path.join(self.mirror, GOPKG_VENDOR_DIR),
+ target_vendor_dir,
+ symlinks=True)
+ except FileExistsError:
+ raise SourceError("{}: Unable to stage vendor directory because it already exists here '{}'"
+ .format(self, target_vendor_dir))
+ except shutil.Error as err:
+ raise SourceError("{}: Unable to stage vendor directory at '{}'"
+ .format(self, target_vendor_dir)) from err
+
+
+# Plugin entry point
+def setup():
+ return DepTransform