You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:07:14 UTC
[buildstream] 38/41: Remove tar artifact cache
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jmac/googlecas_and_virtual_directories_1
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 75c900940a1923eb40f7448062ed973c7d990367
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Fri May 11 16:39:50 2018 +0200
Remove tar artifact cache
No longer used.
---
buildstream/_artifactcache/tarcache.py | 298 ---------------------------------
tests/artifactcache/tar.py | 82 ---------
2 files changed, 380 deletions(-)
diff --git a/buildstream/_artifactcache/tarcache.py b/buildstream/_artifactcache/tarcache.py
deleted file mode 100644
index 10ae9d0..0000000
--- a/buildstream/_artifactcache/tarcache.py
+++ /dev/null
@@ -1,298 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2017 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-# Authors:
-# Tristan Maat <tr...@codethink.co.uk>
-
-import os
-import shutil
-import tarfile
-import subprocess
-
-from .. import utils, ProgramNotFoundError
-from .._exceptions import ArtifactError
-
-from . import ArtifactCache
-
-
-class TarCache(ArtifactCache):
-
- def __init__(self, context):
-
- super().__init__(context)
-
- self.tardir = os.path.join(context.artifactdir, 'tar')
- os.makedirs(self.tardir, exist_ok=True)
-
- ################################################
- # Implementation of abstract methods #
- ################################################
- def contains(self, element, key):
- path = os.path.join(self.tardir, _tarpath(element, key))
- return os.path.isfile(path)
-
- def commit(self, element, content, keys):
- os.makedirs(os.path.join(self.tardir, element._get_project().name, element.normal_name), exist_ok=True)
-
- with utils._tempdir() as temp:
- for key in keys:
- ref = _tarpath(element, key)
-
- refdir = os.path.join(temp, key)
- shutil.copytree(content, refdir, symlinks=True)
-
- _Tar.archive(os.path.join(self.tardir, ref), key, temp)
-
- def extract(self, element, key):
-
- fullname = self.get_artifact_fullname(element, key)
- path = _tarpath(element, key)
-
- # Extracting a nonexistent artifact is a bug
- assert os.path.isfile(os.path.join(self.tardir, path)), "Artifact missing for {}".format(fullname)
-
- # If the destination already exists, the artifact has been extracted
- dest = os.path.join(self.extractdir, fullname)
- if os.path.isdir(dest):
- return dest
-
- os.makedirs(self.extractdir, exist_ok=True)
-
- with utils._tempdir(dir=self.extractdir) as tmpdir:
- _Tar.extract(os.path.join(self.tardir, path), tmpdir)
-
- os.makedirs(os.path.join(self.extractdir, element._get_project().name, element.normal_name),
- exist_ok=True)
- try:
- os.rename(os.path.join(tmpdir, key), dest)
- except OSError as e:
- # With rename, it's possible to get either ENOTEMPTY or EEXIST
- # in the case that the destination path is a not empty directory.
- #
- # If rename fails with these errors, another process beat
- # us to it so just ignore.
- if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]:
- raise ArtifactError("Failed to extract artifact '{}': {}"
- .format(fullname, e)) from e
-
- return dest
-
-
-# _tarpath()
-#
-# Generate a relative tarball path for a given element and it's cache key
-#
-# Args:
-# element (Element): The Element object
-# key (str): The element's cache key
-#
-# Returns:
-# (str): The relative path to use for storing tarballs
-#
-def _tarpath(element, key):
- project = element._get_project()
- return os.path.join(project.name, element.normal_name, key + '.tar.bz2')
-
-
-# A helper class that contains tar archive/extract functions
-class _Tar():
-
- # archive()
- #
- # Attempt to archive the given tarfile with the `tar` command,
- # falling back to python's `tarfile` if this fails.
- #
- # Args:
- # location (str): The path to the tar to create
- # content (str): The path to the content to archvive
- # cwd (str): The cwd
- #
- # This is done since AIX tar does not support 2G+ files.
- #
- @classmethod
- def archive(cls, location, content, cwd=os.getcwd()):
-
- try:
- cls._archive_with_tar(location, content, cwd)
- return
- except tarfile.TarError:
- pass
- except ProgramNotFoundError:
- pass
-
- # If the former did not complete successfully, we try with
- # python's tar implementation (since it's hard to detect
- # specific issues with specific tar implementations - a
- # fallback).
-
- try:
- cls._archive_with_python(location, content, cwd)
- except tarfile.TarError as e:
- raise ArtifactError("Failed to archive {}: {}"
- .format(location, e)) from e
-
- # extract()
- #
- # Attempt to extract the given tarfile with the `tar` command,
- # falling back to python's `tarfile` if this fails.
- #
- # Args:
- # location (str): The path to the tar to extract
- # dest (str): The destination path to extract to
- #
- # This is done since python tarfile extraction is horrendously
- # slow (2 hrs+ for base images).
- #
- @classmethod
- def extract(cls, location, dest):
-
- try:
- cls._extract_with_tar(location, dest)
- return
- except tarfile.TarError:
- pass
- except ProgramNotFoundError:
- pass
-
- try:
- cls._extract_with_python(location, dest)
- except tarfile.TarError as e:
- raise ArtifactError("Failed to extract {}: {}"
- .format(location, e)) from e
-
- # _get_host_tar()
- #
- # Get the host tar command.
- #
- # Raises:
- # ProgramNotFoundError: If the tar executable cannot be
- # located
- #
- @classmethod
- def _get_host_tar(cls):
- tar_cmd = None
-
- for potential_tar_cmd in ['gtar', 'tar']:
- try:
- tar_cmd = utils.get_host_tool(potential_tar_cmd)
- break
- except ProgramNotFoundError:
- continue
-
- # If we still couldn't find tar, raise the ProgramNotfounderror
- if tar_cmd is None:
- raise ProgramNotFoundError("Did not find tar in PATH: {}"
- .format(os.environ.get('PATH')))
-
- return tar_cmd
-
- # _archive_with_tar()
- #
- # Archive with an implementation of the `tar` command
- #
- # Args:
- # location (str): The path to the tar to create
- # content (str): The path to the content to archvive
- # cwd (str): The cwd
- #
- # Raises:
- # TarError: If an error occurs during extraction
- # ProgramNotFoundError: If the tar executable cannot be
- # located
- #
- @classmethod
- def _archive_with_tar(cls, location, content, cwd):
- tar_cmd = cls._get_host_tar()
-
- process = subprocess.Popen(
- [tar_cmd, 'jcaf', location, content],
- cwd=cwd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
-
- _, err = process.communicate()
- if process.poll() != 0:
- # Clean up in case the command failed in a broken state
- try:
- os.remove(location)
- except FileNotFoundError:
- pass
-
- raise tarfile.TarError("Failed to archive '{}': {}"
- .format(content, err.decode('utf8')))
-
- # _archive_with_python()
- #
- # Archive with the python `tarfile` module
- #
- # Args:
- # location (str): The path to the tar to create
- # content (str): The path to the content to archvive
- # cwd (str): The cwd
- #
- # Raises:
- # TarError: If an error occurs during extraction
- #
- @classmethod
- def _archive_with_python(cls, location, content, cwd):
- with tarfile.open(location, mode='w:bz2') as tar:
- tar.add(os.path.join(cwd, content), arcname=content)
-
- # _extract_with_tar()
- #
- # Extract with an implementation of the `tar` command
- #
- # Args:
- # location (str): The path to the tar to extract
- # dest (str): The destination path to extract to
- #
- # Raises:
- # TarError: If an error occurs during extraction
- #
- @classmethod
- def _extract_with_tar(cls, location, dest):
- tar_cmd = cls._get_host_tar()
-
- # Some tar implementations do not support '-C'
- process = subprocess.Popen(
- [tar_cmd, 'jxf', location],
- cwd=dest,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
-
- _, err = process.communicate()
- if process.poll() != 0:
- raise tarfile.TarError("Failed to extract '{}': {}"
- .format(location, err.decode('utf8')))
-
- # _extract_with_python()
- #
- # Extract with the python `tarfile` module
- #
- # Args:
- # location (str): The path to the tar to extract
- # dest (str): The destination path to extract to
- #
- # Raises:
- # TarError: If an error occurs during extraction
- #
- @classmethod
- def _extract_with_python(cls, location, dest):
- with tarfile.open(location) as tar:
- tar.extractall(path=dest)
diff --git a/tests/artifactcache/tar.py b/tests/artifactcache/tar.py
deleted file mode 100644
index ef39be3..0000000
--- a/tests/artifactcache/tar.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import os
-import tarfile
-import tempfile
-from contextlib import ExitStack
-
-import pytest
-
-from buildstream._artifactcache.tarcache import _Tar
-from buildstream import utils, ProgramNotFoundError
-
-
-# Test that it 'works' - this may be equivalent to test_archive_no_tar()
-# on some systems.
-def test_archive_default():
- with ExitStack() as stack:
- src = stack.enter_context(tempfile.TemporaryDirectory())
- tar_dir = stack.enter_context(tempfile.TemporaryDirectory())
- scratch = stack.enter_context(tempfile.TemporaryDirectory())
- test_file = stack.enter_context(open(os.path.join(src, 'test'), 'a'))
- test_file.write('Test')
-
- _Tar.archive(os.path.join(tar_dir, 'test.tar'), '.', src)
-
- with tarfile.open(os.path.join(tar_dir, 'test.tar')) as tar:
- tar.extractall(path=scratch)
-
- assert os.listdir(scratch) == os.listdir(src)
-
-
-def test_archive_no_tar():
- # Modify the path to exclude 'tar'
- old_path = os.environ.get('PATH')
- os.environ['PATH'] = ''
-
- # Ensure we can't find 'tar' or 'gtar'
- try:
- for tar in ['gtar', 'tar']:
- with pytest.raises(ProgramNotFoundError):
- utils.get_host_tool(tar)
-
- # Run the same test as before, this time 'tar' should not be available
- test_archive_default()
-
- # Reset the environment
- finally:
- os.environ['PATH'] = old_path
-
-
-# Same thing as test_archive_default()
-def test_extract_default():
- with ExitStack() as stack:
- src = stack.enter_context(tempfile.TemporaryDirectory())
- tar_dir = stack.enter_context(tempfile.TemporaryDirectory())
- scratch = stack.enter_context(tempfile.TemporaryDirectory())
- test_file = stack.enter_context(open(os.path.join(src, 'test'), 'a'))
- test_file.write('Test')
-
- with tarfile.open(os.path.join(tar_dir, 'test.tar'), 'a:') as tar:
- tar.add(src, 'contents')
-
- _Tar.extract(os.path.join(tar_dir, 'test.tar'), scratch)
-
- assert os.listdir(os.path.join(scratch, 'contents')) == os.listdir(src)
-
-
-def test_extract_no_tar():
- # Modify the path to exclude 'tar'
- old_path = os.environ.get('PATH')
- os.environ['PATH'] = ''
-
- # Ensure we can't find 'tar' or 'gtar'
- for tar in ['gtar', 'tar']:
- with pytest.raises(ProgramNotFoundError):
- utils.get_host_tool(tar)
-
- # Run the same test as before, this time 'tar' should not be available
- try:
- test_extract_default()
-
- # Reset the environment
- finally:
- os.environ['PATH'] = old_path