You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:45:17 UTC
[buildstream] 01/01: WIP: Add OCI element
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch chandan/oci-image
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9d3ba20409e5a1d992532ab290eb85b87f237dee
Author: Chandan Singh <cs...@bloomberg.net>
AuthorDate: Fri Aug 3 18:38:35 2018 +0100
WIP: Add OCI element
---
buildstream/plugins/elements/oci.py | 239 +++++++++++++++++++++
tests/integration/oci.py | 107 +++++++++
tests/integration/project/elements/oci/llamas.bst | 6 +
.../integration/project/elements/oci/ocihello.bst | 5 +
4 files changed, 357 insertions(+)
diff --git a/buildstream/plugins/elements/oci.py b/buildstream/plugins/elements/oci.py
new file mode 100644
index 0000000..5197b76
--- /dev/null
+++ b/buildstream/plugins/elements/oci.py
@@ -0,0 +1,239 @@
+#
+# Copyright 2018 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Chandan singh <cs...@bloomberg.net>
+
+"""
+oci - Generate OCI Image
+========================
+Generate OCI image from its dependencies.
+
+This element is normally used near the end of a pipeline to prepare an OCI
+image that can be used for later deployment.
+
+.. note::
+
+ The ``oci`` element is available since :ref:`format version XX <project_format_version>`
+
+Here is the default configuration for the ``oci`` element in full:
+ .. literalinclude:: ../../../buildstream/plugins/elements/oci.yaml
+ :language: yaml
+"""
+
+import gzip
+import hashlib
+import json
+import os
+import shutil
+import tarfile
+
+from buildstream import Element, Scope, utils
+
+OCIIMAGE_SPEC_VERSION = '1.0.0'
+
+
+################
+# Helper classes
+################
+
+class Blob():
+
+ size = None
+ digest = None
+
+ def __init__(self, basedir):
+ self.basedir = basedir
+ # FIXME consider supporting other hashing algorithms
+ self._algorithm = hashlib.sha256
+ self._algorithm_name = 'sha256'
+
+ @property
+ def path(self):
+ blobs_dir = os.path.join(self.basedir, 'blobs', self._algorithm_name)
+ os.makedirs(blobs_dir, exist_ok=True)
+ return os.path.join(blobs_dir, self.digest)
+
+ @property
+ def digest_str(self):
+ return '{}:{}'.format(self._algorithm_name, self.digest)
+
+
+class RootfsBlob(Blob):
+
+ diff_id = None
+
+ def __init__(self, basedir, inputdir):
+ super().__init__(basedir)
+ self.inputdir = inputdir
+
+ # Create uncompressed tar archive and calculate diff id
+ with tarfile.TarFile(name='files.tar', mode='w') as tar:
+ for f in os.listdir(inputdir):
+ tar.add(os.path.join(inputdir, f), arcname=f)
+ with open('files.tar', 'rb') as f:
+ self.diff_id = self._algorithm(f.read()).hexdigest()
+
+ # Now compress the tar archive and calculate layer data
+ with open('files.tar', 'rb') as raw_archive:
+ with gzip.open('files.tar.gz', 'w') as compressed_archive:
+ compressed_archive.write(raw_archive.read())
+
+ with open('files.tar.gz', 'rb') as f:
+ self.digest = self._algorithm(f.read()).hexdigest()
+ self.size = os.path.getsize('files.tar.gz')
+
+ # Move the compressed tar archive into correct directory and clean up
+ shutil.move('files.tar.gz', self.path)
+ os.remove('files.tar')
+
+ @property
+ def diff_id_str(self):
+ return '{}:{}'.format(self._algorithm_name, self.diff_id)
+
+
+class StringBlob(Blob):
+
+ def __init__(self, basedir, contents):
+ super().__init__(basedir)
+ self.contents = contents = contents.encode()
+ self.size = len(contents)
+ self.digest = self._algorithm(contents).hexdigest()
+
+ # Write the blob
+ with utils.save_file_atomic(self.path, 'wb') as f:
+ f.write(contents)
+
+
+###################
+# OCI Image Element
+###################
+
+class OCIImageElement(Element):
+
+ # The oci element's output is its dependencies, so
+ # we must rebuild if the dependencies change even when
+ # not in strict build plans.
+ BST_STRICT_REBUILD = True
+
+ # OCI artifacts must never have indirect dependencies,
+ # so runtime dependencies are forbidden.
+ BST_FORBID_RDEPENDS = True
+
+ # This element ignores sources, so we should forbid them from being
+ # added, to reduce the potential for confusion
+ BST_FORBID_SOURCES = True
+
+ def configure(self, node):
+ # We don't need anything, yet...
+ self.node_validate(node, [])
+
+ def preflight(self):
+ # All good!
+ pass
+
+ def get_unique_key(self):
+ # All good! We don't need to rebuild if our dependencies haven't
+ # changed
+ return 1
+
+ def configure_sandbox(self, sandbox):
+ pass
+
+ def stage(self, sandbox):
+ pass
+
+ def assemble(self, sandbox):
+ basedir = sandbox.get_directory()
+ inputdir = os.path.join(basedir, 'input')
+ outputdir = os.path.join(basedir, 'output')
+ os.makedirs(inputdir, exist_ok=True)
+ os.makedirs(outputdir, exist_ok=True)
+
+ # Stage deps in the sandbox root
+ with self.timed_activity("Staging dependencies", silent_nested=True):
+ self.stage_dependency_artifacts(sandbox, Scope.BUILD, path='/input')
+
+ with self.timed_activity("Creating OCI image bundle", silent_nested=True):
+ # Generate oci-layout
+ with utils.save_file_atomic(os.path.join(outputdir, 'oci-layout'), 'w') as f:
+ f.write(json.dumps(self._oci_layout()))
+
+ # Generate blobs
+ # 1. rootfs
+ rootfs = RootfsBlob(outputdir, inputdir)
+ # 2. config
+ config_str = json.dumps(self._config(rootfs))
+ config = StringBlob(outputdir, config_str)
+ # 3. manifest
+ manifest_str = json.dumps(self._manifest(config, rootfs))
+ manifest = StringBlob(outputdir, manifest_str)
+
+ # Generate index.json
+ with utils.save_file_atomic(os.path.join(outputdir, 'index.json'), 'w') as f:
+ f.write(json.dumps(self._image_index(manifest)))
+
+ return '/output'
+
+ def _image_index(self, manifest):
+ index = {
+ 'schemaVersion': 2,
+ 'manifests': [{
+ 'mediaType': 'application/vnd.oci.image.manifest.v1+json',
+ 'size': manifest.size,
+ 'digest': manifest.digest_str
+ }],
+ }
+ if self._annotations():
+ index['annotations'] = self._annotations()
+ return index
+
+ def _oci_layout(self):
+ return {
+ 'imageLayoutVersion': OCIIMAGE_SPEC_VERSION,
+ }
+
+ def _manifest(self, config, rootfs):
+ return {
+ 'schemaVersion': 2,
+ 'config': {
+ 'mediaType': 'application/vnd.oci.image.config.v1+json',
+ 'digest': config.digest_str,
+ 'size': config.size
+ },
+ 'layers': [{
+ 'mediaType': 'application/vnd.oci.image.layer.v1.tar+gzip',
+ 'digest': rootfs.digest_str,
+ 'size': rootfs.size
+ }]
+ }
+
+ def _annotations(self):
+ return []
+
+ def _config(self, rootfs):
+ return {
+ 'architecture': 'amd64',
+ 'os': 'linux',
+ 'rootfs': {
+ 'type': 'layers',
+ 'diff_ids': [rootfs.diff_id_str]
+ }
+ }
+
+
+def setup():
+ return OCIImageElement
diff --git a/tests/integration/oci.py b/tests/integration/oci.py
new file mode 100644
index 0000000..af8e2f0
--- /dev/null
+++ b/tests/integration/oci.py
@@ -0,0 +1,107 @@
+import hashlib
+import json
+import os
+import pytest
+import tarfile
+
+from tests.testutils import cli_integration as cli
+from tests.testutils.integration import assert_contains
+
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ "project"
+)
+
+
+# Test that a oci build 'works'
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_oci_build(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ checkout = os.path.join(cli.directory, 'checkout')
+ element_name = 'oci/ocihello.bst'
+
+ result = cli.run(project=project, args=['build', element_name])
+ assert result.exit_code == 0
+
+ result = cli.run(project=project, args=['checkout', element_name, checkout])
+ assert result.exit_code == 0
+
+ # Verify basic directory structure
+ assert_contains(checkout, ['/oci-layout', '/index.json', '/blobs'])
+
+ # Verify that we have at least one manifest
+ with open(os.path.join(checkout, 'index.json')) as f:
+ index = json.load(f)
+ manifests = [x for x in index['manifests']
+ if x['mediaType'] == 'application/vnd.oci.image.manifest.v1+json']
+ assert len(manifests) > 0
+
+ # Now verify that the manifests are valid
+ blobs_dir = os.path.join(checkout, 'blobs')
+ all_layers = []
+ all_diff_ids = []
+ for manifest in manifests:
+ layers, diff_ids = extract_layers(manifest, blobs_dir)
+ all_layers += layers
+ all_diff_ids += diff_ids
+ assert len(all_layers) == len(all_diff_ids)
+
+ # Finally, extract all layers and ensure that only the desired file are
+ # present
+ extract_dir = os.path.join(cli.directory, 'extract')
+ for layer in all_layers:
+ with tarfile.open(layer) as f:
+ f.extractall(path=extract_dir)
+
+ assert_contains(extract_dir, ['/subdir', '/subdir/test.txt', '/test.txt'])
+
+
+# Extract layers from given manifest and verify manifests in the process
+def extract_layers(short_manifest, blobs_dir):
+ manifest_path = get_blob(short_manifest['digest'], short_manifest['size'], blobs_dir)
+
+ with open(manifest_path) as f:
+ manifest = json.load(f)
+
+ # Assert we have both 'config' and 'layers' sections
+ assert 'config' in manifest
+ assert 'layers' in manifest
+
+ # Verify basic layout
+ assert manifest['config']['mediaType'] == 'application/vnd.oci.image.config.v1+json'
+ assert len(manifest['layers']) > 0
+ for layer in manifest['layers']:
+ assert layer['mediaType'] == 'application/vnd.oci.image.layer.v1.tar+gzip'
+
+ config_path = get_blob(manifest['config']['digest'],
+ manifest['config']['size'], blobs_dir)
+ layers_path = [get_blob(layer['digest'], layer['size'], blobs_dir)
+ for layer in manifest['layers']]
+
+ with open(config_path) as f:
+ config = json.load(f)
+
+ assert len(config['rootfs']['diff_ids']) == len(manifest['layers'])
+ return layers_path, config['rootfs']['diff_ids']
+
+
+# Get path to the blob pointed by given digest
+def get_blob(digest_str, size, blobs_dir):
+ algorigthm, digest = digest_str.strip().split(':')
+ # We only support sha256 at present
+ assert algorigthm == 'sha256'
+
+ # Verify that our digest points to a vaild blob and that its attributes
+ # match what we were given
+ blob_path = os.path.join(blobs_dir, algorigthm, digest)
+ assert os.path.isfile(blob_path)
+ assert os.path.getsize(blob_path) == size
+ with open(blob_path, 'rb') as f:
+ assert hashlib.sha256(f.read()).hexdigest() == digest
+
+ return blob_path
diff --git a/tests/integration/project/elements/oci/llamas.bst b/tests/integration/project/elements/oci/llamas.bst
new file mode 100644
index 0000000..aca0922
--- /dev/null
+++ b/tests/integration/project/elements/oci/llamas.bst
@@ -0,0 +1,6 @@
+kind: import
+description: This is a dumb import element, which is here so that we have something to put in the OCI image
+
+sources:
+- kind: local
+ path: files/import-source
diff --git a/tests/integration/project/elements/oci/ocihello.bst b/tests/integration/project/elements/oci/ocihello.bst
new file mode 100644
index 0000000..f7312d2
--- /dev/null
+++ b/tests/integration/project/elements/oci/ocihello.bst
@@ -0,0 +1,5 @@
+kind: oci
+
+depends:
+- filename: oci/llamas.bst
+ type: build