You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:50:55 UTC
[buildstream] branch chandan/oci-image created (now 9d3ba20)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch chandan/oci-image
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 9d3ba20 WIP: Add OCI element
This branch includes the following new commits:
new 9d3ba20 WIP: Add OCI element
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: WIP: Add OCI element
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch chandan/oci-image
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9d3ba20409e5a1d992532ab290eb85b87f237dee
Author: Chandan Singh <cs...@bloomberg.net>
AuthorDate: Fri Aug 3 18:38:35 2018 +0100
WIP: Add OCI element
---
buildstream/plugins/elements/oci.py | 239 +++++++++++++++++++++
tests/integration/oci.py | 107 +++++++++
tests/integration/project/elements/oci/llamas.bst | 6 +
.../integration/project/elements/oci/ocihello.bst | 5 +
4 files changed, 357 insertions(+)
diff --git a/buildstream/plugins/elements/oci.py b/buildstream/plugins/elements/oci.py
new file mode 100644
index 0000000..5197b76
--- /dev/null
+++ b/buildstream/plugins/elements/oci.py
@@ -0,0 +1,239 @@
+#
+# Copyright 2018 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Chandan singh <cs...@bloomberg.net>
+
+"""
+oci - Generate OCI Image
+========================
+Generate OCI image from its dependencies.
+
+This element is normally used near the end of a pipeline to prepare an OCI
+image that can be used for later deployment.
+
+.. note::
+
+ The ``oci`` element is available since :ref:`format version XX <project_format_version>`
+
+Here is the default configuration for the ``oci`` element in full:
+ .. literalinclude:: ../../../buildstream/plugins/elements/oci.yaml
+ :language: yaml
+"""
+
+import gzip
+import hashlib
+import json
+import os
+import shutil
+import tarfile
+
+from buildstream import Element, Scope, utils
+
+OCIIMAGE_SPEC_VERSION = '1.0.0'
+
+
+################
+# Helper classes
+################
+
+class Blob():
+
+ size = None
+ digest = None
+
+ def __init__(self, basedir):
+ self.basedir = basedir
+ # FIXME consider supporting other hashing algorithms
+ self._algorithm = hashlib.sha256
+ self._algorithm_name = 'sha256'
+
+ @property
+ def path(self):
+ blobs_dir = os.path.join(self.basedir, 'blobs', self._algorithm_name)
+ os.makedirs(blobs_dir, exist_ok=True)
+ return os.path.join(blobs_dir, self.digest)
+
+ @property
+ def digest_str(self):
+ return '{}:{}'.format(self._algorithm_name, self.digest)
+
+
+class RootfsBlob(Blob):
+
+ diff_id = None
+
+ def __init__(self, basedir, inputdir):
+ super().__init__(basedir)
+ self.inputdir = inputdir
+
+ # Create uncompressed tar archive and calculate diff id
+ with tarfile.TarFile(name='files.tar', mode='w') as tar:
+ for f in os.listdir(inputdir):
+ tar.add(os.path.join(inputdir, f), arcname=f)
+ with open('files.tar', 'rb') as f:
+ self.diff_id = self._algorithm(f.read()).hexdigest()
+
+ # Now compress the tar archive and calculate layer data
+ with open('files.tar', 'rb') as raw_archive:
+ with gzip.open('files.tar.gz', 'w') as compressed_archive:
+ compressed_archive.write(raw_archive.read())
+
+ with open('files.tar.gz', 'rb') as f:
+ self.digest = self._algorithm(f.read()).hexdigest()
+ self.size = os.path.getsize('files.tar.gz')
+
+ # Move the compressed tar archive into correct directory and clean up
+ shutil.move('files.tar.gz', self.path)
+ os.remove('files.tar')
+
+ @property
+ def diff_id_str(self):
+ return '{}:{}'.format(self._algorithm_name, self.diff_id)
+
+
+class StringBlob(Blob):
+
+ def __init__(self, basedir, contents):
+ super().__init__(basedir)
+ self.contents = contents = contents.encode()
+ self.size = len(contents)
+ self.digest = self._algorithm(contents).hexdigest()
+
+ # Write the blob
+ with utils.save_file_atomic(self.path, 'wb') as f:
+ f.write(contents)
+
+
+###################
+# OCI Image Element
+###################
+
+class OCIImageElement(Element):
+
+ # The oci element's output is its dependencies, so
+ # we must rebuild if the dependencies change even when
+ # not in strict build plans.
+ BST_STRICT_REBUILD = True
+
+ # OCI artifacts must never have indirect dependencies,
+ # so runtime dependencies are forbidden.
+ BST_FORBID_RDEPENDS = True
+
+ # This element ignores sources, so we should forbid them from being
+ # added, to reduce the potential for confusion
+ BST_FORBID_SOURCES = True
+
+ def configure(self, node):
+ # We don't need anything, yet...
+ self.node_validate(node, [])
+
+ def preflight(self):
+ # All good!
+ pass
+
+ def get_unique_key(self):
+ # All good! We don't need to rebuild if our dependencies haven't
+ # changed
+ return 1
+
+ def configure_sandbox(self, sandbox):
+ pass
+
+ def stage(self, sandbox):
+ pass
+
+ def assemble(self, sandbox):
+ basedir = sandbox.get_directory()
+ inputdir = os.path.join(basedir, 'input')
+ outputdir = os.path.join(basedir, 'output')
+ os.makedirs(inputdir, exist_ok=True)
+ os.makedirs(outputdir, exist_ok=True)
+
+ # Stage deps in the sandbox root
+ with self.timed_activity("Staging dependencies", silent_nested=True):
+ self.stage_dependency_artifacts(sandbox, Scope.BUILD, path='/input')
+
+ with self.timed_activity("Creating OCI image bundle", silent_nested=True):
+ # Generate oci-layout
+ with utils.save_file_atomic(os.path.join(outputdir, 'oci-layout'), 'w') as f:
+ f.write(json.dumps(self._oci_layout()))
+
+ # Generate blobs
+ # 1. rootfs
+ rootfs = RootfsBlob(outputdir, inputdir)
+ # 2. config
+ config_str = json.dumps(self._config(rootfs))
+ config = StringBlob(outputdir, config_str)
+ # 3. manifest
+ manifest_str = json.dumps(self._manifest(config, rootfs))
+ manifest = StringBlob(outputdir, manifest_str)
+
+ # Generate index.json
+ with utils.save_file_atomic(os.path.join(outputdir, 'index.json'), 'w') as f:
+ f.write(json.dumps(self._image_index(manifest)))
+
+ return '/output'
+
+ def _image_index(self, manifest):
+ index = {
+ 'schemaVersion': 2,
+ 'manifests': [{
+ 'mediaType': 'application/vnd.oci.image.manifest.v1+json',
+ 'size': manifest.size,
+ 'digest': manifest.digest_str
+ }],
+ }
+ if self._annotations():
+ index['annotations'] = self._annotations()
+ return index
+
+ def _oci_layout(self):
+ return {
+ 'imageLayoutVersion': OCIIMAGE_SPEC_VERSION,
+ }
+
+ def _manifest(self, config, rootfs):
+ return {
+ 'schemaVersion': 2,
+ 'config': {
+ 'mediaType': 'application/vnd.oci.image.config.v1+json',
+ 'digest': config.digest_str,
+ 'size': config.size
+ },
+ 'layers': [{
+ 'mediaType': 'application/vnd.oci.image.layer.v1.tar+gzip',
+ 'digest': rootfs.digest_str,
+ 'size': rootfs.size
+ }]
+ }
+
+ def _annotations(self):
+ return []
+
+ def _config(self, rootfs):
+ return {
+ 'architecture': 'amd64',
+ 'os': 'linux',
+ 'rootfs': {
+ 'type': 'layers',
+ 'diff_ids': [rootfs.diff_id_str]
+ }
+ }
+
+
+def setup():
+ return OCIImageElement
diff --git a/tests/integration/oci.py b/tests/integration/oci.py
new file mode 100644
index 0000000..af8e2f0
--- /dev/null
+++ b/tests/integration/oci.py
@@ -0,0 +1,107 @@
+import hashlib
+import json
+import os
+import pytest
+import tarfile
+
+from tests.testutils import cli_integration as cli
+from tests.testutils.integration import assert_contains
+
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ "project"
+)
+
+
+# Test that a oci build 'works'
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_oci_build(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ checkout = os.path.join(cli.directory, 'checkout')
+ element_name = 'oci/ocihello.bst'
+
+ result = cli.run(project=project, args=['build', element_name])
+ assert result.exit_code == 0
+
+ result = cli.run(project=project, args=['checkout', element_name, checkout])
+ assert result.exit_code == 0
+
+ # Verify basic directory structure
+ assert_contains(checkout, ['/oci-layout', '/index.json', '/blobs'])
+
+ # Verify that we have at least one manifest
+ with open(os.path.join(checkout, 'index.json')) as f:
+ index = json.load(f)
+ manifests = [x for x in index['manifests']
+ if x['mediaType'] == 'application/vnd.oci.image.manifest.v1+json']
+ assert len(manifests) > 0
+
+ # Now verify that the manifests are valid
+ blobs_dir = os.path.join(checkout, 'blobs')
+ all_layers = []
+ all_diff_ids = []
+ for manifest in manifests:
+ layers, diff_ids = extract_layers(manifest, blobs_dir)
+ all_layers += layers
+ all_diff_ids += diff_ids
+ assert len(all_layers) == len(all_diff_ids)
+
+ # Finally, extract all layers and ensure that only the desired file are
+ # present
+ extract_dir = os.path.join(cli.directory, 'extract')
+ for layer in all_layers:
+ with tarfile.open(layer) as f:
+ f.extractall(path=extract_dir)
+
+ assert_contains(extract_dir, ['/subdir', '/subdir/test.txt', '/test.txt'])
+
+
+# Extract layers from given manifest and verify manifests in the process
+def extract_layers(short_manifest, blobs_dir):
+ manifest_path = get_blob(short_manifest['digest'], short_manifest['size'], blobs_dir)
+
+ with open(manifest_path) as f:
+ manifest = json.load(f)
+
+ # Assert we have both 'config' and 'layers' sections
+ assert 'config' in manifest
+ assert 'layers' in manifest
+
+ # Verify basic layout
+ assert manifest['config']['mediaType'] == 'application/vnd.oci.image.config.v1+json'
+ assert len(manifest['layers']) > 0
+ for layer in manifest['layers']:
+ assert layer['mediaType'] == 'application/vnd.oci.image.layer.v1.tar+gzip'
+
+ config_path = get_blob(manifest['config']['digest'],
+ manifest['config']['size'], blobs_dir)
+ layers_path = [get_blob(layer['digest'], layer['size'], blobs_dir)
+ for layer in manifest['layers']]
+
+ with open(config_path) as f:
+ config = json.load(f)
+
+ assert len(config['rootfs']['diff_ids']) == len(manifest['layers'])
+ return layers_path, config['rootfs']['diff_ids']
+
+
+# Get path to the blob pointed by given digest
+def get_blob(digest_str, size, blobs_dir):
+ algorigthm, digest = digest_str.strip().split(':')
+ # We only support sha256 at present
+ assert algorigthm == 'sha256'
+
+ # Verify that our digest points to a vaild blob and that its attributes
+ # match what we were given
+ blob_path = os.path.join(blobs_dir, algorigthm, digest)
+ assert os.path.isfile(blob_path)
+ assert os.path.getsize(blob_path) == size
+ with open(blob_path, 'rb') as f:
+ assert hashlib.sha256(f.read()).hexdigest() == digest
+
+ return blob_path
diff --git a/tests/integration/project/elements/oci/llamas.bst b/tests/integration/project/elements/oci/llamas.bst
new file mode 100644
index 0000000..aca0922
--- /dev/null
+++ b/tests/integration/project/elements/oci/llamas.bst
@@ -0,0 +1,6 @@
+kind: import
+description: This is a dumb import element, which is here so that we have something to put in the OCI image
+
+sources:
+- kind: local
+ path: files/import-source
diff --git a/tests/integration/project/elements/oci/ocihello.bst b/tests/integration/project/elements/oci/ocihello.bst
new file mode 100644
index 0000000..f7312d2
--- /dev/null
+++ b/tests/integration/project/elements/oci/ocihello.bst
@@ -0,0 +1,5 @@
+kind: oci
+
+depends:
+- filename: oci/llamas.bst
+ type: build