You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by mx...@apache.org on 2016/12/01 12:35:23 UTC
[2/6] incubator-ariatosca git commit: ARIA-23 Add initial CSAR support
ARIA-23 Add initial CSAR support
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/d7addbc7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/d7addbc7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/d7addbc7
Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: d7addbc7f8637170e10dc9dffd09fb1dc38189f0
Parents: b54478b
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Thu Nov 17 12:43:49 2016 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Wed Nov 30 09:55:05 2016 +0200
----------------------------------------------------------------------
aria/cli/args_parser.py | 44 +++++++++++
aria/cli/cli.py | 6 ++
aria/cli/commands.py | 71 +++++++++++++++++-
aria/cli/csar.py | 171 +++++++++++++++++++++++++++++++++++++++++++
tox.ini | 1 +
5 files changed, 292 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/args_parser.py
----------------------------------------------------------------------
diff --git a/aria/cli/args_parser.py b/aria/cli/args_parser.py
index 56fd074..8eacf05 100644
--- a/aria/cli/args_parser.py
+++ b/aria/cli/args_parser.py
@@ -69,6 +69,9 @@ def config_parser(parser=None):
add_execute_parser(sub_parser)
add_parse_parser(sub_parser)
add_spec_parser(sub_parser)
+ add_csar_create_parser(sub_parser)
+ add_csar_open_parser(sub_parser)
+ add_csar_validate_parser(sub_parser)
return parser
@@ -199,3 +202,44 @@ def add_spec_parser(spec):
'--csv',
action='store_true',
help='output as CSV')
+
+
+@sub_parser_decorator(
+ name='csar-create',
+ help='Create a CSAR file from a TOSCA service template directory',
+ formatter_class=SmartFormatter)
+def add_csar_create_parser(parse):
+ parse.add_argument(
+ 'source',
+ help='Service template directory')
+ parse.add_argument(
+ 'entry',
+ help='Entry definition file relative to service template directory')
+ parse.add_argument(
+ '-d', '--destination',
+ help='Output CSAR zip destination',
+ required=True)
+
+
+@sub_parser_decorator(
+ name='csar-open',
+ help='Extracts a CSAR file to a TOSCA service template directory',
+ formatter_class=SmartFormatter)
+def add_csar_open_parser(parse):
+ parse.add_argument(
+ 'source',
+ help='CSAR file location')
+ parse.add_argument(
+ '-d', '--destination',
+ help='Output directory to extract the CSAR into',
+ required=True)
+
+
+@sub_parser_decorator(
+ name='csar-validate',
+ help='Validates a CSAR file',
+ formatter_class=SmartFormatter)
+def add_csar_validate_parser(parse):
+ parse.add_argument(
+ 'source',
+ help='CSAR file location')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/cli.py
----------------------------------------------------------------------
diff --git a/aria/cli/cli.py b/aria/cli/cli.py
index ad9784c..c5830d5 100644
--- a/aria/cli/cli.py
+++ b/aria/cli/cli.py
@@ -33,6 +33,9 @@ from .commands import (
ExecuteCommand,
ParseCommand,
SpecCommand,
+ CSARCreateCommand,
+ CSAROpenCommand,
+ CSARValidateCommand,
)
__version__ = '0.1.0'
@@ -50,6 +53,9 @@ class AriaCli(LoggerMixin):
'execute': ExecuteCommand.with_logger(base_logger=self.logger),
'parse': ParseCommand.with_logger(base_logger=self.logger),
'spec': SpecCommand.with_logger(base_logger=self.logger),
+ 'csar-create': CSARCreateCommand.with_logger(base_logger=self.logger),
+ 'csar-open': CSAROpenCommand.with_logger(base_logger=self.logger),
+ 'csar-validate': CSARValidateCommand.with_logger(base_logger=self.logger),
}
def __enter__(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/commands.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands.py b/aria/cli/commands.py
index 17a2564..3426bb0 100644
--- a/aria/cli/commands.py
+++ b/aria/cli/commands.py
@@ -21,6 +21,8 @@ import json
import os
import sys
import csv
+import shutil
+import tempfile
from glob import glob
from importlib import import_module
@@ -43,11 +45,12 @@ from ..parser.consumption import (
Inputs,
Instance
)
-from ..parser.loading import (UriLocation, URI_LOADER_PREFIXES)
+from ..parser.loading import (LiteralLocation, UriLocation, URI_LOADER_PREFIXES)
from ..utils.application import StorageManager
from ..utils.caching import cachedmethod
from ..utils.console import (puts, Colored, indent)
from ..utils.imports import (import_fullname, import_modules)
+from . import csar
from .exceptions import (
AriaCliFormatInputsError,
AriaCliYAMLInputsError,
@@ -394,3 +397,69 @@ class SpecCommand(BaseCommand):
with indent(2):
for k, v in details.iteritems():
puts('%s: %s' % (Colored.magenta(k), v))
+
+
+class BaseCSARCommand(BaseCommand):
+
+ @staticmethod
+ def _parse_and_dump(reader):
+ context = ConsumptionContext()
+ context.loading.prefixes += [os.path.join(reader.destination, 'definitions')]
+ context.presentation.location = LiteralLocation(reader.entry_definitions_yaml)
+ chain = ConsumerChain(context, (Read, Validate, Model, Instance))
+ chain.consume()
+ if context.validation.dump_issues():
+ raise RuntimeError('Validation failed')
+ dumper = chain.consumers[-1]
+ dumper.dump()
+
+ def _read(self, source, destination):
+ reader = csar.read(
+ source=source,
+ destination=destination,
+ logger=self.logger)
+ self.logger.info(
+ 'Path: {r.destination}\n'
+ 'TOSCA meta file version: {r.meta_file_version}\n'
+ 'CSAR Version: {r.csar_version}\n'
+ 'Created By: {r.created_by}\n'
+ 'Entry definitions: {r.entry_definitions}'
+ .format(r=reader))
+ self._parse_and_dump(reader)
+
+ def _validate(self, source):
+ workdir = tempfile.mkdtemp()
+ try:
+ self._read(
+ source=source,
+ destination=workdir)
+ finally:
+ shutil.rmtree(workdir, ignore_errors=True)
+
+
+class CSARCreateCommand(BaseCSARCommand):
+
+ def __call__(self, args_namespace, unknown_args):
+ super(CSARCreateCommand, self).__call__(args_namespace, unknown_args)
+ csar.write(
+ source=args_namespace.source,
+ entry=args_namespace.entry,
+ destination=args_namespace.destination,
+ logger=self.logger)
+ self._validate(args_namespace.destination)
+
+
+class CSAROpenCommand(BaseCSARCommand):
+
+ def __call__(self, args_namespace, unknown_args):
+ super(CSAROpenCommand, self).__call__(args_namespace, unknown_args)
+ self._read(
+ source=args_namespace.source,
+ destination=args_namespace.destination)
+
+
+class CSARValidateCommand(BaseCSARCommand):
+
+ def __call__(self, args_namespace, unknown_args):
+ super(CSARValidateCommand, self).__call__(args_namespace, unknown_args)
+ self._validate(args_namespace.source)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/csar.py
----------------------------------------------------------------------
diff --git a/aria/cli/csar.py b/aria/cli/csar.py
new file mode 100644
index 0000000..933df17
--- /dev/null
+++ b/aria/cli/csar.py
@@ -0,0 +1,171 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pprint
+import tempfile
+import zipfile
+
+import requests
+from ruamel import yaml
+
+
+META_FILE = 'TOSCA-Metadata/TOSCA.meta'
+META_FILE_VERSION_KEY = 'TOSCA-Meta-File-Version'
+META_FILE_VERSION_VALUE = '1.0'
+META_CSAR_VERSION_KEY = 'CSAR-Version'
+META_CSAR_VERSION_VALUE = '1.1'
+META_CREATED_BY_KEY = 'Created-By'
+META_CREATED_BY_VALUE = 'ARIA'
+META_ENTRY_DEFINITIONS_KEY = 'Entry-Definitions'
+BASE_METADATA = {
+ META_FILE_VERSION_KEY: META_FILE_VERSION_VALUE,
+ META_CSAR_VERSION_KEY: META_CSAR_VERSION_VALUE,
+ META_CREATED_BY_KEY: META_CREATED_BY_VALUE,
+}
+
+
+def write(source, entry, destination, logger):
+ source = os.path.expanduser(source)
+ destination = os.path.expanduser(destination)
+ entry_definitions = os.path.join(source, entry)
+ meta_file = os.path.join(source, META_FILE)
+ if not os.path.isdir(source):
+ raise ValueError('{0} is not a directory. Please specify the service template '
+ 'directory.'.format(source))
+ if not os.path.isfile(entry_definitions):
+ raise ValueError('{0} does not exists. Please specify a valid entry point.'
+ .format(entry_definitions))
+ if os.path.exists(destination):
+ raise ValueError('{0} already exists. Please provide a path to where the CSAR should be '
+ 'created.'.format(destination))
+ if os.path.exists(meta_file):
+ raise ValueError('{0} already exists. This commands generates a meta file for you. Please '
+ 'remove the existing metafile.'.format(meta_file))
+ metadata = BASE_METADATA.copy()
+ metadata[META_ENTRY_DEFINITIONS_KEY] = entry
+ logger.debug('Compressing root directory to ZIP')
+ with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as f:
+ for root, _, files in os.walk(source):
+ for file in files:
+ file_full_path = os.path.join(root, file)
+ file_relative_path = os.path.relpath(file_full_path, source)
+ logger.debug('Writing to archive: {0}'.format(file_relative_path))
+ f.write(file_full_path, file_relative_path)
+ logger.debug('Writing new metadata file to {0}'.format(META_FILE))
+ f.writestr(META_FILE, yaml.dump(metadata, default_flow_style=False))
+
+
+class _CSARReader(object):
+
+ def __init__(self, source, destination, logger):
+ self.logger = logger
+ if os.path.isdir(destination) and os.listdir(destination):
+ raise ValueError('{0} already exists and is not empty. '
+ 'Please specify the location where the CSAR '
+ 'should be extracted.'.format(destination))
+ downloaded_csar = '://' in source
+ if downloaded_csar:
+ file_descriptor, download_target = tempfile.mkstemp()
+ os.close(file_descriptor)
+ self._download(source, download_target)
+ source = download_target
+ self.source = os.path.expanduser(source)
+ self.destination = os.path.expanduser(destination)
+ self.metadata = {}
+ try:
+ if not os.path.exists(self.source):
+ raise ValueError('{0} does not exists. Please specify a valid CSAR path.'
+ .format(self.source))
+ if not zipfile.is_zipfile(self.source):
+ raise ValueError('{0} is not a valid CSAR.'.format(self.source))
+ self._extract()
+ self._read_metadata()
+ self._validate()
+ finally:
+ if downloaded_csar:
+ os.remove(self.source)
+
+ @property
+ def created_by(self):
+ return self.metadata.get(META_CREATED_BY_KEY)
+
+ @property
+ def csar_version(self):
+ return self.metadata.get(META_CSAR_VERSION_KEY)
+
+ @property
+ def meta_file_version(self):
+ return self.metadata.get(META_FILE_VERSION_KEY)
+
+ @property
+ def entry_definitions(self):
+ return self.metadata.get(META_ENTRY_DEFINITIONS_KEY)
+
+ @property
+ def entry_definitions_yaml(self):
+ with open(os.path.join(self.destination, self.entry_definitions)) as f:
+ return yaml.load(f)
+
+ def _extract(self):
+ self.logger.debug('Extracting CSAR contents')
+ if not os.path.exists(self.destination):
+ os.mkdir(self.destination)
+ with zipfile.ZipFile(self.source) as f:
+ f.extractall(self.destination)
+ self.logger.debug('CSAR contents successfully extracted')
+
+ def _read_metadata(self):
+ csar_metafile = os.path.join(self.destination, META_FILE)
+ if not os.path.exists(csar_metafile):
+ raise ValueError('Metadata file {0} is missing from the CSAR'.format(csar_metafile))
+ self.logger.debug('CSAR metadata file: {0}'.format(csar_metafile))
+ self.logger.debug('Attempting to parse CSAR metadata YAML')
+ with open(csar_metafile) as f:
+ self.metadata.update(yaml.load(f))
+ self.logger.debug('CSAR metadata:\n{0}'.format(pprint.pformat(self.metadata)))
+
+ def _validate(self):
+ def validate_key(key, expected=None):
+ if not self.metadata.get(key):
+ raise ValueError('{0} is missing from the metadata file.'.format(key))
+ actual = str(self.metadata[key])
+ if expected and actual != expected:
+ raise ValueError('{0} is expected to be {1} in the metadata file while it is in '
+ 'fact {2}.'.format(key, expected, actual))
+ validate_key(META_FILE_VERSION_KEY, expected=META_FILE_VERSION_VALUE)
+ validate_key(META_CSAR_VERSION_KEY, expected=META_CSAR_VERSION_VALUE)
+ validate_key(META_CREATED_BY_KEY)
+ validate_key(META_ENTRY_DEFINITIONS_KEY)
+ self.logger.debug('CSAR entry definitions: {0}'.format(self.entry_definitions))
+ entry_definitions_path = os.path.join(self.destination, self.entry_definitions)
+ if not os.path.isfile(entry_definitions_path):
+ raise ValueError('The entry definitions {0} referenced by the metadata file does not '
+ 'exist.'.format(entry_definitions_path))
+
+ def _download(self, url, target):
+ response = requests.get(url, stream=True)
+ if response.status_code != 200:
+ raise ValueError('Server at {0} returned a {1} status code'
+ .format(url, response.status_code))
+ self.logger.info('Downloading {0} to {1}'.format(url, target))
+ with open(target, 'wb') as f:
+ for chunk in response.iter_content(chunk_size=8192):
+ if chunk:
+ f.write(chunk)
+
+
+def read(source, destination, logger):
+ return _CSARReader(source=source, destination=destination, logger=logger)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/tox.ini
----------------------------------------------------------------------
diff --git a/tox.ini b/tox.ini
index 2efc329..8355b19 100644
--- a/tox.ini
+++ b/tox.ini
@@ -34,3 +34,4 @@ commands=pylint --rcfile=aria/.pylintrc --disable=fixme,missing-docstring --igno
[testenv:pylint_tests]
commands=pylint --rcfile=tests/.pylintrc --disable=fixme,missing-docstring tests
+