[GitHub] [incubator-mxnet] larroy commented on a change in pull request #15051: CD Framework + static binary release

[GitBox <gi...@apache.org>]

larroy commented on a change in pull request #15051: CD Framework + static binary release
URL: https://github.com/apache/incubator-mxnet/pull/15051#discussion_r300192360
 
 

 ##########
 File path: cd/utils/artifact_repository.py
 ##########
 @@ -0,0 +1,615 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Tool for uploading artifacts to the artifact repository
+"""
+
+__author__ = 'Per Goncalves da Silva'
+__version__ = '0.1'
+
+import argparse
+import ctypes
+import glob
+import logging
+import os
+import re
+import sys
+from itertools import chain
+from subprocess import CalledProcessError, check_output
+from typing import Dict, List, Optional
+
+import boto3
+import botocore.exceptions
+import yaml
+
+s3 = boto3.client('s3')
+logger = logging.getLogger(__name__)
+
+
+def config_logging():
+    """
+    Configures default logging settings
+    """
+    logging.root.setLevel(logging.WARNING)
+    logger.setLevel(logging.INFO)
+    logging.basicConfig(format='%(levelname)s: %(message)s')
+
+
+def s3_upload(bucket: str, s3_key_prefix: str, paths: List[str]):
+    """
+    Uploads a list of files to an S3 bucket with a particular S3 key prefix
+    :param bucket: The name of the S3 bucket
+    :param s3_key_prefix: The key prefix to apply to each of the files
+    :param paths: A list of paths to files
+    """
+    for path in paths:
+        s3_key = "{}/{}".format(s3_key_prefix, os.path.basename(path))
+        logger.info('Uploading {}'.format(path))
+        logger.debug("Uploading {} to s3://{}/{}".format(path, bucket, s3_key))
+        with open(path, 'rb') as data:
+            s3.upload_fileobj(Fileobj=data, Key=s3_key, Bucket=bucket)
+
+
+def write_libmxnet_meta(args: argparse.Namespace, destination: str):
+    """
+    Writes a file called libmxnet.meta in the 'destination' folder that contains
+    the libmxnet library information (commit id, type, etc.).
+    :param args: A Namespace object containing the library
+    :param destination: The folder in which to place the libmxnet.meta
+    """
+    with open(os.path.join(destination, 'libmxnet.meta'), 'w') as fp:
+        fp.write(yaml.dump({
+            "variant": args.variant,
+            "os": args.os,
+            "commit_id": args.git_sha,
+            "dependency_linking": args.libtype,
+        }))
+
+
+def try_s3_download(bucket, s3_key_prefix, destination) -> bool:
+    """
+    Downloads a list of files to an S3 bucket with a particular S3 key prefix to 'destination'
+    :param bucket: The name of the S3 bucket
+    :param s3_key_prefix: The key prefix to apply to each of the files
+    :param destination the path to which to download the files
+    :return False if not artifacts were found, True otherwise
+    """
+    response = s3.list_objects_v2(Bucket=bucket, Prefix=s3_key_prefix)
+    if not response:
+        raise RuntimeError('Error listing S3 objects')
+
+    if response.get('KeyCount') is None:
+        logger.debug('Invalid S3 list objects response format')
+        logger.debug(response)
+        raise RuntimeError('Invalid response format.')
+
+    key_count = response.get('KeyCount')
+    if key_count == 0:
+        logger.debug('No artifacts found')
+        return False
+
+    if not response.get('Contents'):
+        logger.debug('Invalid S3 list objects response format')
+        logger.debug(response)
+        raise RuntimeError('Invalid response format.')
+
+    for obj in response.get('Contents'):
+        key = obj['Key']
+
+        # extract file path with any subdirectories and remove the leading file separator
+        output_path = os.path.join(destination, key[len(s3_key_prefix):].lstrip(os.sep))
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        logger.info('Downloading {}'.format(output_path))
+        logger.debug("Downloading s3://{}/{} to {}".format(bucket, key, output_path))
+        with open(output_path, 'wb') as fp:
+            s3.download_fileobj(Fileobj=fp, Key=key, Bucket=bucket)
+
+    return True
+
+
+def get_commit_id_from_cmd() -> Optional[str]:
+    """
+    Returns the output of 'git rev-parse HEAD'
+    :return: A commit id, or None if the command fails
+    """
+    try:
+        logger.debug('Executing "git rev-parse HEAD"')
+        commit_id = check_output("git rev-parse HEAD".split(" ")).decode('UTF-8').strip()
+        logger.debug('Found commit id: {}'.format(commit_id))
+        return commit_id
+    except CalledProcessError as e:
+        logger.debug('Error getting commit id:')
+        logger.debug(format(e))
+        return None
+
+
+def probe_commit_id() -> str:
+    """
+    Probes the system in an attempt to ascertain the mxnet commit id
+    :return: The commit id, or None if not found
+    """
+    logger.debug('Probing for commit id')
+    commit_id = os.environ.get('MXNET_SHA')
+    if not commit_id:
+        logger.debug('MXNET_SHA environment variable not set. Trying GIT_COMMIT')
+        commit_id = os.environ.get('GIT_COMMIT')
+    if not commit_id:
+        logger.debug('GIT_COMMIT environment variable not set. Trying git command')
+        commit_id = get_commit_id_from_cmd()
+    if not commit_id:
+        logger.debug('Could not determine git commit id')
+    else:
+        logger.debug('Commit id is: {}'.format(commit_id))
+    return commit_id
+
+
+def get_linux_os_release_properties() -> Optional[Dict[str, str]]:
+    """
+    Makes a dictionary out of /etc/os-release
+    :return: A dictionary of os release properties
+    """
+    logger.debug('Extracting operating system properties from /etc/os-release')
+    if not os.path.isfile('/etc/os-release'):
+        logger.debug('Error: /etc/os-release not found')
+        return None
+
+    try:
+        with open('/etc/os-release', 'r') as fp:
+            # removes empty spaces and quotation marks from line
+            property_tuple_list = [line.strip().replace('"', '').split('=') for line in fp if line.strip()]
+            return {key: value for (key, value) in property_tuple_list}
+    except Exception as e:
+        logger.debug('Error parsing /etc/os-release')
+        logger.debug(e)
+        return None
+
+
+def get_linux_distribution_and_version() -> Optional[str]:
+    """
+    Returns the linux distribution and version by taking
+    the values of ID and VERSION_ID from /etc/os-release and
+    concatenating them. Eg. centos7, ubuntu16.04, etc.
+    :return: The linux distribution and version string, or None if not found.
+    """
+    logger.debug('Getting linux distribution and version')
+    os_properties = get_linux_os_release_properties()
+    if os_properties:
+        logger.debug('os properties: {}'.format(os_properties))
+        distribution = os_properties['ID']
+        version = os_properties['VERSION_ID']
+        return "{}{}".format(distribution, version)
+
+    logger.debug('Error getting linux distribution and version. Could not determine os properties.')
+    return None
+
+
+def probe_operating_system() -> str:
+    """
+    Probes the system to determine the operating system.
+    :return: The name of the operating system, e.g. win32, darwin, ubuntu16.04, centos7, etc.
+    """
+    logger.debug('Determining operating system')
+    operating_system = sys.platform
+    logger.debug('Found platform: {}'.format(operating_system))
+    if operating_system.startswith('linux'):
+        operating_system = get_linux_distribution_and_version()
+
+    logger.debug('Operating system is {}'.format(operating_system))
+    return operating_system
+
+
+def get_libmxnet_features(libmxnet_path: str) -> Optional[Dict[str, bool]]:
+    """
+    Returns a string -> boolean dictionary mapping feature name
+    to whether it is enabled or not
+    :param libmxnet_path: path to the libmxnet library
+    :return: dictionary of features to whether they are enabled
+    """
+    logger.debug('Getting feature dictionary from {}'.format(libmxnet_path))
+
+    class Feature(ctypes.Structure):
+        _fields_ = [("_name", ctypes.c_char_p), ("enabled", ctypes.c_bool)]
+
+        @property
+        def name(self):
+            return self._name.decode()
+
+    try:
 
 Review comment:
   Are you trying to avoid loading mxnet since you are not using directly the python binding?  Maybe add a comment why loading MXNet is not desirable.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services