You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/04/10 15:41:56 UTC
[incubator-dlab] branch EPMCDLAB-1186 updated: [DLAB-249]: Added
repos configuration for Dataengine-service
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch EPMCDLAB-1186
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/EPMCDLAB-1186 by this push:
new 5041bec [DLAB-249]: Added repos configuration for Dataengine-service
5041bec is described below
commit 5041bec2bbd28bde99ff0139480aacc07c3c7b96
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Wed Apr 10 18:41:47 2019 +0300
[DLAB-249]: Added repos configuration for Dataengine-service
---
infrastructure-provisioning/scripts/deploy_dlab.py | 4 ++++
.../scripts/deploy_repository/deploy_repository.py | 2 ++
.../templates/configureNexus.groovy | 2 ++
.../dataengine-service/templates/repository.repo | 17 +++++++++++++++++
.../src/general/conf/dlab.ini | 4 ++++
.../files/aws/dataengine-service_Dockerfile | 4 ++--
.../src/general/lib/aws/actions_lib.py | 22 +++++++++++++++++++++-
.../scripts/aws/dataengine-service_configure.py | 5 ++++-
.../src/general/scripts/aws/ssn_configure.py | 3 ++-
9 files changed, 58 insertions(+), 5 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_dlab.py b/infrastructure-provisioning/scripts/deploy_dlab.py
index fb15730..9718a38 100644
--- a/infrastructure-provisioning/scripts/deploy_dlab.py
+++ b/infrastructure-provisioning/scripts/deploy_dlab.py
@@ -164,6 +164,10 @@ parser.add_argument('--local_repository_r_repo', type=str, default='',
help='Repository name for R')
parser.add_argument('--local_repository_rrutter_repo', type=str, default='',
help='Repository name for Rrutter')
+parser.add_argument('--local_repository_amazon_main_repo', type=str, default='',
+ help='Repository name for Amazon main')
+parser.add_argument('--local_repository_amazon_updates_repo', type=str, default='',
+ help='Repository name for Amazon updates')
parser.add_argument('--tags', type=str, default='Operation,ItemDescription', help='Column name in report file that '
'contains tags')
parser.add_argument('--action', required=True, type=str, default='', choices=['build', 'deploy', 'create', 'terminate'],
diff --git a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index 1468181..b1ff761 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -1722,6 +1722,8 @@ if __name__ == "__main__":
print("Ubuntu-canonical repository URL: https://{0}/repository/ubuntu-canonical".format(nexus_host))
print("Rrutter repository URL: https://{0}/repository/rrutter".format(nexus_host))
print("R repository URL: https://{0}/repository/r".format(nexus_host))
+ print("Amazon-main repository URL: https://{0}/repository/amazon-main".format(nexus_host))
+ print("Amazon-updates repository URL: https://{0}/repository/amazon-updates".format(nexus_host))
print("Squid proxy: {0}:3128".format(nexus_host))
if args.efs_id:
print('AWS EFS ID: {}'.format(args.efs_id))
diff --git a/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy b/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy
index df30440..60c8594 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy
+++ b/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy
@@ -53,6 +53,8 @@ repository.createRawProxy('jenkins','http://pkg.jenkins.io/debian-stable', 'pack
repository.createRawProxy('mongo','http://repo.mongodb.org/apt/ubuntu', 'packages_store')
repository.createRawHosted('packages', 'packages_store')
repository.createNpmProxy('npm', 'https://registry.npmjs.org', 'packages_store')
+repository.createRawProxy('amazon-main','http://packages.us-west-2.amazonaws.com/2017.09/main/154a6dd467e2/', 'packages_store')
+repository.createRawProxy('amazon-updates','http://packages.us-west-2.amazonaws.com/2017.09/updates/154a6dd467e2/', 'packages_store')
// create a role for service user
def role = new org.sonatype.nexus.security.role.Role(
diff --git a/infrastructure-provisioning/src/dataengine-service/templates/repository.repo b/infrastructure-provisioning/src/dataengine-service/templates/repository.repo
new file mode 100644
index 0000000..76d6aee
--- /dev/null
+++ b/infrastructure-provisioning/src/dataengine-service/templates/repository.repo
@@ -0,0 +1,17 @@
+[dlabrepo-main]
+name=DLab Repository
+baseurl=REPOSITORY_AMAZON_MAIN/$basearch/
+enabled=1
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+priority=1
+sslverify=0
+
+[dlabrepo-updates]
+name=DLab Repository
+baseurl=REPOSITORY_AMAZON_UPDATES/$basearch/
+enabled=1
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+priority=1
+sslverify=0
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 8ed137f..4443164 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -378,4 +378,8 @@ nginx_version = 1.15.1
# r_repo =
### Repository name for Rrutter
# rrutter_repo =
+### Repository name for Amazon main
+# amazon_main_repo =
+### Repository name for Amazon updates
+# amazon_updates_repo =
diff --git a/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile b/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
index c78ad9f..6c4c17c 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
@@ -23,8 +23,7 @@ FROM docker.dlab-base:latest
ARG OS
-COPY dataengine-service/fabfile.py /root/
-COPY dataengine-service/description.json /root/
+COPY dataengine-service/ /root/
COPY general/scripts/aws/dataengine-service_* /root/scripts/
COPY general/scripts/os/common_configure_proxy.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
@@ -32,6 +31,7 @@ COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_availa
COPY general/scripts/os/common_* /root/scripts/
COPY general/lib/os/redhat/common_lib.py /usr/lib/python2.7/dlab/common_lib.py
COPY general/lib/os/redhat/notebook_lib.py /usr/lib/python2.7/dlab/notebook_lib.py
+COPY general/templates/aws/Rprofile.site /root/templates/
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 310ba75..0cc4b40 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1956,4 +1956,24 @@ def find_des_jars(all_jars, des_path):
return all_jars
except Exception as err:
print('Error:', str(err))
- sys.exit(1)
\ No newline at end of file
+ sys.exit(1)
+
+
+def configure_dataengine_service_repos():
+ # Configuring yum
+ put('templates/repository.repo', '/etc/yum.repos.d/repository.repo', use_sudo=True)
+ sudo('sed -i "s|REPOSITORY_AMAZON_MAIN|{}|g" /etc/yum.repos.d/repository.repo'.format(
+ os.environ['local_repository_amazon_main_repo']))
+ sudo('sed -i "s|REPOSITORY_AMAZON_UPDATES|{}|g" /etc/yum.repos.d/repository.repo'.format(
+ os.environ['local_repository_amazon_updates_repo']))
+ sudo('yum-config-manager --disable *')
+ sudo('yum-config-manager --enable dlabrepo*')
+ # Configuring pip
+ sudo('touch /etc/pip.conf')
+ sudo('echo "[global]" > /etc/pip.conf')
+ sudo('echo "timeout = 600" >> /etc/pip.conf')
+ sudo('echo "index-url = {}/simple/" >> /etc/pip.conf'.format(os.environ['local_repository_pypi_repo']))
+ sudo('echo "trusted-host = {}" >> /etc/pip.conf'.format(os.environ['local_repository_host']))
+ # Configuring R
+ put('templates/Rprofile.site', '/usr/lib64/R/etc/Rprofile.site', use_sudo=True)
+ sudo('sed -i "s|R_REPO|{}|g" /usr/lib64/R/etc/Rprofile.site'.format(os.environ['local_repository_r_repo']))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
index cce4074..2cac272 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
@@ -85,7 +85,10 @@ def configure_dataengine_service(instance, emr_conf):
env['connection_attempts'] = 100
env.key_filename = emr_conf['key_path']
env.host_string = emr_conf['os_user'] + '@' + emr_conf['instance_ip']
- sudo('echo "[main]" > /etc/yum/pluginconf.d/priorities.conf ; echo "enabled = 0" >> /etc/yum/pluginconf.d/priorities.conf')
+ sudo('echo "[main]" > /etc/yum/pluginconf.d/priorities.conf ; echo "enabled = 0" >> '
+ '/etc/yum/pluginconf.d/priorities.conf')
+ if os.environ['local_repository_enabled'] == 'True':
+ configure_dataengine_service_repos()
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index beb4a6e..783bcd7 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@ -67,7 +67,8 @@ if __name__ == "__main__":
in os.environ and 'local_repository_user_password' in os.environ:
basic_auth_repos = ('apt_bintray_repo', 'apt_ubuntu_security_repo', 'apt_ubuntu_repo', 'docker_repo',
'jenkins_repo', 'maven_bintray_repo', 'mongo_repo', 'pypi_repo',
- 'packages_repo', 'r_repo', 'rrutter_repo', 'apt_ubuntu_canonical_repo')
+ 'packages_repo', 'r_repo', 'rrutter_repo', 'apt_ubuntu_canonical_repo',
+ 'amazon_main_repo', 'amazon_updates_repo')
for repo in basic_auth_repos:
os.environ['local_repository_{}'.format(repo)] = os.environ['local_repository_{}'.format(
repo)].replace('{}://'.format(os.environ['local_repository_{}'.format(repo)].split(':')[0]),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org