You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/04/15 15:13:14 UTC
[incubator-dlab] branch EPMCDLAB-1186 updated: [DLAB-249]: Added
functionality to update Amazon repositories
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch EPMCDLAB-1186
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/EPMCDLAB-1186 by this push:
new 8887916 [DLAB-249]: Added functionality to update Amazon repositories
8887916 is described below
commit 88879169853a137fdce380283326c33243bdddb3
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Mon Apr 15 18:13:06 2019 +0300
[DLAB-249]: Added functionality to update Amazon repositories
---
.../scripts/deploy_repository/deploy_repository.py | 24 +++++++++-
.../scripts/update_amazon_repositories.py | 53 ++++++++++++++++++++++
.../templates/configureNexus.groovy | 4 +-
.../templates/updateRepositories.groovy | 24 ++++++++++
.../src/general/lib/os/debian/notebook_lib.py | 4 +-
5 files changed, 103 insertions(+), 6 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index b1ff761..add4177 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -849,7 +849,9 @@ def install_nexus():
mounting_disks()
else:
mount_efs()
- sudo('apt-get install -y maven nmap')
+ sudo('apt-get install -y maven nmap python-pip')
+ sudo('pip2 install -UI pip')
+ sudo('pip2 install -U fabric==1.14.0')
sudo('mkdir -p /opt/nexus')
sudo('wget https://sonatype-download.global.ssl.fastly.net/nexus/{0}/nexus-{1}-unix.tar.gz -O \
/opt/nexus-{1}-unix.tar.gz'.format(
@@ -886,6 +888,17 @@ def install_nexus():
sudo('sed -i "s/SERVICE_USER_NAME/{}/g" /tmp/configureNexus.groovy'.format(args.nexus_service_user_name))
sudo('sed -i "s/SERVICE_USER_PASSWORD/{}/g" /tmp/configureNexus.groovy'.format(
args.nexus_service_user_password))
+ sudo('wget http://repo.{}.amazonaws.com/2017.09/main/mirror.list -O /tmp/main_mirror.list'.format(
+ args.region))
+ sudo('wget http://repo.{}.amazonaws.com/2017.09/updates/mirror.list -O /tmp/updates_mirror.list'.format(
+ args.region))
+ amazon_main_repo = sudo("cat /tmp/main_mirror.list | grep {} | sed 's/$basearch//g'".format(args.region))
+ amazon_updates_repo = sudo("cat /tmp/updates_mirror.list | grep {} | sed 's/$basearch//g'".format(
+ args.region))
+ sudo('sed -i "s|AMAZON_MAIN_URL|{}|g" /tmp/configureNexus.groovy'.format(amazon_main_repo))
+ sudo('sed -i "s|AMAZON_UPDATES_URL|{}|g" /tmp/configureNexus.groovy'.format(amazon_updates_repo))
+ sudo('rm -f /tmp/main_mirror.list')
+ sudo('rm -f /tmp/updates_mirror.list')
put('scripts/addUpdateScript.groovy', '/tmp/addUpdateScript.groovy')
script_executed = False
while not script_executed:
@@ -1000,6 +1013,13 @@ def install_nexus():
sudo('echo "admin:{}" > /opt/nexus/credentials'.format(args.nexus_admin_password))
sudo('echo "{0}:{1}" >> /opt/nexus/credentials'.format(args.nexus_service_user_name,
args.nexus_service_user_password))
+ put('templates/updateRepositories.groovy', '/opt/nexus/updateRepositories.groovy', use_sudo=True)
+ put('scripts/update_amazon_repositories.py', '/opt/nexus/update_amazon_repositories.py', use_sudo=True)
+ sudo('sed -i "s|NEXUS_PASSWORD|{}|g" /opt/nexus/update_amazon_repositories.py'.format(
+ args.nexus_admin_password))
+ sudo('touch /var/log/amazon_repo_update.log')
+ sudo('echo "0 0 * * * root /usr/bin/python /opt/nexus/update_amazon_repositories.py --region {} >> '
+ '/var/log/amazon_repo_update.log" >> /etc/crontab'.format(args.region))
sudo('touch /home/{}/.ensure_dir/nexus_ensured'.format(configuration['conf_os_user']))
except Exception as err:
traceback.print_exc(file=sys.stdout)
@@ -1162,7 +1182,7 @@ def download_packages():
'http://central.maven.org/maven2/org/jfree/jfreechart/{0}/jfreechart-{0}.jar'.format('1.0.19'),
'http://central.maven.org/maven2/org/jfree/jcommon/{0}/jcommon-{0}.jar'.format('1.0.24'),
'--no-check-certificate https://brunelvis.org/jar/spark-kernel-brunel-all-{0}.jar'.format('2.3'),
- 'http://archive.apache.org/dist/incubator/toree/0.2.0-incubating/toree-pip/toree-0.2.0.tar.gz',
+ 'http://archive.apache.org/dist/incubator/toree/0.2.0-incubating/toree-`/toree-0.2.0.tar.gz',
'https://download2.rstudio.org/rstudio-server-{}-amd64.deb'.format(
configuration['notebook_rstudio_version']),
'http://us.download.nvidia.com/XFree86/Linux-x86_64/{0}/NVIDIA-Linux-x86_64-{0}.run'.format(
diff --git a/infrastructure-provisioning/scripts/deploy_repository/scripts/update_amazon_repositories.py b/infrastructure-provisioning/scripts/deploy_repository/scripts/update_amazon_repositories.py
new file mode 100644
index 0000000..df8e20f
--- /dev/null
+++ b/infrastructure-provisioning/scripts/deploy_repository/scripts/update_amazon_repositories.py
@@ -0,0 +1,53 @@
+#!/usr/bin/python
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+
+from fabric.api import *
+import argparse
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--region', required=True, type=str, default='', help='AWS region name')
+args = parser.parse_args()
+
+
+if __name__ == "__main__":
+ nexus_password = 'NEXUS_PASSWORD'
+ local('wget http://repo.{}.amazonaws.com/2017.09/main/mirror.list -O /tmp/main_mirror.list'.format(args.region))
+ local('wget http://repo.{}.amazonaws.com/2017.09/updates/mirror.list -O /tmp/updates_mirror.list'.format(
+ args.region))
+ amazon_main_repo = local("cat /tmp/main_mirror.list | grep {} | sed 's/$basearch//g'".format(args.region),
+ capture=True)
+ amazon_updates_repo = local("cat /tmp/updates_mirror.list | grep {} | sed 's/$basearch//g'".format(args.region),
+ capture=True)
+ local('cp -f /opt/nexus/updateRepositories.groovy /tmp/updateRepositories.groovy')
+ local('sed -i "s|AMAZON_MAIN_URL|{}|g" /tmp/updateRepositories.groovy'.format(amazon_main_repo))
+ local('sed -i "s|AMAZON_UPDATES_URL|{}|g" /tmp/updateRepositories.groovy'.format(amazon_updates_repo))
+ local('/usr/local/groovy/latest/bin/groovy /tmp/addUpdateScript.groovy -u "admin" -p "{}" '
+ '-n "updateRepositories" -f "/tmp/updateRepositories.groovy" -h "http://localhost:8081"'.format(
+ nexus_password))
+ local('curl -u admin:{} -X POST --header \'Content-Type: text/plain\' '
+ 'http://localhost:8081/service/rest/v1/script/updateRepositories/run'.format(nexus_password))
+ local('rm -f /tmp/main_mirror.list')
+ local('rm -f /tmp/updates_mirror.list')
+ local('rm -f /tmp/updateRepositories.groovy')
+ print('Amazon repositories have been successfully updated!')
diff --git a/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy b/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy
index 60c8594..5c94123 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy
+++ b/infrastructure-provisioning/scripts/deploy_repository/templates/configureNexus.groovy
@@ -53,8 +53,8 @@ repository.createRawProxy('jenkins','http://pkg.jenkins.io/debian-stable', 'pack
repository.createRawProxy('mongo','http://repo.mongodb.org/apt/ubuntu', 'packages_store')
repository.createRawHosted('packages', 'packages_store')
repository.createNpmProxy('npm', 'https://registry.npmjs.org', 'packages_store')
-repository.createRawProxy('amazon-main','http://packages.us-west-2.amazonaws.com/2017.09/main/154a6dd467e2/', 'packages_store')
-repository.createRawProxy('amazon-updates','http://packages.us-west-2.amazonaws.com/2017.09/updates/154a6dd467e2/', 'packages_store')
+repository.createRawProxy('amazon-main','AMAZON_MAIN_URL', 'packages_store')
+repository.createRawProxy('amazon-updates','AMAZON_UPDATES_URL', 'packages_store')
// create a role for service user
def role = new org.sonatype.nexus.security.role.Role(
diff --git a/infrastructure-provisioning/scripts/deploy_repository/templates/updateRepositories.groovy b/infrastructure-provisioning/scripts/deploy_repository/templates/updateRepositories.groovy
new file mode 100644
index 0000000..3bab507
--- /dev/null
+++ b/infrastructure-provisioning/scripts/deploy_repository/templates/updateRepositories.groovy
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2019, EPAM SYSTEMS INC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.sonatype.nexus.common.entity.*
+
+repository.getRepositoryManager().delete('amazon-main');
+repository.getRepositoryManager().delete('amazon-updates');
+
+repository.createRawProxy('amazon-main','AMAZON_MAIN_URL', 'packages_store')
+repository.createRawProxy('amazon-updates','AMAZON_UPDATES_URL', 'packages_store')
+log.info('Script completed successfully')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index f827318..c73068f 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -528,8 +528,8 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
if os.environ['local_repository_enabled'] == 'True':
sudo('pip2 install jupyter-console=={} --no-cache-dir'.format(
os.environ['notebook_jupyter_console_version']))
- sudo('pip2 install flask graphviz hypothesis jupyter matplotlib==2.0.2 pydot python-nvd3 pyyaml requests '
- 'scikit-image scipy setuptools tornado --no-cache-dir')
+ sudo('pip2 install networkx==2.2 flask graphviz hypothesis jupyter matplotlib==2.0.2 pydot python-nvd3 '
+ 'pyyaml requests scikit-image==0.14.2 scipy setuptools tornado --no-cache-dir')
sudo('pip3 install flask graphviz hypothesis jupyter matplotlib==2.0.2 pydot python-nvd3 pyyaml requests '
'scikit-image scipy setuptools tornado --no-cache-dir')
sudo('cp -f /opt/cudnn/include/* /opt/cuda-8.0/include/')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org