You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/02/26 12:29:09 UTC
[incubator-dlab] branch EPMCDLAB-1186 updated: added support of
anonymous user and repo credentials
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch EPMCDLAB-1186
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/EPMCDLAB-1186 by this push:
new 7d70b9c added support of anonymous user and repo credentials
7d70b9c is described below
commit 7d70b9c509ec0dc93aeeaeda92de35a7e160f231
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Tue Feb 26 14:27:58 2019 +0200
added support of anonymous user and repo credentials
---
infrastructure-provisioning/scripts/deploy_dlab.py | 2 ++
.../src/general/conf/dlab.ini | 4 ++++
.../src/general/files/aws/deeplearning_Dockerfile | 1 +
.../src/general/files/aws/jupyter_Dockerfile | 1 +
.../src/general/files/aws/rstudio_Dockerfile | 1 +
.../src/general/files/aws/ssn_Dockerfile | 1 +
.../src/general/files/aws/tensor-rstudio_Dockerfile | 1 +
.../src/general/files/aws/tensor_Dockerfile | 1 +
.../src/general/files/aws/zeppelin_Dockerfile | 1 +
.../src/general/files/os/npm_login | 21 +++++++++++++++++++++
.../src/general/lib/os/debian/notebook_lib.py | 14 ++++++++++++++
.../src/general/lib/os/debian/ssn_lib.py | 12 +++++++++++-
.../src/general/scripts/aws/ssn_configure.py | 12 ++++++++++++
.../src/ssn/scripts/configure_conf_file.py | 18 +++++++++++++++++-
.../src/ssn/scripts/configure_docker.py | 9 +++++++--
15 files changed, 95 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_dlab.py b/infrastructure-provisioning/scripts/deploy_dlab.py
index 003b869..b79238c 100644
--- a/infrastructure-provisioning/scripts/deploy_dlab.py
+++ b/infrastructure-provisioning/scripts/deploy_dlab.py
@@ -121,6 +121,8 @@ parser.add_argument('--ldap_service_password', type=str, default='service-user-p
parser.add_argument('--local_repository_enabled', type=str, default='False', help='True - to use local repository. '
'Otherwise - False')
parser.add_argument('--local_repository_host', type=str, default='', help='IP address or DNS name of repository')
+parser.add_argument('--local_repository_user_name', type=str, default='', help='Name of repository user')
+parser.add_argument('--local_repository_user_password', type=str, default='', help='Password of repository user')
parser.add_argument('--local_repository_cert_path', type=str, default='', help='Full path to cert')
parser.add_argument('--local_repository_parent_proxy_host', type=str, default='', help='IP address or DNS name of '
'parent proxy')
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 7e0e09d..4f9fd15 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -333,6 +333,10 @@ nginx_version = 1.15.1
# host =
### Full path to repository certificate
# cert_path =
+### Name of repository user
+# repository_user_name =
+### Password of repository user
+# repository_user_password =
### IP address of parent Squid
# parent_proxy_host =
### Port number of parent Squid
diff --git a/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile b/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
index a2b7e17..6e157ff 100644
--- a/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
@@ -36,6 +36,7 @@ COPY general/templates/os/run_template.sh /root/templates/
COPY general/templates/os/tensorboard.service /root/templates/
COPY general/files/os/toree-assembly-0.2.0.jar /root/files/
COPY general/files/os/toree_kernel.tar.gz /root/files/
+COPY general/files/os/npm_login /root/files/
COPY general/templates/os/pyspark_dataengine-service_template.json /root/templates/
COPY general/templates/os/r_dataengine-service_template.json /root/templates/
COPY general/templates/os/toree_dataengine-service_* /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile b/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
index 1a8409c..d4bf666 100644
--- a/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
@@ -38,6 +38,7 @@ COPY general/templates/os/toree_dataengine-service_* /root/templates/
COPY general/templates/aws/Rprofile.site /root/templates/
COPY general/files/os/toree-assembly-0.2.0.jar /root/files/
COPY general/files/os/toree_kernel.tar.gz /root/files/
+COPY general/files/os/npm_login /root/files/
COPY general/templates/os/pyspark_dataengine_template.json /root/templates/
COPY general/templates/os/r_dataengine_template.json /root/templates/
COPY general/templates/os/toree_dataengine_template.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
index d900272..e2e9853 100644
--- a/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
@@ -28,6 +28,7 @@ COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/dlab/notebook_lib.p
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/aws/Rprofile.site /root/templates/
+COPY general/files/os/npm_login /root/files/
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile b/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
index 87e8da0..07c7b7c 100644
--- a/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
@@ -25,6 +25,7 @@ COPY infrastructure-provisioning/src/general/scripts/aws/ssn_* /root/scripts/
COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python2.7/dlab/ssn_lib.py
COPY infrastructure-provisioning/src/general/files/aws/ssn_policy.json /root/files/
COPY infrastructure-provisioning/src/general/templates/aws/jenkins_jobs /root/templates/jenkins_jobs
+COPY infrastructure-provisioning/src/general/files/os/npm_login /root/files/
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
index 04ea9dd..2bba6d4 100644
--- a/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
@@ -30,6 +30,7 @@ COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/tensorboard.service /root/templates/
COPY general/templates/aws/Rprofile.site /root/templates/
+COPY general/files/os/npm_login /root/files/
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
index 653e270..f27d032 100644
--- a/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
@@ -34,6 +34,7 @@ COPY general/templates/os/pyspark_dataengine_template.json /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/tensorboard.service /root/templates/
COPY general/templates/os/pyspark_dataengine-service_template.json /root/templates/
+COPY general/files/os/npm_login /root/files/
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile b/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
index d911775..2df37c3 100644
--- a/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
@@ -33,6 +33,7 @@ COPY general/templates/os/dataengine_interpreter_spark.json /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/aws/Rprofile.site /root/templates/
+COPY general/files/os/npm_login /root/files/
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/files/os/npm_login b/infrastructure-provisioning/src/general/files/os/npm_login
new file mode 100644
index 0000000..fe1c9e4
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/os/npm_login
@@ -0,0 +1,21 @@
+#!/usr/bin/expect -f
+
+set i 0; foreach n $argv {set "p[incr i]" $n}
+
+set timeout 60
+eval spawn $p1
+match_max 100000
+
+expect "Username"
+send "$p2\r"
+
+expect "Password"
+send "$p3\r"
+
+expect "Email"
+send "$p4\r"
+
+expect {
+ timeout exit 1
+ eof
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 7b2d5b5..4e62818 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -438,11 +438,25 @@ def install_nodejs(os_user):
sudo('./deps/npm/bin/npm-cli.js config set sass_binary_path /opt/node/linux-x64-57_binding.node')
sudo('./deps/npm/bin/npm-cli.js config set registry {0}/'.format(
os.environ['local_repository_npm_repo']))
+ if 'local_repository_user_name' in os.environ and 'local_repository_user_password' in os.environ:
+ put('/root/files/npm_login', '/tmp/npm_login')
+
+ sudo('expect -f /tmp/npm_login {0} {1} {2} "{3}"'.format("./deps/npm/bin/npm-cli.js",
+ os.environ['local_repository_user_name'],
+ os.environ[
+ 'local_repository_user_password'],
+ 'example@example.com'))
sudo('./deps/npm/bin/npm-cli.js install npm')
sudo('cp deps/npm/bin/npm /opt/node/')
sudo('npm config set strict-ssl false')
sudo('npm config set registry {0}/'.format(
os.environ['local_repository_npm_repo']))
+ if 'local_repository_user_name' in os.environ and 'local_repository_user_password' in os.environ:
+ sudo('expect -f /tmp/npm_login {0} {1} {2} "{3}"'.format('npm',
+ os.environ['local_repository_user_name'],
+ os.environ[
+ 'local_repository_user_password'],
+ 'example@example.com'))
sudo('npm config set sass_binary_path /opt/node/linux-x64-57_binding.node')
else:
sudo('curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index fb8f519..7cd1d33 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -334,7 +334,7 @@ def install_build_dep():
try:
if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_dlab_path'])):
maven_version = '3.5.4'
- sudo('apt-get install -y openjdk-8-jdk git wget unzip gcc g++ make')
+ sudo('apt-get install -y openjdk-8-jdk git wget unzip gcc g++ make expect')
with cd('/opt/'):
if os.environ['local_repository_enabled'] == 'True':
sudo('wget {0}/apache-maven-{1}-bin.zip'.format(
@@ -366,11 +366,21 @@ def install_build_dep():
sudo('./deps/npm/bin/npm-cli.js config set sass_binary_path /opt/node/linux-x64-57_binding.node')
sudo('./deps/npm/bin/npm-cli.js config set registry {0}/'.format(
os.environ['local_repository_npm_repo']))
+ if 'local_repository_user_name' in os.environ and 'local_repository_user_password' in os.environ:
+ put('/root/files/npm_login', '/tmp/npm_login')
+
+ sudo('expect -f /tmp/npm_login {0} {1} {2} "{3}"'.format("./deps/npm/bin/npm-cli.js",
+ os.environ['local_repository_user_name'], os.environ['local_repository_user_password'],
+ 'example@example.com'))
sudo('./deps/npm/bin/npm-cli.js install npm')
sudo('cp deps/npm/bin/npm /opt/node/')
sudo('npm config set strict-ssl false')
sudo('npm config set registry {0}/'.format(
os.environ['local_repository_npm_repo']))
+ if 'local_repository_user_name' in os.environ and 'local_repository_user_password' in os.environ:
+ sudo('expect -f /tmp/npm_login {0} {1} {2} "{3}"'.format('npm',
+ os.environ['local_repository_user_name'], os.environ['local_repository_user_password'],
+ 'example@example.com'))
sudo('npm config set sass_binary_path /opt/node/linux-x64-57_binding.node')
else:
sudo('bash -c "curl --silent --location https://deb.nodesource.com/setup_8.x | bash -"')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index 73dbe10..56b219e 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@ -58,6 +58,18 @@ if __name__ == "__main__":
billing_enabled = True
dlab_ssh_user = os.environ['conf_os_user']
network_type = os.environ['conf_network_type']
+ if os.environ['local_repository_enabled'] == 'True' and 'local_repository_user_name' \
+ in os.environ and 'local_repository_user_password' in os.environ:
+ basic_auth_repos = ('apt_bintray_repo', 'apt_ubuntu_security_repo', 'apt_ubuntu_repo', 'docker_repo',
+ 'jenkins_repo', 'maven_bintray_repo', 'maven_central_repo', 'mongo_repo', 'pypi_repo',
+ 'packages_repo', 'r_repo', 'rrutter_repo')
+ for repo in basic_auth_repos:
+ os.environ['local_repository_{}'.format(repo)] = os.environ['local_repository_{}'.format(
+ repo)].replace('{}://'.format(os.environ['local_repository_{}'.format(repo)].split(':')[0]),
+ '{0}://{1}:{2}@'.format(os.environ['local_repository_{}'.format(repo)].split(':')[0],
+ os.environ['local_repository_user_name'],
+ os.environ['local_repository_user_password']))
+
if 'ssn_hosted_zone_id' in os.environ and 'ssn_hosted_zone_name' in os.environ and \
'ssn_subdomain' in os.environ:
domain_created = True
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py b/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py
index a0cfa30..a9c58ba 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py
@@ -35,6 +35,9 @@ def modify_conf_file():
try:
variables_list = json.loads(args.variables_list)
conf_list = []
+ basic_auth_repos = ('apt_bintray_repo', 'apt_ubuntu_security_repo', 'apt_ubuntu_repo', 'docker_repo',
+ 'jenkins_repo', 'maven_bintray_repo', 'maven_central_repo', 'mongo_repo', 'pypi_repo',
+ 'packages_repo', 'r_repo', 'rrutter_repo')
conf_file = open('{}sources/infrastructure-provisioning/src/general/conf/dlab.ini'.format(args.dlab_dir), 'r')
for line in conf_file:
conf_list.append(line)
@@ -57,8 +60,21 @@ def modify_conf_file():
except:
print('Such variable doesn`t exist!')
config.remove_option(section, option)
+ # if section == 'local_repository' and 'repository_user_name' in options and 'repository_user_password' in \
+ # options:
+ # for option in options:
+ # if option in basic_auth_repos:
+ # repo_url = config.get(section, option)
+ # updated_repo_url = repo_url.replace('{}://'.format(repo_url.split(':')[0]),
+ # '{0}://{1}:{2}@'.format(
+ # repo_url.split(':')[0],
+ # config.get(section, 'repository_user_name'),
+ # config.get(section,
+ # 'repository_user_password')))
+ # config.set(section, option, updated_repo_url)
- with open('{}sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.dlab_dir), 'w') as conf_file_final:
+ with open('{}sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ args.dlab_dir), 'w') as conf_file_final:
config.write(conf_file_final)
except Exception as error:
print('Error with modifying conf files:')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 65d80ec..545527d 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -64,8 +64,13 @@ def update_repository(dlab_path, repository_host, region):
sudo('sed -i "s|^FROM ubuntu.*|FROM {0}/dlab-pre-base|g" base_Dockerfile'.format(
os.environ['local_repository_docker_internal_repo']))
sudo('sed -i "/pip install/d;/apt-get/d" base_Dockerfile')
- sudo('docker login -u docker-nexus -p docker-nexus {}'.format(
- os.environ['local_repository_docker_internal_repo']))
+ if 'local_repository_user_name' in os.environ and 'local_repository_user_password' in os.environ:
+ sudo('docker login -u {1} -p {2} {0}'.format(os.environ['local_repository_docker_internal_repo'],
+ os.environ['local_repository_user_name'],
+ os.environ['local_repository_user_password']))
+ else:
+ sudo('docker login -u anonymous -p anonymous {0}'.format(
+ os.environ['local_repository_docker_internal_repo']))
else:
sudo('''sed -i "23i RUN sed -i 's|REPOSITORY_UBUNTU|{}|g' /etc/apt/sources.list" base_Dockerfile'''.format(
repository_host))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org