You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/29 08:26:29 UTC
[incubator-datalab] 01/01: [DATALAB-2572]: changed pip libs install for deeplearning
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2572
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6b56ac7713f249d89bb81361755bf236bc8965e9
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 29 11:26:17 2022 +0300
[DATALAB-2572]: changed pip libs install for deeplearning
---
.../src/general/lib/azure/actions_lib.py | 2 ++
.../src/general/lib/azure/meta_lib.py | 2 ++
.../src/general/lib/os/debian/notebook_lib.py | 42 +++++++++++++++-------
3 files changed, 33 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 09ec650..1de7e69 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -62,6 +62,8 @@ class AzureActions:
client_secret=json.dumps(self.sp_creds['clientSecret']).replace('"', ''),
client_id=json.dumps(self.sp_creds['clientId']).replace('"', ''),
resource='https://datalake.azure.net/')
+ logger = logging.getLogger('azure')
+ logger.setLevel(logging.ERROR)
def create_resource_group(self, resource_group_name, region):
try:
diff --git a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
index 510e875..3447e7e 100644
--- a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py
@@ -53,6 +53,8 @@ class AzureMeta:
client_secret=json.dumps(self.sp_creds['clientSecret']).replace('"', ''),
client_id=json.dumps(self.sp_creds['clientId']).replace('"', ''),
resource='https://datalake.azure.net/')
+ logger = logging.getLogger('azure')
+ logger.setLevel(logging.ERROR)
def get_resource_group(self, resource_group_name):
try:
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 66f347c..5c51d62 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -274,23 +274,39 @@ def ensure_python3_libraries(os_user):
manage_pkg('-y install', 'remote', 'libbz2-dev libsqlite3-dev tk-dev libncursesw5-dev libreadline-dev '
'liblzma-dev uuid-dev lzma-dev libgdbm-dev') #necessary for python build
datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff')
- if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true':
+ if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true':
+ datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff')
datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqt5==5.12')
datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqtwebengine==5.12')
datalab.fab.conn.sudo('-i pip3 install setuptools')
+ try:
+ datalab.fab.conn.sudo(
+ '-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
+ os.environ['notebook_nbconvert_version']))
+ except:
+ datalab.fab.conn.sudo(
+ '-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
+ os.environ['notebook_nbconvert_version']))
+ datalab.fab.conn.sudo(
+ '-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+ datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir')
+ datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
else:
- datalab.fab.conn.sudo('-i pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
- try:
- datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
- .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
- os.environ['notebook_nbconvert_version']))
- except:
- datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
- .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
- os.environ['notebook_nbconvert_version']))
- datalab.fab.conn.sudo('-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
- datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir')
- datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
+ datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff')
+ datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
+ try:
+ datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
+ os.environ['notebook_nbconvert_version']))
+ except:
+ datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \
+ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'],
+ os.environ['notebook_nbconvert_version']))
+ datalab.fab.conn.sudo('pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+ datalab.fab.conn.sudo('pip3 install boto3 --no-cache-dir')
+ datalab.fab.conn.sudo('pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured')
except:
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org