You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by lf...@apache.org on 2020/08/11 14:53:46 UTC
[incubator-dlab] branch DLAB-1977 created (now a11522f)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DLAB-1977
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at a11522f [DLAB-1977]: added possibility to get lists of available packages separately for different groups
This branch includes the following new commits:
new 6f0eb36 [DLAB-1977]: added fix to dataproc config failure due to changed install_os_pkg function
new a11522f [DLAB-1977]: added possibility to get lists of available packages separately for different groups
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/02: [DLAB-1977]: added possibility to get lists
of available packages separately for different groups
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1977
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit a11522fb520dd907f3366086b192096a0d5bcf6b
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Aug 11 14:43:22 2020 +0300
[DLAB-1977]: added possibility to get lists of available packages separately for different groups
---
.../scripts/aws/dataengine-service_list_libs.py | 1 +
.../scripts/gcp/dataengine-service_list_libs.py | 5 +++--
.../src/general/scripts/os/dataengine_list_libs.py | 5 +++--
.../general/scripts/os/get_list_available_pkgs.py | 20 +++++++++++---------
.../src/general/scripts/os/notebook_list_libs.py | 9 +++++----
5 files changed, 23 insertions(+), 17 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
index e4a0f38..ba7a25a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
@@ -47,6 +47,7 @@ if __name__ == "__main__":
try:
data_engine['os_user'] = 'ec2-user'
data_engine['cluster_name'] = os.environ['computational_id']
+ data_engine['group_name'] = os.environ['libCacheKey']
data_engine['cluster_id'] = get_emr_id_by_name(data_engine['cluster_name'])
data_engine['cluster_instances'] = get_emr_instances_list(data_engine['cluster_id'], 'MASTER')
data_engine['master_ip'] = data_engine['cluster_instances'][0].get('PrivateIpAddress')
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
index 65c2082..16fbb64 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
@@ -40,11 +40,12 @@ if __name__ == "__main__":
filename=local_log_filepath)
try:
- logging.info('[GETTING ALL AVAILABLE PACKAGES]')
- print('[GETTING ALL AVAILABLE PACKAGES]')
+ logging.info('[GETTING AVAILABLE PACKAGES]')
+ print('[GETTING AVAILABLE PACKAGES]')
data_engine = dict()
try:
data_engine['os_user'] = os.environ['conf_os_user']
+ data_engine['group_name'] = os.environ['libCacheKey']
data_engine['cluster_name'] = os.environ['computational_id']
data_engine['gcp_project_id'] = os.environ['gcp_project_id']
data_engine['gcp_region'] = os.environ['gcp_region']
diff --git a/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py b/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py
index 9f2b18b..b27b486 100644
--- a/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py
@@ -40,12 +40,13 @@ if __name__ == "__main__":
filename=local_log_filepath)
try:
- logging.info('[GETTING ALL AVAILABLE PACKAGES]')
- print('[GETTING ALL AVAILABLE PACKAGES]')
+ logging.info('[GETTING AVAILABLE PACKAGES]')
+ print('[GETTING AVAILABLE PACKAGES]')
data_engine = dict()
try:
data_engine['os_user'] = os.environ['conf_os_user']
data_engine['service_base_name'] = os.environ['conf_service_base_name']
+ data_engine['group_name'] = os.environ['libCacheKey']
data_engine['tag_name'] = data_engine['service_base_name'] + '-tag'
data_engine['cluster_name'] = os.environ['computational_id']
data_engine['master_node_name'] = data_engine['cluster_name'] + '-m'
diff --git a/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py b/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
index 3d2b4b6..0091df8 100644
--- a/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
@@ -34,6 +34,7 @@ parser = argparse.ArgumentParser()
parser.add_argument('--os_user', type=str, default='')
parser.add_argument('--instance_ip', type=str, default='')
parser.add_argument('--keyfile', type=str, default='')
+parser.add_argument('--group', type=str, default='')
args = parser.parse_args()
@@ -77,19 +78,20 @@ if __name__ == "__main__":
env['connection_attempts'] = 100
env.key_filename = [args.keyfile]
env.host_string = '{}@{}'.format(args.os_user, args.instance_ip)
-
all_pkgs = dict()
- all_pkgs['os_pkg'] = get_available_os_pkgs()
- all_pkgs['java'] = {}
-
- if os.environ['application'] in ('jupyter', 'jupyterlab', 'zeppelin', 'deeplearning', 'tensor', 'tensor-rstudio', 'rstudio'):
+ if args.group == 'os_pkg':
+ all_pkgs['os_pkg'] = get_available_os_pkgs()
+ elif args.group == 'java':
+ all_pkgs['java'] = {}
+ elif args.group == 'pip2':
+ all_pkgs['pip2'] = get_available_pip_pkgs("2.7")
+ elif args.group == 'pip3':
+ all_pkgs['pip3'] = get_available_pip_pkgs("3.6")
+ elif args.group == 'others':
all_pkgs['pip2'] = get_available_pip_pkgs("2.7")
all_pkgs['pip3'] = get_available_pip_pkgs("3.6")
all_pkgs['others'] = get_uncategorised_pip_pkgs(all_pkgs['pip2'], all_pkgs['pip3'])
-
- if (os.environ['application'] in ('jupyter', 'jupyterlab', 'zeppelin')
- and os.environ['notebook_r_enabled'] == 'true')\
- or os.environ['application'] in ('rstudio', 'tensor-rstudio'):
+ elif args.group == 'r_pkg':
all_pkgs['r_pkg'] = get_available_r_pkgs()
# Writing response file & json file with all pkgs
diff --git a/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py b/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py
index 820c818..d1f9ada 100644
--- a/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py
@@ -40,11 +40,12 @@ if __name__ == "__main__":
filename=local_log_filepath)
try:
- logging.info('[GETTING ALL AVAILABLE PACKAGES]')
- print('[GETTING ALL AVAILABLE PACKAGES]')
+ logging.info('[GETTING AVAILABLE PACKAGES]')
+ print('[GETTING AVAILABLE PACKAGES]')
notebook_config = dict()
try:
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
+ notebook_config['group_name'] = os.environ['libCacheKey']
notebook_config['os_user'] = os.environ['conf_os_user']
notebook_config['service_base_name'] = os.environ['conf_service_base_name'].lower()
notebook_config['tag_name'] = notebook_config['service_base_name'] + '-tag'
@@ -55,8 +56,8 @@ if __name__ == "__main__":
print('Error: {0}'.format(err))
append_result("Failed to get parameter.", str(err))
sys.exit(1)
- params = "--os_user {} --instance_ip {} --keyfile '{}'" \
- .format(notebook_config['os_user'], notebook_config['notebook_ip'], notebook_config['keyfile'])
+ params = "--os_user {} --instance_ip {} --keyfile '{}' --group {}" \
+ .format(notebook_config['os_user'], notebook_config['notebook_ip'], notebook_config['keyfile'], notebook_config['group_name'])
try:
# Run script to get available libs
local("~/scripts/{}.py {}".format('get_list_available_pkgs', params))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/02: [DLAB-1977]: added fix to dataproc config
failure due to changed install_os_pkg function
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1977
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 6f0eb36e7916ca2d37bc5dd84f298dd639fc8813
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Aug 11 12:21:31 2020 +0300
[DLAB-1977]: added fix to dataproc config failure due to changed install_os_pkg function
---
.../src/general/scripts/gcp/dataengine-service_configure.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
index 05b9c9b..7fa00f7 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
@@ -62,7 +62,7 @@ def configure_dataengine_service(instance, dataproc_conf):
env['connection_attempts'] = 100
env.key_filename = "{}".format(dataproc_conf['key_path'])
env.host_string = dataproc_conf['dlab_ssh_user'] + '@' + dataproc_conf['instance_ip']
- dlab.notebook_lib.install_os_pkg(['python-pip', 'python3-pip'])
+ dlab.notebook_lib.install_os_pkg([['python-pip', 'N/A'], ['python3-pip', 'N/A']])
dlab.fab.configure_data_engine_service_pip(dataproc_conf['instance_ip'], dataproc_conf['dlab_ssh_user'],
dataproc_conf['key_path'])
except:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org