You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/04/20 06:55:37 UTC
[incubator-datalab] 01/01: [DATALAB-2091]: merged develop
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit bb252f1c590f980c3e2af9c5cb4b2f43ff5b9dcc
Merge: ce9908a 9029790
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Apr 20 09:55:07 2021 +0300
[DATALAB-2091]: merged develop
README.md | 2 +-
.../src/general/lib/aws/actions_lib.py | 5 +-
.../src/general/lib/os/debian/edge_lib.py | 34 ++++++-
.../datalab/properties/ChangePropertiesConst.java | 21 +++-
.../properties/ChangePropertiesService.java | 19 ++++
.../properties/ExternalChangeProperties.java | 70 +++++++++----
.../com/epam/datalab/properties/RestartForm.java | 19 ++++
.../java/com/epam/datalab/properties/YmlDTO.java | 19 ++++
.../DynamicChangePropertiesExceptionMapper.java | 19 ++++
.../backendapi/ProvisioningServiceApplication.java | 1 -
.../datalab/backendapi/core/DockerWarmuper.java | 1 -
.../datalab/backendapi/core/MetadataHolder.java | 1 -
.../backendapi/core/commands/DockerCommands.java | 1 -
.../backendapi/core/commands/ICommandExecutor.java | 1 +
.../response/folderlistener/AsyncFileHandler.java | 1 -
.../response/folderlistener/FolderListener.java | 1 -
.../core/response/folderlistener/WatchItem.java | 1 -
.../response/folderlistener/WatchItemList.java | 1 -
.../handlers/CheckInactivityCallbackHandler.java | 1 +
.../handlers/ResourcesStatusCallbackHandler.java | 1 -
.../handlers/ReuploadKeyCallbackHandler.java | 1 +
.../resources/ChangePropertiesResource.java | 39 ++++++++
.../resources/ProvisioningHealthCheckResource.java | 1 -
.../resources/base/InfrastructureService.java | 1 -
.../backendapi/resources/base/KeyResource.java | 1 -
.../process/builder/ProcessInfoBuilder.java | 1 +
.../process/exception/DatalabProcessException.java | 1 +
.../com/epam/datalab/process/model/ProcessId.java | 1 +
.../epam/datalab/process/model/ProcessInfo.java | 1 +
.../epam/datalab/backendapi/dao/BillingDAO.java | 1 +
.../datalab/backendapi/dao/ComputationalDAO.java | 1 -
.../resources/ChangePropertiesResource.java | 46 ++++++---
.../backendapi/resources/ExploratoryResource.java | 2 +-
.../backendapi/resources/SchedulerJobResource.java | 2 +-
.../backendapi/resources/SystemInfoResource.java | 2 +-
.../backendapi/resources/UserSettingsResource.java | 2 +-
.../resources/aws/ComputationalResourceAws.java | 1 -
.../configuration/configuration.component.html | 6 +-
.../configuration/configuration.component.ts | 109 +++++++++++++++------
39 files changed, 356 insertions(+), 82 deletions(-)
diff --cc infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 9e41882,2a4cb9f..7005e85
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@@ -82,79 -82,102 +82,105 @@@ def install_nginx_lua(edge_ip, nginx_ve
os.environ['conf_stepcerts_kid_password'], user))
sans = "--san localhost --san 127.0.0.1 {0}".format(step_cert_sans)
cn = edge_ip
- sudo('step ca token {3} --kid {0} --ca-url "{1}" --root /etc/ssl/certs/root_ca.crt '
+ datalab.fab.conn.sudo('step ca token {3} --kid {0} --ca-url "{1}" --root /etc/ssl/certs/root_ca.crt '
'--password-file /home/{2}/keys/provisioner_password {4} --output-file /tmp/step_token'.format(
os.environ['conf_stepcerts_kid'], os.environ['conf_stepcerts_ca_url'], user, cn, sans))
- token = sudo('cat /tmp/step_token')
- sudo('step ca certificate "{0}" /etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key '
+ token = datalab.fab.conn.sudo('cat /tmp/step_token').stdout.replace('\n','')
+ datalab.fab.conn.sudo('step ca certificate "{0}" /etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key '
'--token "{1}" --kty=RSA --size 2048 --provisioner {2} '.format(cn, token,
os.environ['conf_stepcerts_kid']))
- sudo('touch /var/log/renew_certificates.log')
- put('/root/templates/manage_step_certs.sh', '/usr/local/bin/manage_step_certs.sh', use_sudo=True)
- sudo('chmod +x /usr/local/bin/manage_step_certs.sh')
- sudo('sed -i "s|STEP_ROOT_CERT_PATH|/etc/ssl/certs/root_ca.crt|g" '
+ datalab.fab.conn.sudo('touch /var/log/renew_certificates.log')
+ datalab.fab.conn.put('/root/templates/manage_step_certs.sh', '/tmp/manage_step_certs.sh')
+ datalab.fab.conn.sudo('cp /tmp/manage_step_certs.sh /usr/local/bin/manage_step_certs.sh')
+ datalab.fab.conn.sudo('chmod +x /usr/local/bin/manage_step_certs.sh')
+ datalab.fab.conn.sudo('sed -i "s|STEP_ROOT_CERT_PATH|/etc/ssl/certs/root_ca.crt|g" '
'/usr/local/bin/manage_step_certs.sh')
- sudo('sed -i "s|STEP_CERT_PATH|/etc/ssl/certs/datalab.crt|g" /usr/local/bin/manage_step_certs.sh')
- sudo('sed -i "s|STEP_KEY_PATH|/etc/ssl/certs/datalab.key|g" /usr/local/bin/manage_step_certs.sh')
- sudo('sed -i "s|STEP_CA_URL|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(
+ datalab.fab.conn.sudo('sed -i "s|STEP_CERT_PATH|/etc/ssl/certs/datalab.crt|g" /usr/local/bin/manage_step_certs.sh')
+ datalab.fab.conn.sudo('sed -i "s|STEP_KEY_PATH|/etc/ssl/certs/datalab.key|g" /usr/local/bin/manage_step_certs.sh')
+ datalab.fab.conn.sudo('sed -i "s|STEP_CA_URL|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(
os.environ['conf_stepcerts_ca_url']))
- sudo('sed -i "s|RESOURCE_TYPE|edge|g" /usr/local/bin/manage_step_certs.sh')
- sudo('sed -i "s|SANS|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(sans))
- sudo('sed -i "s|CN|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(cn))
- sudo('sed -i "s|KID|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(
+ datalab.fab.conn.sudo('sed -i "s|RESOURCE_TYPE|edge|g" /usr/local/bin/manage_step_certs.sh')
+ datalab.fab.conn.sudo('sed -i "s|SANS|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(sans))
+ datalab.fab.conn.sudo('sed -i "s|CN|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(cn))
+ datalab.fab.conn.sudo('sed -i "s|KID|{0}|g" /usr/local/bin/manage_step_certs.sh'.format(
os.environ['conf_stepcerts_kid']))
- sudo('sed -i "s|STEP_PROVISIONER_PASSWORD_PATH|/home/{0}/keys/provisioner_password|g" '
+ datalab.fab.conn.sudo('sed -i "s|STEP_PROVISIONER_PASSWORD_PATH|/home/{0}/keys/provisioner_password|g" '
'/usr/local/bin/manage_step_certs.sh'.format(user))
- sudo('bash -c \'echo "0 * * * * root /usr/local/bin/manage_step_certs.sh >> '
+ datalab.fab.conn.sudo('bash -c \'echo "0 * * * * root /usr/local/bin/manage_step_certs.sh >> '
'/var/log/renew_certificates.log 2>&1" >> /etc/crontab \'')
- put('/root/templates/step-cert-manager.service', '/etc/systemd/system/step-cert-manager.service',
- use_sudo=True)
- sudo('systemctl daemon-reload')
- sudo('systemctl enable step-cert-manager.service')
+ datalab.fab.conn.put('/root/templates/step-cert-manager.service', '/tmp/step-cert-manager.service')
+ datalab.fab.conn.sudo('cp -f /tmp/step-cert-manager.service /etc/systemd/system/step-cert-manager.service')
+ datalab.fab.conn.sudo('systemctl daemon-reload')
+ datalab.fab.conn.sudo('systemctl enable step-cert-manager.service')
else:
- sudo('openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/datalab.key \
+ datalab.fab.conn.sudo('openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/datalab.key \
-out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"'.format(hostname))
- sudo('mkdir -p /tmp/src')
- with cd('/tmp/src/'):
- sudo('wget https://luarocks.org/releases/luarocks-3.3.1.tar.gz')
- sudo('tar -xzf luarocks-3.3.1.tar.gz')
+ datalab.fab.conn.sudo('mkdir -p /tmp/src')
+ datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/ && wget https://luarocks.org/releases/luarocks-3.3.1.tar.gz' ''')
+ datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/ && tar -xzf luarocks-3.3.1.tar.gz' ''')
- sudo('wget -O - https://openresty.org/package/pubkey.gpg | sudo apt-key add -')
- sudo('add-apt-repository -y "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main"')
- sudo('apt-get update')
- sudo('apt-get -y install openresty=1.15.8.1-1~bionic1')
+ datalab.fab.conn.sudo('wget -O - https://openresty.org/package/pubkey.gpg | sudo apt-key add -')
+ datalab.fab.conn.sudo('add-apt-repository -y "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main"')
+ datalab.fab.conn.sudo('apt-get update')
+ datalab.fab.conn.sudo('apt-get -y install openresty=1.19.3.1-1~focal1')
- with cd('/tmp/src/luarocks-3.3.1/'):
- sudo('./configure')
- sudo('make install')
- try:
- allow = False
- counter = 0
- while not allow:
- if counter > 5:
- sys.exit(1)
+ datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/luarocks-3.3.1/ && ./configure' ''')
+ datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/luarocks-3.3.1/ && make install' ''')
- datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/luarocks-3.3.1/ && luarocks install lua-resty-jwt 0.2.2 --tree /usr/local/openresty/lualib/resty/' ''')
- datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/luarocks-3.3.1/ && luarocks install lua-resty-openidc --tree /usr/local/openresty/lualib/resty/' ''')
++ try:
++ allow = False
++ counter = 0
++ while not allow:
++ if counter > 5:
++ sys.exit(1)
++ else:
++ if 'Could not fetch' in datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/luarocks-3.3.1/ && luarocks install lua-resty-jwt 0.2.2 --tree /usr/local/openresty/lualib/resty/' ''').stdout \
++ or 'Could not fetch' in datalab.fab.conn.sudo('''bash -c 'cd /tmp/src/luarocks-3.3.1/ && luarocks install lua-resty-openidc --tree /usr/local/openresty/lualib/resty/' ''').stdout:
++ counter += 1
++ time.sleep(10)
+ else:
- if 'Could not fetch' in sudo('luarocks install lua-resty-jwt 0.2.2 --tree /usr/local/openresty/lualib/resty/ 2>&1') or 'Could not fetch' in sudo('luarocks install lua-resty-openidc --tree /usr/local/openresty/lualib/resty/ 2>&1'):
- counter += 1
- time.sleep(10)
- else:
- allow = True
- except:
- sys.exit(1)
++ allow = True
++ except:
++ sys.exit(1)
- datalab.fab.conn.sudo('luarocks install lua-resty-jwt 0.2.2')
- datalab.fab.conn.sudo('luarocks install lua-resty-openidc')
+ try:
+ allow = False
+ counter = 0
+ while not allow:
+ if counter > 5:
+ sys.exit(1)
+ else:
- if 'Could not fetch' in sudo('luarocks install lua-resty-jwt 0.2.2 2>&1') or 'Could not fetch' in sudo('luarocks install lua-resty-openidc 2>&1'):
++ if 'Could not fetch' in datalab.fab.conn.sudo('luarocks install lua-resty-jwt 0.2.2').stdout \
++ or 'Could not fetch' in datalab.fab.conn.sudo('luarocks install lua-resty-openidc').stdout:
+ counter += 1
+ time.sleep(10)
+ else:
+ allow = True
+ except:
+ sys.exit(1)
- sudo('useradd -r nginx')
+ datalab.fab.conn.sudo('useradd -r nginx')
- sudo('mkdir -p /opt/datalab/templates')
- put('/root/templates', '/opt/datalab', use_sudo=True)
- sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(edge_ip))
- sudo('sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
+ datalab.fab.conn.sudo('mkdir -p /opt/datalab/templates')
+ datalab.fab.conn.local('''bash -c 'cd /root/templates; tar -zcvf /tmp/templates.tar.gz *' ''')
+ datalab.fab.conn.put('/tmp/templates.tar.gz', '/tmp/templates.tar.gz')
+ datalab.fab.conn.sudo('tar -zxvf /tmp/templates.tar.gz -C /opt/datalab/templates/')
+ datalab.fab.conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(edge_ip))
+ datalab.fab.conn.sudo('sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
keycloak_auth_server_url))
- sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
+ datalab.fab.conn.sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
keycloak_realm_name))
- sudo('sed -i \'s/KEYCLOAK_CLIENT_ID/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
+ datalab.fab.conn.sudo('sed -i \'s/KEYCLOAK_CLIENT_ID/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
keycloak_client_id))
- sudo('sed -i \'s/KEYCLOAK_CLIENT_SECRET/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
+ datalab.fab.conn.sudo('sed -i \'s/KEYCLOAK_CLIENT_SECRET/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
keycloak_client_secret))
- sudo('cp /opt/datalab/templates/nginx.conf /usr/local/openresty/nginx/conf')
- sudo('mkdir /usr/local/openresty/nginx/conf/conf.d')
- sudo('cp /opt/datalab/templates/conf.d/proxy.conf /usr/local/openresty/nginx/conf/conf.d/')
- sudo('mkdir /usr/local/openresty/nginx/conf/locations')
- sudo('systemctl start openresty')
- sudo('touch /tmp/nginx_installed')
+ datalab.fab.conn.sudo('cp /opt/datalab/templates/nginx.conf /usr/local/openresty/nginx/conf')
+ datalab.fab.conn.sudo('mkdir /usr/local/openresty/nginx/conf/conf.d')
+ datalab.fab.conn.sudo('cp /opt/datalab/templates/conf.d/proxy.conf /usr/local/openresty/nginx/conf/conf.d/')
+ datalab.fab.conn.sudo('mkdir /usr/local/openresty/nginx/conf/locations')
+ datalab.fab.conn.sudo('systemctl start openresty')
+ datalab.fab.conn.sudo('touch /tmp/nginx_installed')
if os.environ['conf_letsencrypt_enabled'] == 'true':
print("Configuring letsencrypt certificates.")
install_certbot(os.environ['conf_os_family'])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org