You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/08/18 09:08:10 UTC
[incubator-datalab] 02/02: [DATALAB-2409] - edge lib refactored
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 645e7b63f67406d9e81c78cc71638c0f0384eb6e
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Wed Aug 18 12:07:50 2021 +0300
[DATALAB-2409] - edge lib refactored
---
.../src/edge/scripts/configure_http_proxy.py | 3 +-
.../src/general/lib/os/debian/edge_lib.py | 56 +---------------------
.../src/general/lib/os/fab.py | 51 ++++++++++++++++++++
.../src/general/lib/os/redhat/edge_lib.py | 31 ------------
.../src/project/scripts/configure_http_proxy.py | 3 +-
.../src/project/scripts/configure_nftables.py | 3 +-
.../src/project/templates/squid.conf | 6 +--
7 files changed, 57 insertions(+), 96 deletions(-)
diff --git a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
index 3580b43..0e9034e 100644
--- a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
+++ b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
@@ -24,7 +24,6 @@
import argparse
import json
import sys
-from datalab.edge_lib import configure_http_proxy_server
from fabric import *
from datalab.fab import *
@@ -48,5 +47,5 @@ if __name__ == "__main__":
sys.exit(2)
print("Installing proxy for notebooks.")
- configure_http_proxy_server(deeper_config)
+ datalab.fab.configure_http_proxy_server(deeper_config)
conn.close()
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 7a91691..34d1273 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -23,41 +23,12 @@
import os
import sys
-from datalab.common_lib import manage_pkg
from fabric import *
from patchwork.files import exists
from patchwork import files
import datalab.fab
-
-def configure_http_proxy_server(config):
- try:
- if not exists(datalab.fab.conn,'/tmp/http_proxy_ensured'):
- manage_pkg('-y install', 'remote', 'squid')
- template_file = config['template_file']
- proxy_subnet = config['exploratory_subnet']
- datalab.fab.conn.put(template_file, '/tmp/squid.conf')
- datalab.fab.conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
-# datalab.fab.conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
-# datalab.fab.conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib/squid/basic_ldap_auth'))
- replace_string = ''
- for cidr in config['vpc_cidrs']:
- replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
- datalab.fab.conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- replace_string = ''
- for cidr in config['allowed_ip_cidr']:
- replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
- datalab.fab.conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- datalab.fab.conn.sudo('systemctl restart squid')
- datalab.fab.conn.sudo('touch /tmp/http_proxy_ensured')
- except Exception as err:
- print("Failed to install and configure squid: " + str(err))
- sys.exit(1)
-
+from datalab.common_lib import manage_pkg
+from datalab.logger import logging
def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id,
keycloak_client_secret, user, hostname, step_cert_sans):
@@ -189,27 +160,4 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
datalab.fab.configure_nginx_LE(os.environ['conf_letsencrypt_domain_name'], os.environ['project_name'].lower())
except Exception as err:
print("Failed install nginx with ldap: " + str(err))
- sys.exit(1)
-
-def configure_nftables(config):
- try:
- if not exists(datalab.fab.conn,'/tmp/nftables_ensured'):
- manage_pkg('-y install', 'remote', 'nftables')
- datalab.fab.conn.sudo('systemctl enable nftables.service')
- datalab.fab.conn.sudo('systemctl start nftables')
- datalab.fab.conn.sudo('sysctl net.ipv4.ip_forward=1')
- if os.environ['conf_cloud_provider'] == 'aws':
- interface = 'eth0'
- elif os.environ['conf_cloud_provider'] == 'gcp':
- interface = 'ens4'
- datalab.fab.conn.sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf')
- datalab.fab.conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip']))
- datalab.fab.conn.sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface))
- datalab.fab.conn.sudo(
- 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet']))
- datalab.fab.conn.sudo('cp /opt/datalab/templates/nftables.conf /etc/')
- datalab.fab.conn.sudo('systemctl restart nftables')
- datalab.fab.conn.sudo('touch /tmp/nftables_ensured')
- except Exception as err:
- print("Failed to configure nftables: " + str(err))
sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 40c6c92..707bc60 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -255,6 +255,57 @@ def configure_nginx_LE(domain_name, node):
sys.exit(1)
+#function for edge node only
+def configure_http_proxy_server(config):
+ try:
+ if not exists(datalab.fab.conn,'/tmp/http_proxy_ensured'):
+ manage_pkg('-y install', 'remote', 'squid')
+ template_file = config['template_file']
+ proxy_subnet = config['exploratory_subnet']
+ conn.put(template_file, '/tmp/squid.conf')
+ conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
+ conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
+ replace_string = ''
+ for cidr in config['vpc_cidrs']:
+ replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
+ conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
+ replace_string = ''
+ for cidr in config['allowed_ip_cidr']:
+ replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
+ conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
+ conn.sudo('systemctl restart squid')
+ fab.conn.sudo('touch /tmp/http_proxy_ensured')
+ except Exception as err:
+ logging.error('Fai to install and configure squid:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configure_nftables(config):
+ try:
+ if not exists(datalab.fab.conn,'/tmp/nftables_ensured'):
+ manage_pkg('-y install', 'remote', 'nftables')
+ conn.sudo('systemctl enable nftables.service')
+ conn.sudo('systemctl start nftables')
+ conn.sudo('sysctl net.ipv4.ip_forward=1')
+ if os.environ['conf_cloud_provider'] == 'aws':
+ interface = 'eth0'
+ elif os.environ['conf_cloud_provider'] == 'gcp':
+ interface = 'ens4'
+ conn.sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf')
+ conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip']))
+ conn.sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface))
+ conn.sudo(
+ 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet']))
+ conn.sudo('cp /opt/datalab/templates/nftables.conf /etc/')
+ conn.sudo('systemctl restart nftables')
+ conn.sudo('touch /tmp/nftables_ensured')
+ except Exception as err:
+ logging.error('Failed to configure nftables:', (err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
# functions for all computation resources
def ensure_python_venv(python_venv_version):
try:
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index ae81a2b..7617419 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -29,37 +29,6 @@ from patchwork.files import exists
from patchwork import files
-def configure_http_proxy_server(config):
- try:
- if not exists(conn,'/tmp/http_proxy_ensured'):
- manage_pkg('-y install', 'remote', 'squid')
- template_file = config['template_file']
- proxy_subnet = config['exploratory_subnet']
- conn.put(template_file, '/tmp/squid.conf')
- conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
- conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
- conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name']))
- conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host']))
- conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn']))
- conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user']))
- conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password']))
- conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib64/squid/basic_ldap_auth'))
- replace_string = ''
- for cidr in config['vpc_cidrs']:
- replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
- conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- replace_string = ''
- for cidr in config['allowed_ip_cidr']:
- replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
- conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- conn.sudo('systemctl restart squid')
- conn.sudo('chkconfig squid on')
- conn.sudo('touch /tmp/http_proxy_ensured')
- except Exception as err:
- print("Failed to install and configure squid: " + str(err))
- sys.exit(1)
-
-
def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id,
keycloak_client_secret, user, hostname, step_cert_sans):
try:
diff --git a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
index a692145..4af93ff 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
@@ -24,7 +24,6 @@
import argparse
import json
import sys
-from datalab.edge_lib import configure_http_proxy_server
from fabric import *
from datalab.fab import *
@@ -48,6 +47,6 @@ if __name__ == "__main__":
sys.exit(2)
print("Installing proxy for notebooks.")
- configure_http_proxy_server(deeper_config)
+ datalab.fab.configure_http_proxy_server(deeper_config)
conn.close()
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/project/scripts/configure_nftables.py b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
index b3c24a9..8fe14cd 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_nftables.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
@@ -24,7 +24,6 @@
import argparse
import json
import sys
-from datalab.edge_lib import configure_nftables
from fabric import *
from datalab.fab import *
@@ -48,5 +47,5 @@ if __name__ == "__main__":
sys.exit(2)
print("Configuring nftables on edge node.")
- configure_nftables(deeper_config)
+ datalab.fab.configure_nftables(deeper_config)
conn.close()
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/project/templates/squid.conf b/infrastructure-provisioning/src/project/templates/squid.conf
index 0948b46..39a6cbf 100644
--- a/infrastructure-provisioning/src/project/templates/squid.conf
+++ b/infrastructure-provisioning/src/project/templates/squid.conf
@@ -19,8 +19,6 @@
#
# ******************************************************************************
-#auth_param basic program LDAP_AUTH_PATH -b "LDAP_DN" -D "LDAP_SERVICE_USERNAME,LDAP_DN" -w LDAP_SERVICE_PASSWORD -f uid=%s LDAP_HOST
-
acl DataLab_user_src_subnet src PROXY_SUBNET
VPC_CIDRS
ALLOWED_CIDRS
@@ -40,13 +38,11 @@ acl Safe_ports port 591 # filemaker
acl Safe_ports port 777 # multiling http
acl CONNECT method CONNECT
-#acl ldap-auth proxy_auth EDGE_USER_NAME
-
http_access deny !Safe_ports
http_access allow localhost manager
http_access deny manager
http_access allow DataLab_user_src_subnet
-http_access allow AllowedCIDRS ldap-auth
+http_access allow AllowedCIDRS
http_access allow localhost
http_access deny all
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org