You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/08/18 09:08:08 UTC
[incubator-datalab] branch DATALAB-2409 updated (5f241f1 -> 645e7b6)
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a change to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
from 5f241f1 [DATALAB-2409] - debian common lib refactored
new 6ca8653 [DATALAB-2409] - debian edge lib refactored
new 645e7b6 [DATALAB-2409] - edge lib refactored
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/edge/scripts/configure_http_proxy.py | 3 +-
.../src/general/lib/os/debian/edge_lib.py | 56 +---------------------
.../src/general/lib/os/fab.py | 51 ++++++++++++++++++++
.../src/general/lib/os/redhat/edge_lib.py | 31 ------------
.../src/project/scripts/configure_http_proxy.py | 3 +-
.../src/project/scripts/configure_nftables.py | 3 +-
.../src/project/templates/squid.conf | 6 +--
7 files changed, 57 insertions(+), 96 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-2409] - debian edge lib
refactored
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6ca86537a4c2561741174272e0831fe585f942ca
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Mon Aug 16 12:36:26 2021 +0300
[DATALAB-2409] - debian edge lib refactored
---
.../src/general/lib/os/debian/edge_lib.py | 14 +++++++-------
.../src/project/templates/squid.conf | 4 ++--
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index b5ad1e6..7a91691 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -37,13 +37,13 @@ def configure_http_proxy_server(config):
proxy_subnet = config['exploratory_subnet']
datalab.fab.conn.put(template_file, '/tmp/squid.conf')
datalab.fab.conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
- datalab.fab.conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
- datalab.fab.conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name']))
- datalab.fab.conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host']))
- datalab.fab.conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn']))
- datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user']))
- datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password']))
- datalab.fab.conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib/squid/basic_ldap_auth'))
+# datalab.fab.conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
+# datalab.fab.conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name']))
+# datalab.fab.conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host']))
+# datalab.fab.conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn']))
+# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user']))
+# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password']))
+# datalab.fab.conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib/squid/basic_ldap_auth'))
replace_string = ''
for cidr in config['vpc_cidrs']:
replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
diff --git a/infrastructure-provisioning/src/project/templates/squid.conf b/infrastructure-provisioning/src/project/templates/squid.conf
index 9d45c44..0948b46 100644
--- a/infrastructure-provisioning/src/project/templates/squid.conf
+++ b/infrastructure-provisioning/src/project/templates/squid.conf
@@ -19,7 +19,7 @@
#
# ******************************************************************************
-auth_param basic program LDAP_AUTH_PATH -b "LDAP_DN" -D "LDAP_SERVICE_USERNAME,LDAP_DN" -w LDAP_SERVICE_PASSWORD -f uid=%s LDAP_HOST
+#auth_param basic program LDAP_AUTH_PATH -b "LDAP_DN" -D "LDAP_SERVICE_USERNAME,LDAP_DN" -w LDAP_SERVICE_PASSWORD -f uid=%s LDAP_HOST
acl DataLab_user_src_subnet src PROXY_SUBNET
VPC_CIDRS
@@ -40,7 +40,7 @@ acl Safe_ports port 591 # filemaker
acl Safe_ports port 777 # multiling http
acl CONNECT method CONNECT
-acl ldap-auth proxy_auth EDGE_USER_NAME
+#acl ldap-auth proxy_auth EDGE_USER_NAME
http_access deny !Safe_ports
http_access allow localhost manager
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-2409] - edge lib refactored
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 645e7b63f67406d9e81c78cc71638c0f0384eb6e
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Wed Aug 18 12:07:50 2021 +0300
[DATALAB-2409] - edge lib refactored
---
.../src/edge/scripts/configure_http_proxy.py | 3 +-
.../src/general/lib/os/debian/edge_lib.py | 56 +---------------------
.../src/general/lib/os/fab.py | 51 ++++++++++++++++++++
.../src/general/lib/os/redhat/edge_lib.py | 31 ------------
.../src/project/scripts/configure_http_proxy.py | 3 +-
.../src/project/scripts/configure_nftables.py | 3 +-
.../src/project/templates/squid.conf | 6 +--
7 files changed, 57 insertions(+), 96 deletions(-)
diff --git a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
index 3580b43..0e9034e 100644
--- a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
+++ b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
@@ -24,7 +24,6 @@
import argparse
import json
import sys
-from datalab.edge_lib import configure_http_proxy_server
from fabric import *
from datalab.fab import *
@@ -48,5 +47,5 @@ if __name__ == "__main__":
sys.exit(2)
print("Installing proxy for notebooks.")
- configure_http_proxy_server(deeper_config)
+ datalab.fab.configure_http_proxy_server(deeper_config)
conn.close()
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 7a91691..34d1273 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -23,41 +23,12 @@
import os
import sys
-from datalab.common_lib import manage_pkg
from fabric import *
from patchwork.files import exists
from patchwork import files
import datalab.fab
-
-def configure_http_proxy_server(config):
- try:
- if not exists(datalab.fab.conn,'/tmp/http_proxy_ensured'):
- manage_pkg('-y install', 'remote', 'squid')
- template_file = config['template_file']
- proxy_subnet = config['exploratory_subnet']
- datalab.fab.conn.put(template_file, '/tmp/squid.conf')
- datalab.fab.conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
-# datalab.fab.conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
-# datalab.fab.conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password']))
-# datalab.fab.conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib/squid/basic_ldap_auth'))
- replace_string = ''
- for cidr in config['vpc_cidrs']:
- replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
- datalab.fab.conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- replace_string = ''
- for cidr in config['allowed_ip_cidr']:
- replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
- datalab.fab.conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- datalab.fab.conn.sudo('systemctl restart squid')
- datalab.fab.conn.sudo('touch /tmp/http_proxy_ensured')
- except Exception as err:
- print("Failed to install and configure squid: " + str(err))
- sys.exit(1)
-
+from datalab.common_lib import manage_pkg
+from datalab.logger import logging
def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id,
keycloak_client_secret, user, hostname, step_cert_sans):
@@ -189,27 +160,4 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
datalab.fab.configure_nginx_LE(os.environ['conf_letsencrypt_domain_name'], os.environ['project_name'].lower())
except Exception as err:
print("Failed install nginx with ldap: " + str(err))
- sys.exit(1)
-
-def configure_nftables(config):
- try:
- if not exists(datalab.fab.conn,'/tmp/nftables_ensured'):
- manage_pkg('-y install', 'remote', 'nftables')
- datalab.fab.conn.sudo('systemctl enable nftables.service')
- datalab.fab.conn.sudo('systemctl start nftables')
- datalab.fab.conn.sudo('sysctl net.ipv4.ip_forward=1')
- if os.environ['conf_cloud_provider'] == 'aws':
- interface = 'eth0'
- elif os.environ['conf_cloud_provider'] == 'gcp':
- interface = 'ens4'
- datalab.fab.conn.sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf')
- datalab.fab.conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip']))
- datalab.fab.conn.sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface))
- datalab.fab.conn.sudo(
- 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet']))
- datalab.fab.conn.sudo('cp /opt/datalab/templates/nftables.conf /etc/')
- datalab.fab.conn.sudo('systemctl restart nftables')
- datalab.fab.conn.sudo('touch /tmp/nftables_ensured')
- except Exception as err:
- print("Failed to configure nftables: " + str(err))
sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 40c6c92..707bc60 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -255,6 +255,57 @@ def configure_nginx_LE(domain_name, node):
sys.exit(1)
+#function for edge node only
+def configure_http_proxy_server(config):
+ try:
+ if not exists(datalab.fab.conn,'/tmp/http_proxy_ensured'):
+ manage_pkg('-y install', 'remote', 'squid')
+ template_file = config['template_file']
+ proxy_subnet = config['exploratory_subnet']
+ conn.put(template_file, '/tmp/squid.conf')
+ conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
+ conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
+ replace_string = ''
+ for cidr in config['vpc_cidrs']:
+ replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
+ conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
+ replace_string = ''
+ for cidr in config['allowed_ip_cidr']:
+ replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
+ conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
+ conn.sudo('systemctl restart squid')
+ fab.conn.sudo('touch /tmp/http_proxy_ensured')
+ except Exception as err:
+ logging.error('Fai to install and configure squid:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configure_nftables(config):
+ try:
+ if not exists(datalab.fab.conn,'/tmp/nftables_ensured'):
+ manage_pkg('-y install', 'remote', 'nftables')
+ conn.sudo('systemctl enable nftables.service')
+ conn.sudo('systemctl start nftables')
+ conn.sudo('sysctl net.ipv4.ip_forward=1')
+ if os.environ['conf_cloud_provider'] == 'aws':
+ interface = 'eth0'
+ elif os.environ['conf_cloud_provider'] == 'gcp':
+ interface = 'ens4'
+ conn.sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf')
+ conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip']))
+ conn.sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface))
+ conn.sudo(
+ 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet']))
+ conn.sudo('cp /opt/datalab/templates/nftables.conf /etc/')
+ conn.sudo('systemctl restart nftables')
+ conn.sudo('touch /tmp/nftables_ensured')
+ except Exception as err:
+ logging.error('Failed to configure nftables:', (err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
# functions for all computation resources
def ensure_python_venv(python_venv_version):
try:
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index ae81a2b..7617419 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -29,37 +29,6 @@ from patchwork.files import exists
from patchwork import files
-def configure_http_proxy_server(config):
- try:
- if not exists(conn,'/tmp/http_proxy_ensured'):
- manage_pkg('-y install', 'remote', 'squid')
- template_file = config['template_file']
- proxy_subnet = config['exploratory_subnet']
- conn.put(template_file, '/tmp/squid.conf')
- conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
- conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
- conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name']))
- conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host']))
- conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn']))
- conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user']))
- conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password']))
- conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib64/squid/basic_ldap_auth'))
- replace_string = ''
- for cidr in config['vpc_cidrs']:
- replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
- conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- replace_string = ''
- for cidr in config['allowed_ip_cidr']:
- replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
- conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
- conn.sudo('systemctl restart squid')
- conn.sudo('chkconfig squid on')
- conn.sudo('touch /tmp/http_proxy_ensured')
- except Exception as err:
- print("Failed to install and configure squid: " + str(err))
- sys.exit(1)
-
-
def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id,
keycloak_client_secret, user, hostname, step_cert_sans):
try:
diff --git a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
index a692145..4af93ff 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
@@ -24,7 +24,6 @@
import argparse
import json
import sys
-from datalab.edge_lib import configure_http_proxy_server
from fabric import *
from datalab.fab import *
@@ -48,6 +47,6 @@ if __name__ == "__main__":
sys.exit(2)
print("Installing proxy for notebooks.")
- configure_http_proxy_server(deeper_config)
+ datalab.fab.configure_http_proxy_server(deeper_config)
conn.close()
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/project/scripts/configure_nftables.py b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
index b3c24a9..8fe14cd 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_nftables.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
@@ -24,7 +24,6 @@
import argparse
import json
import sys
-from datalab.edge_lib import configure_nftables
from fabric import *
from datalab.fab import *
@@ -48,5 +47,5 @@ if __name__ == "__main__":
sys.exit(2)
print("Configuring nftables on edge node.")
- configure_nftables(deeper_config)
+ datalab.fab.configure_nftables(deeper_config)
conn.close()
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/project/templates/squid.conf b/infrastructure-provisioning/src/project/templates/squid.conf
index 0948b46..39a6cbf 100644
--- a/infrastructure-provisioning/src/project/templates/squid.conf
+++ b/infrastructure-provisioning/src/project/templates/squid.conf
@@ -19,8 +19,6 @@
#
# ******************************************************************************
-#auth_param basic program LDAP_AUTH_PATH -b "LDAP_DN" -D "LDAP_SERVICE_USERNAME,LDAP_DN" -w LDAP_SERVICE_PASSWORD -f uid=%s LDAP_HOST
-
acl DataLab_user_src_subnet src PROXY_SUBNET
VPC_CIDRS
ALLOWED_CIDRS
@@ -40,13 +38,11 @@ acl Safe_ports port 591 # filemaker
acl Safe_ports port 777 # multiling http
acl CONNECT method CONNECT
-#acl ldap-auth proxy_auth EDGE_USER_NAME
-
http_access deny !Safe_ports
http_access allow localhost manager
http_access deny manager
http_access allow DataLab_user_src_subnet
-http_access allow AllowedCIDRS ldap-auth
+http_access allow AllowedCIDRS
http_access allow localhost
http_access deny all
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org