You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by yu...@apache.org on 2024/12/23 08:39:12 UTC
(incubator-gluten) branch main updated: [GLUTEN-7641][VL] Fix security issue for perf analysis scripts (#8309)
This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b8279f9623 [GLUTEN-7641][VL] Fix security issue for perf analysis scripts (#8309)
b8279f9623 is described below
commit b8279f96235a752fb16e7c5e8485a7595f5b4d66
Author: Rong Ma <ro...@intel.com>
AuthorDate: Mon Dec 23 16:39:08 2024 +0800
[GLUTEN-7641][VL] Fix security issue for perf analysis scripts (#8309)
Fix security issue for perf analysis scripts and some minor fixes to the scripts
---
.../benchmark_velox/analysis/requirements.txt | 30 +++++------
.../benchmark_velox/analysis/sparklog.ipynb | 6 +--
tools/workload/benchmark_velox/initialize.ipynb | 62 ++++++++++++++--------
3 files changed, 57 insertions(+), 41 deletions(-)
diff --git a/tools/workload/benchmark_velox/analysis/requirements.txt b/tools/workload/benchmark_velox/analysis/requirements.txt
index f230f8aa99..5461e5277d 100644
--- a/tools/workload/benchmark_velox/analysis/requirements.txt
+++ b/tools/workload/benchmark_velox/analysis/requirements.txt
@@ -5,24 +5,23 @@ argon2-cffi-bindings==21.2.0
arrow==1.3.0
asttokens==2.4.1
attrs==23.2.0
-Automat==20.2.0
-Babel==2.8.0
+Babel>=2.9.1
bcrypt==3.2.0
beautifulsoup4==4.12.3
black==24.4.2
bleach==6.1.0
blinker==1.4
-certifi==2020.6.20
+certifi>=2023.7.22
cffi==1.16.0
chardet==4.0.0
charset-normalizer==3.4.0
click==8.0.3
colorama==0.4.4
comm==0.2.2
-configobj==5.0.6
+configobj>=5.0.9
constantly==15.1.0
contourpy==1.2.1
-cryptography==3.4.8
+cryptography>=42.0.2
cycler==0.12.1
debugpy==1.8.1
decorator==5.1.1
@@ -43,7 +42,6 @@ httplib2==0.20.2
hyperlink==21.0.0
idna==3.10
importlib-metadata==4.6.4
-incremental==21.3.0
ipykernel==6.29.4
ipython==8.24.0
ipython-genutils==0.2.0
@@ -51,7 +49,7 @@ ipywidgets==8.1.3
isoduration==20.11.0
jedi==0.19.1
jeepney==0.7.1
-Jinja2==3.0.3
+Jinja2>=3.1.4
jsonpatch==1.32
jsonpointer==2.0
jsonschema==4.22.0
@@ -63,7 +61,7 @@ jupyter_core==5.7.2
jupyter-events==0.10.0
jupyter-highlight-selected-word==0.2.0
jupyter-nbextensions-configurator==0.6.3
-jupyter_server==2.14.0
+jupyter_server>=2.14.1
jupyter-server-mathjax==0.2.6
jupyter_server_terminals==0.5.3
jupyterlab_pygments==0.3.0
@@ -92,7 +90,7 @@ notebook==6.5.6
notebook_shim==0.2.4
NotebookScripter==6.0.0
numpy==1.26.4
-oauthlib==3.2.0
+oauthlib>=3.2.2
overrides==7.7.0
packaging==24.0
pandas==1.5.3
@@ -114,15 +112,15 @@ pyarrow==16.1.0
pyasn1==0.4.8
pyasn1-modules==0.2.1
pycparser==2.22
-Pygments==2.11.2
+Pygments>=2.15.0
PyHamcrest==2.0.2
PyHDFS==0.3.1
-PyJWT==2.3.0
+pyjwt>=2.4.0
pyOpenSSL==21.0.0
pyparsing==2.4.7
pyrsistent==0.18.1
pyserial==3.5
-pyspark==3.3.1
+pyspark>=3.3.2
python-dateutil==2.9.0.post0
python-json-logger==2.0.7
pytz==2022.1
@@ -154,15 +152,15 @@ termcolor==2.5.0
terminado==0.18.1
tinycss2==1.3.0
tomli==2.0.1
-tornado==6.4
+tornado>=6.4.2
tqdm==4.66.4
traitlets==5.14.3
-Twisted==22.1.0
+twisted>=24.7.0rc1
types-python-dateutil==2.9.0.20240316
typing_extensions==4.12.0
tzdata==2024.1
uri-template==1.3.0
-urllib3==1.26.5
+urllib3>=1.26.19
wadllib==1.3.6
wcwidth==0.2.13
webcolors==1.13
@@ -170,5 +168,5 @@ webencodings==0.5.1
websocket-client==1.8.0
wheel==0.44.0
widgetsnbextension==4.0.11
-zipp==1.0.0
+zipp>=3.19.1
zope.interface==5.4.0
diff --git a/tools/workload/benchmark_velox/analysis/sparklog.ipynb b/tools/workload/benchmark_velox/analysis/sparklog.ipynb
index 4ecce64f99..fe671a3d6f 100644
--- a/tools/workload/benchmark_velox/analysis/sparklog.ipynb
+++ b/tools/workload/benchmark_velox/analysis/sparklog.ipynb
@@ -268,7 +268,7 @@
" with open(outputfolder, 'w') as outfile: \n",
" outfile.write(output)\n",
" \n",
- " traceview_link=f'http://{localhost}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{appidx}.json'\n",
+ " traceview_link=f'http://{local_ip}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{appidx}.json'\n",
" display(HTML(f\"<a href={traceview_link}>{traceview_link}</a>\"))\n",
" return traceview_link"
]
@@ -5219,7 +5219,7 @@
" with open('/home/sparkuser/trace_result/'+self.appid+'.json', 'w') as outfile: \n",
" outfile.write(output)\n",
" \n",
- " traceview_link=f'http://{localhost}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{self.appid}.json'\n",
+ " traceview_link=f'http://{local_ip}:1088/tracing_examples/trace_viewer.html#/tracing/test_data/{self.appid}.json'\n",
" display(HTML(f\"<a href={traceview_link}>{traceview_link}</a>\"))\n",
" return traceview_link\n",
"\n",
@@ -5774,7 +5774,7 @@
" if fs.exists(changelog):\n",
" with fs.open(changelog) as f:\n",
" for l in f.readlines():\n",
- " l = l.decode('ascii')\n",
+ " l = l.decode('utf-8')\n",
" if l.startswith(\"commit\"):\n",
" out.append(re.sub(r\"commit +(.+)\",r\"<font color=#BDCA57>commit </font><font color=#23C2BF>\\1</font>\",l))\n",
" elif l.startswith(\"Author\"):\n",
diff --git a/tools/workload/benchmark_velox/initialize.ipynb b/tools/workload/benchmark_velox/initialize.ipynb
index 30574f8c16..0f84da52cc 100644
--- a/tools/workload/benchmark_velox/initialize.ipynb
+++ b/tools/workload/benchmark_velox/initialize.ipynb
@@ -2976,18 +2976,26 @@
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {
"hidden": true
},
+ "outputs": [],
"source": [
- "\n",
- "```bash\n",
- "cd ~\n",
- "virtualenv -p python3 -v paus-env\n",
- "source paus-env/bin/activate\n",
- "python3 -m pip install -r ~/gluten/tools/workload/benchmark_velox/analysis/requirements.txt\n",
- "```"
+ "%cd ~"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "hidden": true
+ },
+ "outputs": [],
+ "source": [
+ "!virtualenv -p python3 -v paus-env\n",
+ "!source ~/paus-env/bin/activate && python3 -m pip install -r ~/gluten/tools/workload/benchmark_velox/analysis/requirements.txt"
]
},
{
@@ -2996,15 +3004,18 @@
"hidden": true
},
"source": [
- "\n",
- "We will put all perf analysis notebooks under `$HOME/PAUS`. Create the directory and start the notebook:\n",
- "\n",
- "```bash\n",
- "mkdir -p ~/PAUS\n",
- "cd ~/PAUS\n",
- "nohup jupyter notebook --ip=0.0.0.0 --port=8889 &\n",
- "```\n",
- "\n"
+ "We will put all perf analysis notebooks under `$HOME/PAUS`. Create the directory and start the notebook:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "hidden": true
+ },
+ "outputs": [],
+ "source": [
+ "!mkdir -p ~/PAUS && cd ~/PAUS && source ~/paus-env/bin/activate && nohup jupyter notebook --ip=0.0.0.0 --port=8889 &"
]
},
{
@@ -3013,11 +3024,18 @@
"hidden": true
},
"source": [
- "Package the virtual environment so that it can be distributed to other nodes:\n",
- "```bash\n",
- "cd ~\n",
- "tar -czf paus-env.tar.gz paus-env\n",
- "```"
+ "Package the virtual environment so that it can be distributed to other nodes:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "hidden": true
+ },
+ "outputs": [],
+ "source": [
+ "!cd ~ && tar -czf paus-env.tar.gz paus-env"
]
},
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org