You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by hu...@apache.org on 2019/08/09 14:21:22 UTC
[climate] branch master updated (8ba9330 -> 81c6501)
This is an automated email from the ASF dual-hosted git repository.
huikyole pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/climate.git.
from 8ba9330 Merge branch 'CLIMATE-965'
new 40dd07f CLIMATE-966 - Jupyter Notebook to run cordex.py on AWS
new d943a72 agoodm's suggested changes have been accepted
new 81c6501 Merge branch 'CLIMATE-966'
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
RCMES/CORDEX/CORDEX.ipynb | 166 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 166 insertions(+)
create mode 100644 RCMES/CORDEX/CORDEX.ipynb
[climate] 02/03: agoodm's suggested changes have been accepted
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huikyole pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/climate.git
commit d943a723853c1d65197d46394472a736d8a432ca
Author: huikyole <hu...@jpl.nasa.gov>
AuthorDate: Thu Aug 8 22:13:39 2019 -0700
agoodm's suggested changes have been accepted
---
RCMES/CORDEX/CORDEX.ipynb | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/RCMES/CORDEX/CORDEX.ipynb b/RCMES/CORDEX/CORDEX.ipynb
index 9574260..2cef98e 100644
--- a/RCMES/CORDEX/CORDEX.ipynb
+++ b/RCMES/CORDEX/CORDEX.ipynb
@@ -14,7 +14,7 @@
"from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor\n",
"from tqdm import tqdm_notebook as tqdm\n",
"from glob import glob\n",
- "from IPython.display import Markdown, Image"
+ "from IPython.display import Markdown, Image, FileLink"
]
},
{
@@ -132,13 +132,13 @@
" var_name = obs_info['variable']\n",
" display(Markdown('Instrument: '+inst_name+'& Variable: '+var_name))\n",
" for season in seasons:\n",
- " os.chdir(workdir+'/'+domain+'/'+inst_name+'/'+var_name+'/'+season)\n",
- " png_files = glob('*.png')\n",
+ " savedir = os.path.join('evaluation_result', domain, inst_name, var_name, season)\n",
+ " png_files = glob(os.path.join(savedir, '*.png'))\n",
" for png_file in png_files: \n",
- " display(Image(os.getcwd()+'/'+png_file))\n",
- " nc_file = glob('*.nc')[0]\n",
- " display(Markdown('['+nc_file+'](http://'+ip_address+':8888'+os.getcwd()+'/'+nc_file+')'))\n",
- "os.chdir(cwd) \n"
+ " display(Image(png_file))\n",
+ " nc_file = glob(os.path.join(savedir, '*.nc'))[0]\n",
+ " display(FileLink(nc_file))\n",
+ "os.chdir(cwd) \n"
]
}
],
[climate] 01/03: CLIMATE-966 - Jupyter Notebook to run cordex.py on
AWS
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huikyole pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/climate.git
commit 40dd07f7eb8dd4af35a4100fe6996be028319704
Author: huikyole <hu...@jpl.nasa.gov>
AuthorDate: Wed Aug 7 21:08:37 2019 -0700
CLIMATE-966 - Jupyter Notebook to run cordex.py on AWS
---
RCMES/CORDEX/CORDEX.ipynb | 166 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 166 insertions(+)
diff --git a/RCMES/CORDEX/CORDEX.ipynb b/RCMES/CORDEX/CORDEX.ipynb
new file mode 100644
index 0000000..9574260
--- /dev/null
+++ b/RCMES/CORDEX/CORDEX.ipynb
@@ -0,0 +1,166 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import sys\n",
+ "import subprocess\n",
+ "import jinja2\n",
+ "import requests\n",
+ "from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor\n",
+ "from tqdm import tqdm_notebook as tqdm\n",
+ "from glob import glob\n",
+ "from IPython.display import Markdown, Image"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "domain should be one of these three: 'AFR-44', 'EUR-11', 'NAM-44'\n",
+ "* AFR-44: CORDEX Africa RCMs at 44 km resolution\n",
+ "* EUR-11: CORDEX Europe RCMs at 11 km resolution\n",
+ "* NAM-44: CORDEX North America RCMs at 44 km resolution"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "domain = 'NAM-44'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The output directory\n",
+ "cwd = os.getcwd()\n",
+ "workdir = cwd +'/evaluation_result'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Location of obs4Mips files\n",
+ "obs_dir = '/mnt/efs/obs4mips'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Location of CORDEX files\n",
+ "models_dir = '/mnt/efs/'+domain+'/*'.format(domain=domain)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract metadata from model and obs files, pairing up files with the same\n",
+ "# variables for separate evaluations\n",
+ "obs_extractor = obs4MIPSMetadataExtractor(obs_dir)\n",
+ "models_extractor = CORDEXMetadataExtractor(models_dir)\n",
+ "groups = obs_extractor.group(models_extractor, 'variable')\n",
+ "\n",
+ "# Configuration file template, to be rendered repeatedly for each evaluation\n",
+ "# run\n",
+ "env = jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),\n",
+ " trim_blocks=True, lstrip_blocks=True)\n",
+ "t = env.get_template('CORDEX.yaml.template')\n",
+ "\n",
+ "# Each group represents a single evaluation. Repeat the evaluation for\n",
+ "# three seasons: Summer, Winter, and Annual.\n",
+ "seasons = ['annual', 'winter', 'summer']\n",
+ "errored = []\n",
+ "for group in tqdm(groups, desc='variable loop'):\n",
+ " obs_info, models_info = group\n",
+ " instrument = obs_info['instrument']\n",
+ " variable = obs_info['variable']\n",
+ " for season in tqdm(seasons, desc='season loop'):\n",
+ " configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'\n",
+ " configfile_path = os.path.join(workdir, domain, instrument,\n",
+ " variable, season)\n",
+ " if not os.path.exists(configfile_path):\n",
+ " os.makedirs(configfile_path)\n",
+ " configfile_path = os.path.join(configfile_path, configfile_basename)\n",
+ " with open(configfile_path, 'w') as configfile:\n",
+ " configfile.write(t.render(obs_info=obs_info, models_info=models_info,\n",
+ " season=season, output_dir=workdir))\n",
+ "\n",
+ " # TODO: Do this in parallel. Will change this once this approach\n",
+ " # is well tested.\n",
+ " code = subprocess.call([sys.executable, '../run_RCMES.py', configfile_path])\n",
+ " if code:\n",
+ " errored.append(configfile_path)\n",
+ "print(\"All runs done. The following ended with an error: {}\".format(errored))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Check the evaluation result or download the processed obs4mips and model output."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown('Evaluation results'))\n",
+ "ip_address = (requests.get('http://169.254.169.254/latest/meta-data/public-ipv4').content).decode('utf-8')\n",
+ "for obs_info in obs_extractor.data:\n",
+ " inst_name = obs_info['instrument']\n",
+ " var_name = obs_info['variable']\n",
+ " display(Markdown('Instrument: '+inst_name+'& Variable: '+var_name))\n",
+ " for season in seasons:\n",
+ " os.chdir(workdir+'/'+domain+'/'+inst_name+'/'+var_name+'/'+season)\n",
+ " png_files = glob('*.png')\n",
+ " for png_file in png_files: \n",
+ " display(Image(os.getcwd()+'/'+png_file))\n",
+ " nc_file = glob('*.nc')[0]\n",
+ " display(Markdown('['+nc_file+'](http://'+ip_address+':8888'+os.getcwd()+'/'+nc_file+')'))\n",
+ "os.chdir(cwd) \n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
[climate] 03/03: Merge branch 'CLIMATE-966'
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huikyole pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/climate.git
commit 81c65012db25a0696da286a6b2daebbf9cb5fdbc
Merge: 8ba9330 d943a72
Author: huikyole <hu...@argo.jpl.nasa.gov>
AuthorDate: Fri Aug 9 07:21:04 2019 -0700
Merge branch 'CLIMATE-966'
RCMES/CORDEX/CORDEX.ipynb | 166 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 166 insertions(+)