You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by go...@apache.org on 2017/09/29 06:50:41 UTC
[1/2] climate git commit: CLIMATE-927 - RCMES script for running
multiple evaluations
Repository: climate
Updated Branches:
refs/heads/master 77463b926 -> 58d6cdf16
CLIMATE-927 - RCMES script for running multiple evaluations
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fb02c749
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fb02c749
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fb02c749
Branch: refs/heads/master
Commit: fb02c74925b977abd2c03f19ea0ccd9b15faec23
Parents: 77463b9
Author: Alex <ag...@users.noreply.github.com>
Authored: Thu Sep 28 23:11:51 2017 -0700
Committer: Alex <ag...@users.noreply.github.com>
Committed: Thu Sep 28 23:11:51 2017 -0700
----------------------------------------------------------------------
RCMES/CORDEX/cordex.py | 56 ++++++++++++++++++
RCMES/CORDEX/metadata_extractor.py | 35 +++++++++--
RCMES/CORDEX/templates/CORDEX.yaml.template | 8 +--
RCMES/metrics_and_plots.py | 2 +-
RCMES/run_RCMES.py | 24 +++++---
ocw/plotter.py | 74 ++++++++++++------------
6 files changed, 143 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/cordex.py
----------------------------------------------------------------------
diff --git a/RCMES/CORDEX/cordex.py b/RCMES/CORDEX/cordex.py
new file mode 100644
index 0000000..dc80be4
--- /dev/null
+++ b/RCMES/CORDEX/cordex.py
@@ -0,0 +1,56 @@
+import os
+import subprocess
+import jinja2
+from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor
+
+# These should be modified. TODO: domains can also be made into separate group
+# CORDEX domain
+domain = 'NAM-44'
+
+# The output directory
+workdir = '/home/goodman/data_processing/CORDEX/analysis'
+
+# Location of osb4Mips files
+obs_dir = '/proj3/data/obs4mips'
+
+# Location of CORDEX files
+models_dir = '/proj3/data/CORDEX/{domain}/*'.format(domain=domain)
+
+# Extract metadata from model and obs files, pairing up files with the same
+# variables for separate evaluations
+obs_extractor = obs4MIPSMetadataExtractor(obs_dir)
+models_extractor = CORDEXMetadataExtractor(models_dir)
+groups = obs_extractor.group(models_extractor, 'variable')
+
+# Configuration file template, to be rendered repeatedly for each evaluation
+# run
+env = jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),
+ trim_blocks=True, lstrip_blocks=True)
+t = env.get_template('CORDEX.yaml.template')
+
+# Each group represents a single evaluation. Repeat the evaluation for
+# three seasons: Summer, Winter, and Annual.
+seasons = ['annual', 'winter', 'summer']
+for group in groups:
+ obs_info, models_info = group
+ instrument = obs_info['instrument']
+ variable = obs_info['variable']
+ for season in seasons:
+ configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'
+ configfile_path = os.path.join(workdir, domain, instrument,
+ variable, season)
+ if not os.path.exists(configfile_path):
+ os.makedirs(configfile_path)
+ configfile_path = os.path.join(configfile_path, configfile_basename)
+ with open(configfile_path, 'w') as configfile:
+ configfile.write(t.render(obs_info=obs_info, models_info=models_info,
+ season=season, output_dir=workdir))
+
+ # TODO: Do this in parallel. Will change this once this approach
+ # is well tested.
+ code = subprocess.call(['python', '../run_RCMES.py', configfile_path])
+ errored = []
+ if code:
+ errored.append(configfile_path)
+
+print("All runs done. The following ended with an error: {}".format(errored))
http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/metadata_extractor.py
----------------------------------------------------------------------
diff --git a/RCMES/CORDEX/metadata_extractor.py b/RCMES/CORDEX/metadata_extractor.py
index cb18946..b97d890 100644
--- a/RCMES/CORDEX/metadata_extractor.py
+++ b/RCMES/CORDEX/metadata_extractor.py
@@ -84,6 +84,13 @@ class MetadataExtractor(object):
"""
return self.get_field('variable')
+ @property
+ def field_filters(self):
+ """
+ Override this to filter out specific characters contained in a field.
+ """
+ return dict()
+
def query(self, **kwargs):
"""
Narrow down the list of files by field names.
@@ -95,7 +102,8 @@ class MetadataExtractor(object):
data = self.data
for field, value in kwargs.items():
value = value if isinstance(value, list) else [value]
- data = [meta for meta in data if meta[field] in value]
+ data = [meta for meta in data
+ if self._match_filter(meta, field) in value]
return data
def group(self, extractor, field):
@@ -116,7 +124,7 @@ class MetadataExtractor(object):
groups = []
for meta in results:
- val = meta[field]
+ val = self._match_filter(meta, field)
kwargs.update({field: val})
match = extractor.query(**kwargs)
groups.append((meta, match))
@@ -151,6 +159,16 @@ class MetadataExtractor(object):
pattern = '_'.join(base[:len(self.fields)] + ['*.nc'])
return pattern
+ def _match_filter(self, meta, field):
+ """
+ Filter (ignore) certain character patterns when matching a field.
+ """
+ val = meta[field]
+ if field in self.field_filters:
+ for pattern in self.field_filters[field]:
+ val = val.replace(pattern, '')
+ return val
+
def _extract(self):
"""
Do the actual metadata extraction from the list of filename given
@@ -183,6 +201,13 @@ class obs4MIPSMetadataExtractor(MetadataExtractor):
fields = ['variable', 'instrument', 'processing_level', 'version']
return fields
+ @property
+ def field_filters(self):
+ """
+ Field filters for CALIPSO
+ """
+ return dict(variable=['calipso', 'Lidarsr532'])
+
def filter_filename(self, fname):
"""
CALIPSO files have odd naming conventions, so we will use
@@ -190,8 +215,6 @@ class obs4MIPSMetadataExtractor(MetadataExtractor):
"""
fname = os.path.basename(fname)
fname = fname.replace('_obs4MIPs_', '_')
- fname = fname.replace('calipso', '')
- fname = fname.replace('Lidarsr532', '')
return fname
def get_pattern(self, fname):
@@ -202,8 +225,8 @@ class obs4MIPSMetadataExtractor(MetadataExtractor):
offset = -2 if len(base) != 5 else -1
pattern = '_'.join(base[:offset] + ['*.nc'])
return pattern
-
-
+
+
class CORDEXMetadataExtractor(MetadataExtractor):
@property
def models(self):
http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/templates/CORDEX.yaml.template
----------------------------------------------------------------------
diff --git a/RCMES/CORDEX/templates/CORDEX.yaml.template b/RCMES/CORDEX/templates/CORDEX.yaml.template
index daf6ec5..c0a1994 100644
--- a/RCMES/CORDEX/templates/CORDEX.yaml.template
+++ b/RCMES/CORDEX/templates/CORDEX.yaml.template
@@ -1,6 +1,6 @@
{% set domain = models_info[0].domain %}
{% set instrument = obs_info.instrument %}
-{% set variable = obs_info.variable %}
+{% set variable = models_info[0].variable %}
{% set basename = [variable, instrument, domain, season]|join('_') %}
workdir: {{ [output_dir, domain, instrument, variable, season]|join('/') }}
output_netcdf_filename: {{ basename }}.nc
@@ -23,7 +23,7 @@ time:
average_each_year: True
space:
- boundary_type: {{ domain[:3] }}
+ boundary_type: CORDEX {{ domain[:3] }}
regrid:
regrid_on_reference: True
@@ -32,12 +32,12 @@ datasets:
- loader_name: local_split
name: {{ instrument }}
file_path: {{ obs_info.filename }}
- variable_name: {{ variable }}
+ variable_name: {{ obs_info.variable }}
{% for model_info in models_info %}
- loader_name: local_split
name: {{ model_info.model }}
file_path: {{ model_info.filename }}
- variable_name: {{ variable }}
+ variable_name: {{ model_info.variable }}
lat_name: lat
lon_name: lon
{% endfor %}
http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/metrics_and_plots.py
----------------------------------------------------------------------
diff --git a/RCMES/metrics_and_plots.py b/RCMES/metrics_and_plots.py
index 78c2f14..0eb1f02 100644
--- a/RCMES/metrics_and_plots.py
+++ b/RCMES/metrics_and_plots.py
@@ -59,7 +59,7 @@ def Map_plot_bias_of_multiyear_climatology(obs_dataset, obs_name, model_datasets
string_list = list(string.ascii_lowercase)
nmodels = len(model_datasets)
- row, column = plotter._best_grid_shape((row, column), nmodels + 1)
+ row, column = plotter._best_grid_shape(nmodels + 1, (row, column))
ax = fig.add_subplot(row,column,1)
if map_projection == 'npstere':
m = Basemap(ax=ax, projection ='npstere', boundinglat=lat_min, lon_0=0,
http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/run_RCMES.py
----------------------------------------------------------------------
diff --git a/RCMES/run_RCMES.py b/RCMES/run_RCMES.py
index 7641299..f9ec2c3 100644
--- a/RCMES/run_RCMES.py
+++ b/RCMES/run_RCMES.py
@@ -25,6 +25,11 @@ from datetime import datetime
from glob import glob
from getpass import getpass
import numpy as np
+
+# Need these lines to run RCMES through SSH without X11
+import matplotlib
+matplotlib.use('Agg')
+
import ocw.utils as utils
import ocw.dataset_processor as dsp
from ocw.dataset import Bounds
@@ -68,7 +73,7 @@ time_info = config['time']
temporal_resolution = time_info['temporal_resolution']
# Read time info
-maximum_overlap_period = space_info.get('maximum_overlap_period', False)
+maximum_overlap_period = time_info.get('maximum_overlap_period', False)
if not maximum_overlap_period:
start_time = datetime.strptime(time_info['start_time'].strftime('%Y%m%d'),'%Y%m%d')
end_time = datetime.strptime(time_info['end_time'].strftime('%Y%m%d'),'%Y%m%d')
@@ -86,7 +91,8 @@ if not 'boundary_type' in space_info:
else:
domain = space_info['boundary_type']
if domain.startswith('CORDEX '):
- domain = domain.replace('CORDEX ', '').lower()
+ domain = domain.replace('CORDEX', '').lower()
+ domain = domain.replace(' ', '')
min_lat, max_lat, min_lon, max_lon = utils.CORDEX_boundary(domain)
# Additional arguments for the DatasetLoader
@@ -110,9 +116,9 @@ multiplying_factor = np.ones(len(datasets))
multiplying_factor[0] = fact
names = [dataset.name for dataset in datasets]
for i, dataset in enumerate(datasets):
- if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
- datasets[i] = dsp.normalize_dataset_datetimes(dataset,
- temporal_resolution)
+ res = dataset.temporal_resolution()
+ if res == 'daily' or res == 'monthly':
+ datasets[i] = dsp.normalize_dataset_datetimes(dataset, res)
if multiplying_factor[i] != 1:
datasets[i].values *= multiplying_factor[i]
@@ -148,7 +154,7 @@ else:
for i, dataset in enumerate(datasets):
datasets[i] = dsp.subset(dataset, bounds)
if dataset.temporal_resolution() != temporal_resolution:
- datasets[i] = dsp.temporal_rebin(dataset, temporal_resolution)
+ datasets[i] = dsp.temporal_rebin(datasets[i], temporal_resolution)
# Temporally subset both observation and model datasets
# for the user specified season
@@ -269,8 +275,10 @@ if nmetrics > 0:
file_name = workdir+plot_info['file_name']
print('metrics {0}/{1}: {2}'.format(imetric, nmetrics, metrics_name))
+ default_shape = (int(np.ceil(np.sqrt(ntarget + 2))),
+ int(np.ceil(np.sqrt(ntarget + 2))))
if metrics_name == 'Map_plot_bias_of_multiyear_climatology':
- row, column = plot_info.get('subplots_array', (1, 1))
+ row, column = plot_info.get('subplots_array', default_shape)
if 'map_projection' in plot_info.keys():
Map_plot_bias_of_multiyear_climatology(
reference_dataset, reference_name, target_datasets, target_names,
@@ -287,7 +295,7 @@ if nmetrics > 0:
elif config['use_subregions']:
if (metrics_name == 'Timeseries_plot_subregion_interannual_variability'
and average_each_year):
- row, column = plot_info.get('subplots_array', (1, 1))
+ row, column = plot_info.get('subplots_array', default_shape)
Time_series_subregion(
reference_subregion_mean, reference_name, target_subregion_mean,
target_names, False, file_name, row, column,
http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/ocw/plotter.py
----------------------------------------------------------------------
diff --git a/ocw/plotter.py b/ocw/plotter.py
index 4896a64..7f9b092 100755
--- a/ocw/plotter.py
+++ b/ocw/plotter.py
@@ -80,12 +80,12 @@ def _nice_intervals(data, nlevs):
else:
mnlvl = mn
mxlvl = mx
-
+
# hack to make generated intervals from mpl the same for all versions
autolimit_mode = mpl.rcParams.get('axes.autolimit_mode')
if autolimit_mode:
mpl.rc('axes', autolimit_mode='round_numbers')
-
+
locator = mpl.ticker.MaxNLocator(nlevs)
clevs = locator.tick_values(mnlvl, mxlvl)
if autolimit_mode:
@@ -122,14 +122,14 @@ def _best_grid_shape(nplots, oldshape):
# rows and columns for gridshape, automatically
# correct it so that it fits only as many plots
# as needed
- while diff >= ncols:
- nrows -= 1
+ while diff >= nrows:
+ ncols -= 1
size = nrows * ncols
diff = size - nplots
# Don't forget to remove unnecessary columns too
- if nrows == 1:
- ncols = nplots
+ if ncols == 1:
+ nrows = nplots
newshape = nrows, ncols
return newshape
@@ -1096,24 +1096,24 @@ def draw_histogram(dataset_array, data_names, fname, fmt='png', nbins=10):
def fill_US_states_with_color(regions, fname, fmt='png', ptitle='',
colors=False, values=None, region_names=None):
-
- ''' Fill the States over the contiguous US with colors
-
- :param regions: The list of subregions(lists of US States) to be filled
+
+ ''' Fill the States over the contiguous US with colors
+
+ :param regions: The list of subregions(lists of US States) to be filled
with different colors.
- :type regions: :class:`list`
+ :type regions: :class:`list`
:param fname: The filename of the plot.
:type fname: :mod:`string`
:param fmt: (Optional) filetype for the output.
:type fmt: :mod:`string`
:param ptitle: (Optional) plot title.
:type ptitle: :mod:`string`
- :param colors: (Optional) : If True, each region will be filled
+ :param colors: (Optional) : If True, each region will be filled
with different colors without using values
- :type colors: :class:`bool`
- :param values: (Optional) : If colors==False, color for each region scales
- an associated element in values
- :type values: :class:`numpy.ndarray`
+ :type colors: :class:`bool`
+ :param values: (Optional) : If colors==False, color for each region scales
+ an associated element in values
+ :type values: :class:`numpy.ndarray`
'''
nregion = len(regions)
@@ -1140,7 +1140,7 @@ def fill_US_states_with_color(regions, fname, fmt='png', ptitle='',
lons=np.empty(0)
for shape in shapes:
patches.append(Polygon(np.array(shape), True))
-
+
lons = np.append(lons, shape[:,0])
lats = np.append(lats, shape[:,1])
if colors:
@@ -1164,13 +1164,13 @@ def draw_plot_to_compare_trends(obs_data, ens_data, model_data,
fname, fmt='png', ptitle='', data_labels=None,
xlabel='', ylabel=''):
- ''' Fill the States over the contiguous US with colors
-
+ ''' Fill the States over the contiguous US with colors
+
:param obs_data: An array of observed trend and standard errors for regions
:type obs_data: :class:'numpy.ndarray'
- :param ens_data: An array of trend and standard errors from a multi-model ensemble for regions
+ :param ens_data: An array of trend and standard errors from a multi-model ensemble for regions
:type ens_data: : class:'numpy.ndarray'
- :param model_data: An array of trends from models for regions
+ :param model_data: An array of trends from models for regions
:type model_data: : class:'numpy.ndarray'
:param fname: The filename of the plot.
:type fname: :mod:`string`
@@ -1182,36 +1182,36 @@ def draw_plot_to_compare_trends(obs_data, ens_data, model_data,
:type data_labels: :mod:`list`
:param xlabel: (Optional) a label for x-axis
:type xlabel: :mod:`string`
- :param ylabel: (Optional) a label for y-axis
+ :param ylabel: (Optional) a label for y-axis
:type ylabel: :mod:`string`
'''
- nregions = obs_data.shape[1]
+ nregions = obs_data.shape[1]
# Set up the figure
fig = plt.figure()
fig.set_size_inches((8.5, 11.))
fig.dpi = 300
ax = fig.add_subplot(111)
-
- b_plot = ax.boxplot(model_data, widths=np.repeat(0.2, nregions), positions=np.arange(nregions)+1.3)
+
+ b_plot = ax.boxplot(model_data, widths=np.repeat(0.2, nregions), positions=np.arange(nregions)+1.3)
plt.setp(b_plot['medians'], color='black')
plt.setp(b_plot['whiskers'], color='black')
plt.setp(b_plot['boxes'], color='black')
plt.setp(b_plot['fliers'], color='black')
- ax.errorbar(np.arange(nregions)+0.8, obs_data[0,:], yerr=obs_data[1,:],
- fmt='o', color='r', ecolor='r')
- ax.errorbar(np.arange(nregions)+1., ens_data[0,:], yerr=ens_data[1,:],
- fmt='o', color='b', ecolor='b')
+ ax.errorbar(np.arange(nregions)+0.8, obs_data[0,:], yerr=obs_data[1,:],
+ fmt='o', color='r', ecolor='r')
+ ax.errorbar(np.arange(nregions)+1., ens_data[0,:], yerr=ens_data[1,:],
+ fmt='o', color='b', ecolor='b')
ax.set_xticks(np.arange(nregions)+1)
ax.set_xlim([0, nregions+1])
-
+
if data_labels:
ax.set_xticklabels(data_labels)
- fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight')
+ fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight')
-def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_names,
- output_file, title=None, diff_plot=False, cmap = cm.BrBG,
- cbar_ticks=[0.01, 0.10, 0.5, 2, 5, 25],
+def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_names,
+ output_file, title=None, diff_plot=False, cmap = cm.BrBG,
+ cbar_ticks=[0.01, 0.10, 0.5, 2, 5, 25],
cbar_label=['0.01', '0.10', '0.5', '2', '5', '25']):
'''
:param plot_data: a numpy array of data to plot (dimY, dimX)
@@ -1219,11 +1219,11 @@ def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_n
:param plot_level: levels to plot
:type plot_level: :class:'numpy.ndarray'
:param x_ticks: x values where tick makrs are located
- :type x_ticks: :class:'numpy.ndarray'
+ :type x_ticks: :class:'numpy.ndarray'
:param x_names: labels for the ticks on x-axis (dimX)
:type x_names: :class:'list'
:param y_ticks: y values where tick makrs are located
- :type y_ticks: :class:'numpy.ndarray'
+ :type y_ticks: :class:'numpy.ndarray'
:param y_names: labels for the ticks on y-axis (dimY)
:type y_names: :class:'list'
:param output_file: name of output png file
@@ -1246,7 +1246,7 @@ def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_n
dimY, dimX = plot_data.shape
plot_data2 = np.zeros([dimY,dimX]) # sectioned array for plotting
- # fontsize
+ # fontsize
rcParams['axes.labelsize'] = 8
rcParams['xtick.labelsize'] = 8
rcParams['ytick.labelsize'] = 8
[2/2] climate git commit: Update plotter unit tests for
_best_gridshape changes
Posted by go...@apache.org.
Update plotter unit tests for _best_gridshape changes
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/58d6cdf1
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/58d6cdf1
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/58d6cdf1
Branch: refs/heads/master
Commit: 58d6cdf16e52da9c326381718a80ecc33e880eb4
Parents: fb02c74
Author: Alex <ag...@users.noreply.github.com>
Authored: Thu Sep 28 23:33:13 2017 -0700
Committer: Alex <ag...@users.noreply.github.com>
Committed: Thu Sep 28 23:33:13 2017 -0700
----------------------------------------------------------------------
ocw/tests/test_plotter.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/58d6cdf1/ocw/tests/test_plotter.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_plotter.py b/ocw/tests/test_plotter.py
index 19ba66a..894e0c4 100644
--- a/ocw/tests/test_plotter.py
+++ b/ocw/tests/test_plotter.py
@@ -51,14 +51,14 @@ class TestBestGridShapeFunction(unittest.TestCase):
def test_returned_shape_small(self):
nplots = 2
oldshape = (2, 2)
- expected_shape = (1, 2)
+ expected_shape = (2, 1)
new_shape = plotter._best_grid_shape(nplots, oldshape)
self.assertEqual(new_shape, expected_shape)
def test_returned_shape_large(self):
nplots = 57
- oldshape = (220, 12)
- expected_shape = (5, 12)
+ oldshape = (12, 220)
+ expected_shape = (12, 5)
new_shape = plotter._best_grid_shape(nplots, oldshape)
self.assertEqual(new_shape, expected_shape)