You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by jo...@apache.org on 2015/06/05 17:50:08 UTC
[01/10] climate git commit: CLIMATE-638 - Fix urllib import and minor
clean up
Repository: climate
Updated Branches:
refs/heads/master 82dd6b69d -> fe190b636
CLIMATE-638 - Fix urllib import and minor clean up
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/59c70625
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/59c70625
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/59c70625
Branch: refs/heads/master
Commit: 59c70625d5f947f50061c8a84fabcd48b2e9d04c
Parents: 82dd6b6
Author: Michael Joyce <jo...@apache.org>
Authored: Thu Jun 4 07:56:14 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Thu Jun 4 08:54:10 2015 -0700
----------------------------------------------------------------------
examples/subregions_portrait_diagram.py | 23 +++++++++--------------
1 file changed, 9 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/59c70625/examples/subregions_portrait_diagram.py
----------------------------------------------------------------------
diff --git a/examples/subregions_portrait_diagram.py b/examples/subregions_portrait_diagram.py
index e0d789a..075de2d 100644
--- a/examples/subregions_portrait_diagram.py
+++ b/examples/subregions_portrait_diagram.py
@@ -13,6 +13,7 @@ import numpy as np
import numpy.ma as ma
from os import path
+import urllib
# File URL leader
FILE_LEADER = "http://zipper.jpl.nasa.gov/dist/"
@@ -36,28 +37,22 @@ EVAL_BOUNDS = Bounds(LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END)
varName = 'pr'
#regridding parameters
-gridLonStep=0.5
-gridLatStep=0.5
+gridLonStep = 0.5
+gridLatStep = 0.5
#some vars for this evaluation
-target_datasets_ensemble=[]
-target_datasets =[]
-allNames =[]
+target_datasets_ensemble = []
+target_datasets = []
+allNames = []
# Download necessary NetCDF file if not present
-if path.exists(FILE_1):
- pass
-else:
+if not path.exists(FILE_1):
urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1)
-if path.exists(FILE_2):
- pass
-else:
+if not path.exists(FILE_2):
urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2)
-if path.exists(FILE_3):
- pass
-else:
+if not path.exists(FILE_3):
urllib.urlretrieve(FILE_LEADER + FILE_3, FILE_3)
""" Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list"""
[10/10] climate git commit: Resolve CLIMATE-583. Merge PR #205.
Posted by jo...@apache.org.
Resolve CLIMATE-583. Merge PR #205.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fe190b63
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fe190b63
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fe190b63
Branch: refs/heads/master
Commit: fe190b63603e4ca08c2e2d1409b4dafdf82a42ca
Parents: 2fb94d7 d6be7cf
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Jun 5 08:49:06 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Jun 5 08:49:06 2015 -0700
----------------------------------------------------------------------
ocw-config-runner/configuration_parsing.py | 50 ++++--
ocw-config-runner/evaluation_creation.py | 35 +++-
ocw-config-runner/example/portrait_diagram.yaml | 30 ++++
ocw-config-runner/example/subregion.yaml | 30 ++++
.../example/taylor_diagram_example.yaml | 36 ++++
.../example/time_series_plot_example.yaml | 31 ++++
ocw-config-runner/ocw_evaluation_from_config.py | 19 ++-
ocw-config-runner/plot_generation.py | 166 +++++++++++++++++--
ocw-config-runner/tests/test_config_parsing.py | 82 +++++++++
ocw/evaluation.py | 2 +-
10 files changed, 438 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
[05/10] climate git commit: CLIMATE-583 - Add portrait diagram support
Posted by jo...@apache.org.
CLIMATE-583 - Add portrait diagram support
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/c8539d00
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/c8539d00
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/c8539d00
Branch: refs/heads/master
Commit: c8539d009d1eefe0efa59d4c554f66c95f12928e
Parents: f9e13d7
Author: Michael Joyce <jo...@apache.org>
Authored: Thu Jun 4 09:50:24 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Thu Jun 4 14:44:42 2015 -0700
----------------------------------------------------------------------
ocw-config-runner/configuration_parsing.py | 9 ++++--
ocw-config-runner/example/portrait_diagram.yaml | 30 ++++++++++++++++++++
ocw-config-runner/plot_generation.py | 16 ++++++++++-
3 files changed, 51 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/c8539d00/ocw-config-runner/configuration_parsing.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/configuration_parsing.py b/ocw-config-runner/configuration_parsing.py
index 283a39d..ff853b7 100644
--- a/ocw-config-runner/configuration_parsing.py
+++ b/ocw-config-runner/configuration_parsing.py
@@ -123,9 +123,9 @@ def _config_is_well_formed(config_data):
is_well_formed = False
# Ensure that if we're trying to make a plot that require
# subregion info that the config has this present.
- elif plot['type'] in ['subregion']:
+ elif plot['type'] in ['subregion', 'portrait']:
if ('subregions' not in config_data or
- len(config_data['subregions'] < 1)):
+ len(config_data['subregions']) < 1):
logger.error(
'Plot config that requires subregion information is present '
'in a config file without adequate subregion information '
@@ -240,7 +240,10 @@ def _valid_plot_config_data(plot_config_data):
elif plot_type == 'time_series':
logger.warn('Time series plots are currently unsupported. Skipping validation')
elif plot_type == 'portrait':
- logger.warn('Portrait diagrams are currently unsupported. Skipping validation')
+ required_keys = set([
+ 'metric_index',
+ 'output_name'
+ ])
else:
logger.error('Invalid plot type specified.')
return False
http://git-wip-us.apache.org/repos/asf/climate/blob/c8539d00/ocw-config-runner/example/portrait_diagram.yaml
----------------------------------------------------------------------
diff --git a/ocw-config-runner/example/portrait_diagram.yaml b/ocw-config-runner/example/portrait_diagram.yaml
new file mode 100644
index 0000000..f3e45de
--- /dev/null
+++ b/ocw-config-runner/example/portrait_diagram.yaml
@@ -0,0 +1,30 @@
+evaluation:
+ temporal_time_delta: 365
+ spatial_regrid_lats: !!python/tuple [-40, 40, 1]
+ spatial_regrid_lons: !!python/tuple [-40, 40, 1]
+
+datasets:
+ reference:
+ data_source: local
+ file_count: 1
+ path: /tmp/AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc
+ variable: tasmax
+
+ targets:
+ - data_source: local
+ file_count: 1
+ path: /tmp/AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc
+ variable: tasmax
+ optional_args:
+ name: WRF
+metrics:
+ - PatternCorrelation
+
+plots:
+ - type: portrait
+ metric_index: 0
+ output_name: portrait_test
+
+subregions:
+ - [-10.0, 0.0, 29.0, 36.5]
+ - [0.0, 10.0, 29.0, 37.5]
http://git-wip-us.apache.org/repos/asf/climate/blob/c8539d00/ocw-config-runner/plot_generation.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/plot_generation.py b/ocw-config-runner/plot_generation.py
index 9da97c2..c0bfa03 100644
--- a/ocw-config-runner/plot_generation.py
+++ b/ocw-config-runner/plot_generation.py
@@ -43,7 +43,7 @@ def plot_from_config(evaluation, config_data):
elif plot['type'] == 'time_series':
logger.warn('Time series plots are currently unsupported. Skipping ...')
elif plot['type'] == 'portrait':
- logger.warn('Portrait diagrams are currently unsupported. Skipping ...')
+ _draw_portrait_diagram(evaluation, plot)
else:
logger.error('Unrecognized plot type requested: {}'.format(plot['type']))
@@ -104,3 +104,17 @@ def _draw_subregion_diagram(evaluation, plot_config):
lons,
plot_config['output_name'],
**plot_config.get('optional_args', {}))
+
+def _draw_portrait_diagram(evaluation, plot_config):
+ """"""
+ metric_index = plot_config['metric_index']
+
+ diagram_data = np.array(evaluation.results[:][metric_index][:])
+ subregion_names = ["R{}".format(i) for i in range(len(evaluation.subregions))]
+ target_names = [t.name for t in evaluation.target_datasets]
+
+ plots.draw_portrait_diagram(diagram_data,
+ target_names,
+ subregion_names,
+ fname=plot_config['output_name'],
+ **plot_config.get('optional_args', {}))
[02/10] climate git commit: CLIMATE-583 - Allow for partial
evaluation runs
Posted by jo...@apache.org.
CLIMATE-583 - Allow for partial evaluation runs
Allow for a user to do partial evaluation runs by specifying that config
parsing errors should be ignored. For instance, suppose that a user
wants to generate only a subregion plot. Normally they would have to
load and run a fake evaluation just to get the subregion plot. With this
new optional argument the user can pass only subregion information and
plot information in the config and still run the evaluation.
Note, there are no guarantees that any partial evaluation will work.
This option gives the user plenty of rope to hang themselves and should
really only be used when you want to generate plots that don't require a
full evaluation run (such as subregion plots).
- Add optional command line arg that tells the parser/runner to ignore
config parsing errors. This will cause the evaluation to "run" even if
the configuration isn't value.
- When running an evaluation, only run if the evaluation object passes
an is_valid check. Normally this is done in the evaluation object
anyway when calling run but calling this early prevents error messages
from being dumped.
- Make evaluation creation a bit more robust. Instead of just assuming
that 'datasets' is always present we first check that the value is not
None in the config before trying to check for 'reference' or
'targets'. Similarly with metrics. This is necessary for "partial"
evaluations.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/ba855628
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/ba855628
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/ba855628
Branch: refs/heads/master
Commit: ba8556287b3558eb6554a32596976c6a9ae98b11
Parents: f316aef
Author: Michael Joyce <jo...@apache.org>
Authored: Mon Jun 1 15:09:18 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Thu Jun 4 14:44:13 2015 -0700
----------------------------------------------------------------------
ocw-config-runner/evaluation_creation.py | 21 +++++++++++---------
ocw-config-runner/ocw_evaluation_from_config.py | 19 ++++++++++++++----
2 files changed, 27 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/ba855628/ocw-config-runner/evaluation_creation.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/evaluation_creation.py b/ocw-config-runner/evaluation_creation.py
index d5bc6d0..88394de 100644
--- a/ocw-config-runner/evaluation_creation.py
+++ b/ocw-config-runner/evaluation_creation.py
@@ -45,18 +45,21 @@ def generate_evaluation_from_config(config_data):
"""
# Load datasets
reference = None
- targets = None
- if 'reference' in config_data['datasets']:
- reference = _load_dataset(config_data['datasets']['reference'])
+ targets = []
+ if config_data['datasets']:
+ if 'reference' in config_data['datasets']:
+ reference = _load_dataset(config_data['datasets']['reference'])
- if 'targets' in config_data['datasets']:
- targets = [_load_dataset(t) for t in config_data['datasets']['targets']]
+ if 'targets' in config_data['datasets']:
+ targets = [_load_dataset(t) for t in config_data['datasets']['targets']]
- reference, targets = _prepare_datasets_for_evaluation(reference,
- targets,
- config_data)
+ reference, targets = _prepare_datasets_for_evaluation(reference,
+ targets,
+ config_data)
# Load metrics
- eval_metrics = [_load_metric(m)() for m in config_data['metrics']]
+ eval_metrics = []
+ if config_data['metrics']:
+ eval_metrics = [_load_metric(m)() for m in config_data['metrics']]
# Load Subregions (if present)
subregions = None
http://git-wip-us.apache.org/repos/asf/climate/blob/ba855628/ocw-config-runner/ocw_evaluation_from_config.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/ocw_evaluation_from_config.py b/ocw-config-runner/ocw_evaluation_from_config.py
index e27acc2..93b411a 100644
--- a/ocw-config-runner/ocw_evaluation_from_config.py
+++ b/ocw-config-runner/ocw_evaluation_from_config.py
@@ -27,7 +27,7 @@ import yaml
logging.basicConfig()
logger = logging.getLogger(__name__)
-def run_evaluation_from_config(config_file_path):
+def run_evaluation_from_config(config_file_path, ignore_config_errors=False):
""" Run an OCW evaluation specified by a config file.
:param config_file_path: The file path to a OCW compliant YAML file
@@ -35,18 +35,28 @@ def run_evaluation_from_config(config_file_path):
the valid options that you can set in the config please check the
project wiki https://cwiki.apache.org/confluence/display/climate/home#'.
:type config_file_path: :mod:`string`
+
+ :param ignore_config_errors: When this is true configuration parsing errors
+ will NOT interrupt the evaluation run. Note, it is very unlikely that
+ you will want this value set. However it is possible that you will want
+ to graph something that doesn't require a full evaluation run. This is
+ provided for that situation.
+ :type ignore_config_errors: :func:`bool`
"""
config = yaml.load(open(config_file_path, 'r'))
- if not is_config_valid(config):
+ if not ignore_config_errors and not is_config_valid(config):
logger.warning(
'Unable to validate configuration file. Exiting evaluation. '
'Please check documentation for config information.'
)
+
sys.exit(1)
evaluation = generate_evaluation_from_config(config)
- evaluation.run()
+
+ if evaluation._evaluation_is_valid():
+ evaluation.run()
plot_from_config(evaluation, config)
@@ -56,6 +66,7 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser(description=description, epilog=epilog)
parser.add_argument('config', help='Path to YAML config file for the evaluation')
+ parser.add_argument('ignore_config_errors', nargs='?', default=False, type=bool)
args = parser.parse_args()
- run_evaluation_from_config(args.config)
+ run_evaluation_from_config(args.config, args.ignore_config_errors)
[06/10] climate git commit: CLIMATE-583 - Add time series plot support
Posted by jo...@apache.org.
CLIMATE-583 - Add time series plot support
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/d6be7cf6
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/d6be7cf6
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/d6be7cf6
Branch: refs/heads/master
Commit: d6be7cf6266988e7e08305f5cfa481105875f2e7
Parents: c8539d0
Author: Michael Joyce <jo...@apache.org>
Authored: Thu Jun 4 14:22:19 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Thu Jun 4 14:45:22 2015 -0700
----------------------------------------------------------------------
ocw-config-runner/configuration_parsing.py | 4 +-
.../example/time_series_plot_example.yaml | 31 +++++
ocw-config-runner/plot_generation.py | 116 ++++++++++++++++---
3 files changed, 134 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/d6be7cf6/ocw-config-runner/configuration_parsing.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/configuration_parsing.py b/ocw-config-runner/configuration_parsing.py
index ff853b7..5c28249 100644
--- a/ocw-config-runner/configuration_parsing.py
+++ b/ocw-config-runner/configuration_parsing.py
@@ -238,7 +238,9 @@ def _valid_plot_config_data(plot_config_data):
'output_name'
])
elif plot_type == 'time_series':
- logger.warn('Time series plots are currently unsupported. Skipping validation')
+ required_keys = set([
+ 'time_range'
+ ])
elif plot_type == 'portrait':
required_keys = set([
'metric_index',
http://git-wip-us.apache.org/repos/asf/climate/blob/d6be7cf6/ocw-config-runner/example/time_series_plot_example.yaml
----------------------------------------------------------------------
diff --git a/ocw-config-runner/example/time_series_plot_example.yaml b/ocw-config-runner/example/time_series_plot_example.yaml
new file mode 100644
index 0000000..b5599cc
--- /dev/null
+++ b/ocw-config-runner/example/time_series_plot_example.yaml
@@ -0,0 +1,31 @@
+evaluation:
+ temporal_time_delta: 30
+ spatial_regrid_lats: !!python/tuple [-20, 20, 1]
+ spatial_regrid_lons: !!python/tuple [-20, 20, 1]
+ subset: [-180, 180, -90, 90, "1989-01-01", "1990-12-01"]
+
+datasets:
+ reference:
+ data_source: local
+ file_count: 1
+ path: /tmp/AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc
+ variable: tasmax
+ optional_args:
+ name: KNMI
+
+ targets:
+ - data_source: local
+ file_count: 1
+ path: /tmp/AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc
+ variable: tasmax
+ optional_args:
+ name: WRF
+metrics:
+
+plots:
+ - type: time_series
+ time_range: monthly
+
+subregions:
+ - [-10.0, 0.0, -19.0, 19.0]
+ - [0.0, 10.0, -10.0, 10.0]
http://git-wip-us.apache.org/repos/asf/climate/blob/d6be7cf6/ocw-config-runner/plot_generation.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/plot_generation.py b/ocw-config-runner/plot_generation.py
index c0bfa03..392331d 100644
--- a/ocw-config-runner/plot_generation.py
+++ b/ocw-config-runner/plot_generation.py
@@ -17,7 +17,9 @@
import logging
+import ocw.dataset_processor as dsp
import ocw.plotter as plots
+import ocw.utils as utils
import numpy as np
@@ -41,7 +43,7 @@ def plot_from_config(evaluation, config_data):
elif plot['type'] == 'taylor':
_draw_taylor_diagram(evaluation, plot)
elif plot['type'] == 'time_series':
- logger.warn('Time series plots are currently unsupported. Skipping ...')
+ _draw_time_series_plot(evaluation, plot)
elif plot['type'] == 'portrait':
_draw_portrait_diagram(evaluation, plot)
else:
@@ -57,13 +59,20 @@ def _draw_contour_plot(evaluation, plot_config):
if type(lons) != type(list):
lons = np.arange(lons['range_min'], lons['range_max'], lons['range_step'])
- for i, (row, col) in enumerate(plot_config['results_indices']):
+ for i, index in enumerate(plot_config['results_indices']):
+ if len(index) == 2:
+ target, metric = index
+ vals = evaluation.results[target][metric]
+ elif len(index) == 3:
+ target, metric, subregion = index
+ vals = evaluation.results[target][metric][subregion]
+
plot_name = plot_config['output_name'] + '_{}'.format(i)
- plots.draw_contour_map(evaluation.results[row][col],
- np.array(lats),
- np.array(lons),
- plot_name,
- **plot_config.get('optional_args', {}))
+ plots.draw_contour_map(vals,
+ np.array(lats),
+ np.array(lons),
+ plot_name,
+ **plot_config.get('optional_args', {}))
def _draw_taylor_diagram(evaluation, plot_config):
""""""
@@ -71,15 +80,26 @@ def _draw_taylor_diagram(evaluation, plot_config):
ref_dataset_name = evaluation.ref_dataset.name
target_dataset_names = [t.name for t in evaluation.target_datasets]
- stddev_results = [
- evaluation.results[row][col]
- for (row, col) in plot_config['stddev_results_indices']
- ]
-
- pattern_corr_results = [
- evaluation.results[row][col]
- for (row, col) in plot_config['pattern_corr_results_indices']
- ]
+ if len(plot_config['stddev_results_indices'][0]) == 2:
+ stddev_results = [
+ evaluation.results[tar][met]
+ for (tar, met) in plot_config['stddev_results_indices']
+ ]
+
+ pattern_corr_results = [
+ evaluation.results[tar][met]
+ for (tar, met) in plot_config['pattern_corr_results_indices']
+ ]
+ elif len(plot_config['stddev_results_indices'][0]) == 3:
+ stddev_results = [
+ evaluation.results[tar][met][sub]
+ for (tar, met, sub) in plot_config['stddev_results_indices']
+ ]
+
+ pattern_corr_results = [
+ evaluation.results[tar][met][sub]
+ for (tar, met, sub) in plot_config['pattern_corr_results_indices']
+ ]
plot_data = np.array([stddev_results, pattern_corr_results]).transpose()
@@ -118,3 +138,67 @@ def _draw_portrait_diagram(evaluation, plot_config):
subregion_names,
fname=plot_config['output_name'],
**plot_config.get('optional_args', {}))
+
+def _draw_time_series_plot(evaluation, plot_config):
+ """"""
+ time_range_info = plot_config['time_range']
+ ref_ds = evaluation.ref_dataset
+ target_ds = evaluation.target_datasets
+
+ if time_range_info == 'monthly':
+ ref_ds.values, ref_ds.times = utils.calc_climatology_monthly(ref_ds)
+
+ for t in target_ds:
+ t.values, t.times = utils.calc_climatology_monthly(t)
+ else:
+ logger.error(
+ 'Invalid time range provided. Only monthly is supported '
+ 'at the moment'
+ )
+ return
+
+ if evaluation.subregions:
+ for bound_count, bound in enumerate(evaluation.subregions):
+ results = []
+ labels = []
+
+ subset = dsp.subset(
+ bound,
+ ref_ds,
+ subregion_name="R{}_{}".format(bound_count, ref_ds.name)
+ )
+
+ results.append(utils.calc_time_series(subset))
+ labels.append(subset.name)
+
+ for t in target_ds:
+ subset = dsp.subset(
+ bound,
+ t,
+ subregion_name="R{}_{}".format(bound_count, t.name)
+ )
+ results.append(utils.calc_time_series(subset))
+ labels.append(subset.name)
+
+ plots.draw_time_series(np.array(results),
+ ref_ds.times,
+ labels,
+ 'R{}'.format(bound_count),
+ **plot_config.get('optional_args', {}))
+
+ else:
+ results = []
+ labels = []
+
+ results.append(utils.calc_time_series(ref_ds))
+ labels.append(ref_ds.name)
+
+ for t in target_ds:
+ results.append(utils.calc_time_series(t))
+ labels.append(t.name)
+
+ plots.draw_time_series(np.array(results),
+ ref_ds.times,
+ labels,
+ 'time_series',
+ **plot_config.get('optional_args', {}))
[04/10] climate git commit: CLIMATE-583 - Add subregion plot support
Posted by jo...@apache.org.
CLIMATE-583 - Add subregion plot support
- Move subregion checking above plot validity checks.
- When checking for plots, if the config is deemed valid then we need to
check to ensure that subregion information is provided for the plots
that require it. At the moment this check is only done for subregion
plots.
- Add required keys for subregion plots
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/f9e13d7f
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/f9e13d7f
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/f9e13d7f
Branch: refs/heads/master
Commit: f9e13d7f4bb737d54b71b8bdd71b9b786be833e9
Parents: ba85562
Author: Michael Joyce <jo...@apache.org>
Authored: Mon Jun 1 15:01:46 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Thu Jun 4 14:44:29 2015 -0700
----------------------------------------------------------------------
ocw-config-runner/configuration_parsing.py | 27 +++++--
ocw-config-runner/example/subregion.yaml | 30 ++++++++
ocw-config-runner/plot_generation.py | 23 +++++-
ocw-config-runner/tests/test_config_parsing.py | 82 +++++++++++++++++++++
4 files changed, 153 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/f9e13d7f/ocw-config-runner/configuration_parsing.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/configuration_parsing.py b/ocw-config-runner/configuration_parsing.py
index a0643de..283a39d 100644
--- a/ocw-config-runner/configuration_parsing.py
+++ b/ocw-config-runner/configuration_parsing.py
@@ -112,15 +112,28 @@ def _config_is_well_formed(config_data):
logger.warn(warning)
is_well_formed = False
+ if 'subregions' in config_data:
+ for subregion in config_data['subregions']:
+ if not _valid_subregion_config_data(subregion):
+ is_well_formed = False
+
if 'plots' in config_data:
for plot in config_data['plots']:
if not _valid_plot_config_data(plot):
is_well_formed = False
+ # Ensure that if we're trying to make a plot that require
+ # subregion info that the config has this present.
+ elif plot['type'] in ['subregion']:
+ if ('subregions' not in config_data or
+ len(config_data['subregions'] < 1)):
+ logger.error(
+ 'Plot config that requires subregion information is present '
+ 'in a config file without adequate subregion information '
+ 'provided. Please ensure that you have properly supplied 1 or '
+ 'more subregion config values.'
+ )
+ is_well_formed = False
- if 'subregions' in config_data:
- for subregion in config_data['subregions']:
- if not _valid_subregion_config_data(subregion):
- is_well_formed = False
return is_well_formed
@@ -219,7 +232,11 @@ def _valid_plot_config_data(plot_config_data):
'output_name'
])
elif plot_type == 'subregion':
- logger.warn('Subregion plots are currently unsupported. Skipping validation')
+ required_keys = set([
+ 'lats',
+ 'lons',
+ 'output_name'
+ ])
elif plot_type == 'time_series':
logger.warn('Time series plots are currently unsupported. Skipping validation')
elif plot_type == 'portrait':
http://git-wip-us.apache.org/repos/asf/climate/blob/f9e13d7f/ocw-config-runner/example/subregion.yaml
----------------------------------------------------------------------
diff --git a/ocw-config-runner/example/subregion.yaml b/ocw-config-runner/example/subregion.yaml
new file mode 100644
index 0000000..d39e39b
--- /dev/null
+++ b/ocw-config-runner/example/subregion.yaml
@@ -0,0 +1,30 @@
+subregions:
+ - [-10.0, 0.0, 29.0, 36.5]
+ - [0.0, 10.0, 29.0, 37.5]
+ - [10.0, 20.0, 25.0, 32.5]
+ - [20.0, 33.0, 25.0, 32.5]
+ - [-19.3,-10.2,12.0, 20.0]
+ - [15.0, 30.0, 15.0, 25.0]
+ - [-10.0, 10.0, 7.3, 15.0]
+ - [-10.9, 10.0, 5.0, 7.3]
+ - [33.9, 40.0, 6.9, 15.0]
+ - [10.0, 25.0, 0.0, 10.0]
+ - [10.0, 25.0,-10.0, 0.0]
+ - [30.0, 40.0,-15.0, 0.0]
+ - [33.0, 40.0, 25.0, 35.0]
+
+datasets:
+
+metrics:
+
+plots:
+ - type: subregion
+ output_name: subregion_plot
+ lats:
+ range_min: -45.0
+ range_max: 42.24
+ range_step: 0.5
+ lons:
+ range_min: -24.0
+ range_max: 60.0
+ range_step: 0.5
http://git-wip-us.apache.org/repos/asf/climate/blob/f9e13d7f/ocw-config-runner/plot_generation.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/plot_generation.py b/ocw-config-runner/plot_generation.py
index 677188a..9da97c2 100644
--- a/ocw-config-runner/plot_generation.py
+++ b/ocw-config-runner/plot_generation.py
@@ -37,7 +37,7 @@ def plot_from_config(evaluation, config_data):
if plot['type'] == 'contour':
_draw_contour_plot(evaluation, plot)
elif plot['type'] == 'subregion':
- logger.warn('Subregion plots are currently unsupported. Skipping ...')
+ _draw_subregion_diagram(evaluation, plot)
elif plot['type'] == 'taylor':
_draw_taylor_diagram(evaluation, plot)
elif plot['type'] == 'time_series':
@@ -51,12 +51,11 @@ def _draw_contour_plot(evaluation, plot_config):
""""""
lats = plot_config['lats']
if type(lats) != type(list):
- lats = range(lats['range_min'], lats['range_max'], lats['range_step'])
+ lats = np.arange(lats['range_min'], lats['range_max'], lats['range_step'])
lons = plot_config['lons']
if type(lons) != type(list):
- lons = range(lons['range_min'], lons['range_max'], lons['range_step'])
-
+ lons = np.arange(lons['range_min'], lons['range_max'], lons['range_step'])
for i, (row, col) in enumerate(plot_config['results_indices']):
plot_name = plot_config['output_name'] + '_{}'.format(i)
@@ -89,3 +88,19 @@ def _draw_taylor_diagram(evaluation, plot_config):
ref_dataset_name,
fname=plot_name,
**plot_config.get('optional_args', {}))
+
+def _draw_subregion_diagram(evaluation, plot_config):
+ """"""
+ lats = plot_config['lats']
+ if type(lats) != type(list):
+ lats = np.arange(lats['range_min'], lats['range_max'], lats['range_step'])
+
+ lons = plot_config['lons']
+ if type(lons) != type(list):
+ lons = np.arange(lons['range_min'], lons['range_max'], lons['range_step'])
+
+ plots.draw_subregions(evaluation.subregions,
+ lats,
+ lons,
+ plot_config['output_name'],
+ **plot_config.get('optional_args', {}))
http://git-wip-us.apache.org/repos/asf/climate/blob/f9e13d7f/ocw-config-runner/tests/test_config_parsing.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/tests/test_config_parsing.py b/ocw-config-runner/tests/test_config_parsing.py
index 88a08f7..e918405 100644
--- a/ocw-config-runner/tests/test_config_parsing.py
+++ b/ocw-config-runner/tests/test_config_parsing.py
@@ -682,6 +682,53 @@ class ContourMapConfig(unittest.TestCase):
mock_logger.error.assert_called_with(err)
+class TestSubregionPlotConfig(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ valid_subregion_config = """
+ type: subregion
+ lats:
+ range_min: -20
+ range_max: 20
+ range_step: 1
+ lons:
+ range_min: -20
+ range_max: 20
+ range_step: 1
+ output_name: fake_plot_name
+ """
+ self.valid_subregion = yaml.load(valid_subregion_config)
+
+ missing_keys_subregion_config = """
+ type: subregion
+ """
+ self.missing_keys_subregion = yaml.load(missing_keys_subregion_config)
+
+ self.required_subregion_keys = set([
+ 'lats',
+ 'lons',
+ 'output_name'
+ ])
+
+ def test_valid_subregion(self):
+ ret = parser._valid_plot_config_data(self.valid_subregion)
+ self.assertTrue(ret)
+
+ @patch('configuration_parsing.logger')
+ def test_missing_keys_subregion(self, mock_logger):
+ ret = parser._valid_plot_config_data(self.missing_keys_subregion)
+
+ present_keys = set(self.missing_keys_subregion.keys())
+ missing_keys = self.required_subregion_keys - present_keys
+ missing = sorted(list(missing_keys))
+
+ err = (
+ 'Plot config does not contain required keys. '
+ 'The following keys are missing: {}'
+ ).format(', '.join(missing))
+ mock_logger.error.assert_called_with(err)
+
+
class TestInvalidPlotConfig(unittest.TestCase):
@classmethod
def setUpClass(self):
@@ -705,6 +752,29 @@ class TestInvalidPlotConfig(unittest.TestCase):
"""
self.missing_plot_type = yaml.load(missing_plot_type_config)
+ missing_subregions_for_plot_type = """
+ datasets:
+ - blah
+
+ metrics:
+ - blah
+
+ plots:
+ - type: subregion
+ results_indices:
+ - !!python/tuple [0, 0]
+ lats:
+ range_min: -20
+ range_max: 20
+ range_step: 1
+ lons:
+ range_min: -20
+ range_max: 20
+ range_step: 1
+ output_name: wrf_bias_compared_to_knmi
+ """
+ self.missing_subregions = yaml.load(missing_subregions_for_plot_type)
+
@patch('configuration_parsing.logger')
def test_invalid_plot_type(self, mock_logger):
ret = parser._valid_plot_config_data(self.bad_plot_type)
@@ -722,3 +792,15 @@ class TestInvalidPlotConfig(unittest.TestCase):
mock_logger.error.assert_called_with(
'Plot config does not include a type attribute.'
)
+
+ @patch('configuration_parsing.logger')
+ def test_missing_subregion(self, mock_logger):
+ ret = parser._config_is_well_formed(self.missing_subregions)
+ self.assertFalse(ret)
+
+ mock_logger.error.assert_called_with(
+ 'Plot config that requires subregion information is present '
+ 'in a config file without adequate subregion information '
+ 'provided. Please ensure that you have properly supplied 1 or '
+ 'more subregion config values.'
+ )
[08/10] climate git commit: Resolve CLIMATE-638. Merge PR #204.
Posted by jo...@apache.org.
Resolve CLIMATE-638. Merge PR #204.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/569eb153
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/569eb153
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/569eb153
Branch: refs/heads/master
Commit: 569eb1533a1a70f4ea85fd35ba9ae3c0102b0930
Parents: 82dd6b6 59c7062
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Jun 5 08:48:03 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Jun 5 08:48:03 2015 -0700
----------------------------------------------------------------------
examples/subregions_portrait_diagram.py | 23 +++++++++--------------
1 file changed, 9 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[07/10] climate git commit: CLIMATE-640 - Fix missing evaluation
variable in time series example
Posted by jo...@apache.org.
CLIMATE-640 - Fix missing evaluation variable in time series example
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/653dc73a
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/653dc73a
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/653dc73a
Branch: refs/heads/master
Commit: 653dc73a2fcfb7dd552bab6a900b0f6c14dbe7f6
Parents: 82dd6b6
Author: Michael Joyce <jo...@apache.org>
Authored: Thu Jun 4 14:48:53 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Thu Jun 4 14:48:53 2015 -0700
----------------------------------------------------------------------
examples/time_series_with_regions.py | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/653dc73a/examples/time_series_with_regions.py
----------------------------------------------------------------------
diff --git a/examples/time_series_with_regions.py b/examples/time_series_with_regions.py
index b11a807..1d552a8 100644
--- a/examples/time_series_with_regions.py
+++ b/examples/time_series_with_regions.py
@@ -28,6 +28,8 @@ LON_MAX = 60.0
START = datetime.datetime(2000, 01, 1)
END = datetime.datetime(2007, 12, 31)
+EVAL_BOUNDS = Bounds(LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END)
+
varName = 'pr'
gridLonStep=0.44
gridLatStep=0.44
[09/10] climate git commit: Resolve CLIMATE-640. Merge PR #206.
Posted by jo...@apache.org.
Resolve CLIMATE-640. Merge PR #206.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/2fb94d7d
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/2fb94d7d
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/2fb94d7d
Branch: refs/heads/master
Commit: 2fb94d7df4885c844f53aecf1cd50182d53ef434
Parents: 569eb15 653dc73
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Jun 5 08:48:38 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Jun 5 08:48:38 2015 -0700
----------------------------------------------------------------------
examples/time_series_with_regions.py | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
[03/10] climate git commit: CLIMATE-583 - Add Taylor diagram support
Posted by jo...@apache.org.
CLIMATE-583 - Add Taylor diagram support
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/f316aefc
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/f316aefc
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/f316aefc
Branch: refs/heads/master
Commit: f316aefcc4ae731f2e88b1bbd4b007948417f5e0
Parents: 82dd6b6
Author: Michael Joyce <jo...@apache.org>
Authored: Fri May 15 08:22:34 2015 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Thu Jun 4 14:44:13 2015 -0700
----------------------------------------------------------------------
ocw-config-runner/configuration_parsing.py | 14 +++++---
ocw-config-runner/evaluation_creation.py | 14 ++++++++
.../example/taylor_diagram_example.yaml | 36 ++++++++++++++++++++
ocw-config-runner/plot_generation.py | 31 +++++++++++++----
ocw/evaluation.py | 2 +-
5 files changed, 85 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/f316aefc/ocw-config-runner/configuration_parsing.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/configuration_parsing.py b/ocw-config-runner/configuration_parsing.py
index be8bd00..a0643de 100644
--- a/ocw-config-runner/configuration_parsing.py
+++ b/ocw-config-runner/configuration_parsing.py
@@ -207,13 +207,17 @@ def _valid_plot_config_data(plot_config_data):
if plot_type == 'contour':
required_keys = set([
- 'results_indices',
- 'lats',
- 'lons',
- 'output_name'
+ 'results_indices',
+ 'lats',
+ 'lons',
+ 'output_name'
])
elif plot_type == 'taylor':
- logger.warn('Taylor diagrams are currently unsupported. Skipping validation')
+ required_keys = set([
+ 'stddev_results_indices',
+ 'pattern_corr_results_indices',
+ 'output_name'
+ ])
elif plot_type == 'subregion':
logger.warn('Subregion plots are currently unsupported. Skipping validation')
elif plot_type == 'time_series':
http://git-wip-us.apache.org/repos/asf/climate/blob/f316aefc/ocw-config-runner/evaluation_creation.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/evaluation_creation.py b/ocw-config-runner/evaluation_creation.py
index 94967d3..d5bc6d0 100644
--- a/ocw-config-runner/evaluation_creation.py
+++ b/ocw-config-runner/evaluation_creation.py
@@ -106,6 +106,20 @@ def _prepare_datasets_for_evaluation(reference, targets, config_data):
spatial_regrid_lats = config_data['evaluation'].get('spatial_regrid_lats', None)
spatial_regrid_lons = config_data['evaluation'].get('spatial_regrid_lons', None)
+ # If we have a temporal time delta and it's daily (i.e., 1) we will
+ # normalize the data as daily data (which means we adjust the start times
+ # for each bucket of data to be consistent). By default we will normalize
+ # the data as monthly. Note that this will not break yearly data so it's
+ # safer to do this no matter what. This keeps us from ending up with 1-off
+ # errors in the resulting dataset shape post-temporal/spatial adjustments
+ # that break evaluations.
+ string_time_delta = 'monthly'
+ if temporal_time_delta and temporal_time_delta == 1:
+ string_time_delta = 'daily'
+
+ reference = dsp.normalize_dataset_datetimes(reference, string_time_delta)
+ targets = [dsp.normalize_dataset_datetimes(t, string_time_delta) for t in targets]
+
if subset:
start = dateutil.parser.parse(subset[4])
end = dateutil.parser.parse(subset[5])
http://git-wip-us.apache.org/repos/asf/climate/blob/f316aefc/ocw-config-runner/example/taylor_diagram_example.yaml
----------------------------------------------------------------------
diff --git a/ocw-config-runner/example/taylor_diagram_example.yaml b/ocw-config-runner/example/taylor_diagram_example.yaml
new file mode 100644
index 0000000..66eab3b
--- /dev/null
+++ b/ocw-config-runner/example/taylor_diagram_example.yaml
@@ -0,0 +1,36 @@
+evaluation:
+ temporal_time_delta: 30
+ spatial_regrid_lats: !!python/tuple [-45, 42, 1]
+ spatial_regrid_lons: !!python/tuple [-24, 60, 1]
+ subset: [-45, 42, -24, 60, "1989-01-01", "1989-12-01"]
+
+datasets:
+ reference:
+ data_source: local
+ file_count: 1
+ path: /tmp/AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc
+ variable: tasmax
+ optional_args:
+ name: dataset1
+
+ targets:
+ - data_source: local
+ file_count: 1
+ path: /tmp/AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc
+ variable: tasmax
+ optional_args:
+ name: dataset2
+metrics:
+ - StdDevRatio
+ - PatternCorrelation
+
+plots:
+ - type: taylor
+ stddev_results_indices:
+ - !!python/tuple [0, 0]
+ pattern_corr_results_indices:
+ - !!python/tuple [0, 1]
+ output_name: taylor_plot
+ optional_args:
+ fmt: png
+ frameon: False
http://git-wip-us.apache.org/repos/asf/climate/blob/f316aefc/ocw-config-runner/plot_generation.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/plot_generation.py b/ocw-config-runner/plot_generation.py
index 2a74878..677188a 100644
--- a/ocw-config-runner/plot_generation.py
+++ b/ocw-config-runner/plot_generation.py
@@ -38,19 +38,14 @@ def plot_from_config(evaluation, config_data):
_draw_contour_plot(evaluation, plot)
elif plot['type'] == 'subregion':
logger.warn('Subregion plots are currently unsupported. Skipping ...')
- continue
elif plot['type'] == 'taylor':
- logger.warn('Taylor diagrams are currently unsupported. Skipping ...')
- continue
+ _draw_taylor_diagram(evaluation, plot)
elif plot['type'] == 'time_series':
logger.warn('Time series plots are currently unsupported. Skipping ...')
- continue
elif plot['type'] == 'portrait':
logger.warn('Portrait diagrams are currently unsupported. Skipping ...')
- continue
else:
logger.error('Unrecognized plot type requested: {}'.format(plot['type']))
- continue
def _draw_contour_plot(evaluation, plot_config):
""""""
@@ -70,3 +65,27 @@ def _draw_contour_plot(evaluation, plot_config):
np.array(lons),
plot_name,
**plot_config.get('optional_args', {}))
+
+def _draw_taylor_diagram(evaluation, plot_config):
+ """"""
+ plot_name = plot_config['output_name']
+ ref_dataset_name = evaluation.ref_dataset.name
+ target_dataset_names = [t.name for t in evaluation.target_datasets]
+
+ stddev_results = [
+ evaluation.results[row][col]
+ for (row, col) in plot_config['stddev_results_indices']
+ ]
+
+ pattern_corr_results = [
+ evaluation.results[row][col]
+ for (row, col) in plot_config['pattern_corr_results_indices']
+ ]
+
+ plot_data = np.array([stddev_results, pattern_corr_results]).transpose()
+
+ plots.draw_taylor_diagram(plot_data,
+ target_dataset_names,
+ ref_dataset_name,
+ fname=plot_name,
+ **plot_config.get('optional_args', {}))
http://git-wip-us.apache.org/repos/asf/climate/blob/f316aefc/ocw/evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/evaluation.py b/ocw/evaluation.py
index 9581d96..89e34b4 100644
--- a/ocw/evaluation.py
+++ b/ocw/evaluation.py
@@ -260,7 +260,7 @@ class Evaluation(object):
elif run_unary:
return unary_valid
else:
- return false
+ return False
def _should_run_regular_metrics(self):
return len(self.metrics) > 0