You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by hu...@apache.org on 2015/08/25 19:06:51 UTC
[1/7] climate git commit: CLIMATE-658 - Restructure evaluation
results to handle missing data values
Repository: climate
Updated Branches:
refs/heads/master 45779fb98 -> 77443d7ad
CLIMATE-658 - Restructure evaluation results to handle missing data values
- now the evaluation results maintain missing value information
- update _run_no_subregion_evaluation and _run_unary_metric_evaluation in evaluation.py
- update test_evaluation
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/c088cfab
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/c088cfab
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/c088cfab
Branch: refs/heads/master
Commit: c088cfabd9f9df760a83f8b9f003e26a11730a29
Parents: d4eeb03
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Wed Aug 12 01:09:23 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Wed Aug 12 01:09:23 2015 -0700
----------------------------------------------------------------------
ocw/evaluation.py | 29 +++++++++++++++++++----------
ocw/tests/test_evaluation.py | 29 +++++++++--------------------
2 files changed, 28 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/c088cfab/ocw/evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/evaluation.py b/ocw/evaluation.py
index 89e34b4..11c1ae3 100644
--- a/ocw/evaluation.py
+++ b/ocw/evaluation.py
@@ -25,6 +25,8 @@ from metrics import Metric, UnaryMetric, BinaryMetric
from dataset import Dataset, Bounds
import ocw.dataset_processor as DSP
+import numpy.ma as ma
+
logger = logging.getLogger(__name__)
class Evaluation(object):
@@ -289,23 +291,30 @@ class Evaluation(object):
def _run_no_subregion_evaluation(self):
results = []
- for target in self.target_datasets:
- results.append([])
- for metric in self.metrics:
- run_result = metric.run(self.ref_dataset, target)
- results[-1].append(run_result)
+ for metric in self.metrics:
+ run_result_shape = list((metric.run(self.ref_dataset, self.target_datasets[0])).shape)
+ run_result_shape.insert(0, len(self.target_datasets))
+ run_result = ma.zeros(run_result_shape)
+
+ for itarget, target in enumerate(self.target_datasets):
+ run_result[itarget,:] = metric.run(self.ref_dataset, target)
+ results.append(run_result)
return results
def _run_unary_metric_evaluation(self):
unary_results = []
for metric in self.unary_metrics:
- unary_results.append([])
# Unary metrics should be run over the reference Dataset also
if self.ref_dataset:
- unary_results[-1].append(metric.run(self.ref_dataset))
-
- for target in self.target_datasets:
- unary_results[-1].append(metric.run(target))
+ unary_results.append(metric.run(self.ref_dataset))
+
+ unary_result_shape = list((metric.run(self.target_datasets[0])).shape)
+ unary_result_shape.insert(0, len(self.target_datasets))
+ unary_result = ma.zeros(unary_result_shape)
+ for itarget, target in enumerate(self.target_datasets):
+ unary_result[itarget,:] = metric.run(target)
+ unary_results.append(unary_result)
+
return unary_results
def _run_subregion_unary_evaluation(self):
http://git-wip-us.apache.org/repos/asf/climate/blob/c088cfab/ocw/tests/test_evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_evaluation.py b/ocw/tests/test_evaluation.py
index 4a02714..c2b85de 100644
--- a/ocw/tests/test_evaluation.py
+++ b/ocw/tests/test_evaluation.py
@@ -128,40 +128,29 @@ class TestEvaluation(unittest.TestCase):
def test_result_shape(self):
bias_eval = Evaluation(
self.test_dataset,
- [self.another_test_dataset, self.another_test_dataset],
- [Bias()]
+ [self.another_test_dataset, self.another_test_dataset, self.another_test_dataset],
+ [Bias(), Bias()]
)
bias_eval.run()
# Expected result shape is
- # [
- # [
- # bias.run(reference, target1)
- # ],
- # [
- # bias.run(reference, target2)
- # ]
- # ]
+ # [bias, bias] where bias.shape[0] = number of datasets
self.assertTrue(len(bias_eval.results) == 2)
- self.assertTrue(len(bias_eval.results[0]) == 1)
- self.assertTrue(len(bias_eval.results[1]) == 1)
+ self.assertTrue(bias_eval.results[0].shape[0] == 3)
def test_unary_result_shape(self):
new_eval = Evaluation(
self.test_dataset,
- [self.another_test_dataset, self.another_test_dataset],
+ [self.another_test_dataset, self.another_test_dataset, self.another_test_dataset, self.another_test_dataset],
[TemporalStdDev()]
)
new_eval.run()
# Expected result shape is
- # [
- # temporalstddev.run(reference),
- # temporalstddev.run(target1),
- # temporalstddev.run(target2)
- # ]
- self.assertTrue(len(new_eval.unary_results) == 1)
- self.assertTrue(len(new_eval.unary_results[0]) == 3)
+ # [stddev] where stddev.shape[0] = number of datasets
+
+ self.assertTrue(len(new_eval.unary_results) == 2)
+ self.assertTrue(new_eval.unary_results[1].shape[0] == 4)
def test_subregion_result_shape(self):
bound = Bounds(
[2/7] climate git commit: Revised description on the structure of
evaluation results
Posted by hu...@apache.org.
Revised description on the structure of evaluation results
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/19e8d0c0
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/19e8d0c0
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/19e8d0c0
Branch: refs/heads/master
Commit: 19e8d0c052801b9b45e4d3db47877492395ed751
Parents: c088cfa
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Wed Aug 12 08:41:06 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Wed Aug 12 08:41:06 2015 -0700
----------------------------------------------------------------------
ocw/evaluation.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/19e8d0c0/ocw/evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/evaluation.py b/ocw/evaluation.py
index 11c1ae3..becced1 100644
--- a/ocw/evaluation.py
+++ b/ocw/evaluation.py
@@ -95,8 +95,10 @@ class Evaluation(object):
self._subregions = subregions
#: A list containing the results of running regular metric evaluations.
- #: The shape of results is ``(num_target_datasets, num_metrics)`` if
- #: the user doesn't specify subregion information. Otherwise the shape
+ #: The shape of results list is ``len(results_list) = num_metrics``.
+ #: Each element of the list is a numpy.Masked array whose first dimension
+ #: is num_target_datasets.
+ #: If the user specify subregion information, the shape
#: is ``(num_target_datasets, num_metrics, num_subregions)``.
self.results = []
#: A list containing the results of running the unary metric
[4/7] climate git commit: SSL certificate issue
Posted by hu...@apache.org.
SSL certificate issue
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/549191d4
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/549191d4
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/549191d4
Branch: refs/heads/master
Commit: 549191d47ac0b5db8a5ef58c4c184b891c287a59
Parents: d120a8f
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Wed Aug 19 09:40:55 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Wed Aug 19 09:40:55 2015 -0700
----------------------------------------------------------------------
examples/knmi_to_cru31_full_bias.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/549191d4/examples/knmi_to_cru31_full_bias.py
----------------------------------------------------------------------
diff --git a/examples/knmi_to_cru31_full_bias.py b/examples/knmi_to_cru31_full_bias.py
index c6dac47..ac629e9 100644
--- a/examples/knmi_to_cru31_full_bias.py
+++ b/examples/knmi_to_cru31_full_bias.py
@@ -28,7 +28,9 @@ import ocw.dataset_processor as dsp
import ocw.evaluation as evaluation
import ocw.metrics as metrics
import ocw.plotter as plotter
-
+import ssl
+if hasattr(ssl, '_create_unverified_context'):
+ ssl._create_default_https_context = ssl._create_unverified_context
# File URL leader
FILE_LEADER = "http://zipper.jpl.nasa.gov/dist/"
# This way we can easily adjust the time span of the retrievals
[3/7] climate git commit: subregion evaluation has been updated - one
less hierarchy in the result list
Posted by hu...@apache.org.
subregion evaluation has been updated - one less hierarchy in the result list
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/d120a8f8
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/d120a8f8
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/d120a8f8
Branch: refs/heads/master
Commit: d120a8f8ca9229bafcb51b927d81a90f5e900c3c
Parents: 19e8d0c
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Tue Aug 18 17:21:04 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Tue Aug 18 17:21:04 2015 -0700
----------------------------------------------------------------------
ocw/evaluation.py | 70 ++++++++++++++++++++-------------------
ocw/tests/test_evaluation.py | 31 ++++-------------
2 files changed, 42 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/d120a8f8/ocw/evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/evaluation.py b/ocw/evaluation.py
index becced1..760325d 100644
--- a/ocw/evaluation.py
+++ b/ocw/evaluation.py
@@ -98,8 +98,10 @@ class Evaluation(object):
#: The shape of results list is ``len(results_list) = num_metrics``.
#: Each element of the list is a numpy.Masked array whose first dimension
#: is num_target_datasets.
- #: If the user specify subregion information, the shape
- #: is ``(num_target_datasets, num_metrics, num_subregions)``.
+ #: If the user specify subregion information, the shape of result list
+ #: is list[num_subregions][num_metrics]. Each element of this hierarchical
+ #: list is also a numpy.Masked array whose first dimension
+ #: is num_target_datasets.
self.results = []
#: A list containing the results of running the unary metric
#: evaluations. The shape of unary_results is
@@ -274,21 +276,21 @@ class Evaluation(object):
def _run_subregion_evaluation(self):
results = []
- new_refs = [DSP.subset(s, self.ref_dataset) for s in self.subregions]
-
- for target in self.target_datasets:
- results.append([])
- new_targets = [DSP.subset(s, target) for s in self.subregions]
-
+ for s in self.subregions:
+ subregion_results=[]
+ new_refs = DSP.subset(s, self.ref_dataset)
+ new_target0= DSP.subset(s, self.target_datasets[0])
for metric in self.metrics:
- results[-1].append([])
-
- for i in range(len(self.subregions)):
- new_ref = new_refs[i]
- new_tar = new_targets[i]
+ run_result_shape = list((metric.run(new_refs, new_target0)).shape)
+ run_result_shape.insert(0, len(self.target_datasets))
+ run_result = ma.zeros(run_result_shape)
+
+ for itarget, target in enumerate(self.target_datasets):
+ new_target= DSP.subset(s, target)
+ run_result[itarget,:] = metric.run(new_refs, new_target)
+ subregion_results.append(run_result)
+ results.append(subregion_results)
- run_result = metric.run(new_ref, new_tar)
- results[-1][-1].append(run_result)
return results
def _run_no_subregion_evaluation(self):
@@ -321,26 +323,26 @@ class Evaluation(object):
def _run_subregion_unary_evaluation(self):
unary_results = []
- if self.ref_dataset:
- new_refs = [DSP.subset(s, self.ref_dataset) for s in self.subregions]
-
- new_targets = [
- [DSP.subset(s, t) for s in self.subregions]
- for t in self.target_datasets
- ]
-
- for metric in self.unary_metrics:
- unary_results.append([])
-
- for i in range(len(self.subregions)):
- unary_results[-1].append([])
-
+ for s in self.subregions:
+ subregion_results=[]
+ for metric in self.unary_metrics:
+ unary_result_shape = list((metric.run(self.target_datasets[0])).shape)
if self.ref_dataset:
- unary_results[-1][-1].append(metric.run(new_refs[i]))
-
- for t in range(len(self.target_datasets)):
- unary_results[-1][-1].append(metric.run(new_targets[t][i]))
-
+ unary_result_shape.insert(0, len(self.target_datasets)+1)
+ num_refs = 1
+ else:
+ unary_result_shape.insert(0, len(self.target_datasets))
+ num_refs = 0
+ unary_result = ma.zeros(unary_result_shape)
+ for itarget, target in enumerate(self.target_datasets):
+ new_target = DSP.subset(s, target)
+ unary_result[itarget+num_refs,:] = metric.run(new_target)
+ if self.ref_dataset:
+ new_refs = DSP.subset(s, self.ref_dataset)
+ unary_result[0,:] = metric.run(new_refs)
+
+ subregion_results.append(unary_result)
+ unary_results.append(subregion_results)
return unary_results
def __str__(self):
http://git-wip-us.apache.org/repos/asf/climate/blob/d120a8f8/ocw/tests/test_evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_evaluation.py b/ocw/tests/test_evaluation.py
index c2b85de..566de86 100644
--- a/ocw/tests/test_evaluation.py
+++ b/ocw/tests/test_evaluation.py
@@ -168,26 +168,14 @@ class TestEvaluation(unittest.TestCase):
# Expected result shape is
# [
- # [
# [ # Subregions cause this extra layer
- # bias.run(reference, target1)
+ # [number of targets, bias.run(reference, target1).shape]
# ]
# ],
- # [
- # [
- # bias.run(reference, target2)
- # ]
- # ]
- # ]
- self.assertTrue(len(bias_eval.results) == 2)
-
+ self.assertTrue(len(bias_eval.results) == 1)
self.assertTrue(len(bias_eval.results[0]) == 1)
- self.assertTrue(type(bias_eval.results[0]) == type([]))
- self.assertTrue(len(bias_eval.results[1]) == 1)
- self.assertTrue(type(bias_eval.results[1]) == type([]))
-
- self.assertTrue(len(bias_eval.results[0][0]) == 1)
- self.assertTrue(len(bias_eval.results[1][0]) == 1)
+ self.assertTrue(bias_eval.results[0][0].shape[0] == 2)
+ self.assertTrue(type(bias_eval.results) == type([]))
def test_subregion_unary_result_shape(self):
bound = Bounds(
@@ -205,20 +193,13 @@ class TestEvaluation(unittest.TestCase):
# Expected result shape is
# [
- # [
# [ # Subregions cause this extra layer
- # temporalstddev.run(reference),
- # temporalstddev.run(target1),
- # temporalstddev.run(target2)
+ # [3, temporalstddev.run(reference).shape],
# ]
- # ]
# ]
self.assertTrue(len(new_eval.unary_results) == 1)
self.assertTrue(type(new_eval.unary_results) == type([]))
-
- self.assertTrue(len(new_eval.unary_results[0]) == 1)
-
- self.assertTrue(len(new_eval.unary_results[0][0]) == 3)
+ self.assertTrue(new_eval.unary_results[0][0].shape[0] == 3)
if __name__ == '__main__':
[5/7] climate git commit: two functions converting evaluation results
have been added
Posted by hu...@apache.org.
two functions converting evaluation results have been added
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/c6f5ed75
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/c6f5ed75
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/c6f5ed75
Branch: refs/heads/master
Commit: c6f5ed75066e6e38ed2e308e615b083f350a234c
Parents: 549191d
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Thu Aug 20 20:04:12 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Thu Aug 20 20:04:12 2015 -0700
----------------------------------------------------------------------
ocw/evaluation.py | 169 +++++++++++++++++++++++++-------------
ocw/tests/test_evaluation.py | 13 +--
2 files changed, 117 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/c6f5ed75/ocw/evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/evaluation.py b/ocw/evaluation.py
index 760325d..ff9ad33 100644
--- a/ocw/evaluation.py
+++ b/ocw/evaluation.py
@@ -95,13 +95,9 @@ class Evaluation(object):
self._subregions = subregions
#: A list containing the results of running regular metric evaluations.
- #: The shape of results list is ``len(results_list) = num_metrics``.
- #: Each element of the list is a numpy.Masked array whose first dimension
- #: is num_target_datasets.
- #: If the user specify subregion information, the shape of result list
- #: is list[num_subregions][num_metrics]. Each element of this hierarchical
- #: list is also a numpy.Masked array whose first dimension
- #: is num_target_datasets.
+ #: The shape of results is ``(num_target_datasets, num_metrics)`` if
+ #: the user doesn't specify subregion information. Otherwise the shape
+ #: is ``(num_target_datasets, num_metrics, num_subregions)``.
self.results = []
#: A list containing the results of running the unary metric
#: evaluations. The shape of unary_results is
@@ -276,74 +272,67 @@ class Evaluation(object):
def _run_subregion_evaluation(self):
results = []
- for s in self.subregions:
- subregion_results=[]
- new_refs = DSP.subset(s, self.ref_dataset)
- new_target0= DSP.subset(s, self.target_datasets[0])
+ new_refs = [DSP.subset(s, self.ref_dataset) for s in self.subregions]
+
+ for target in self.target_datasets:
+ results.append([])
+ new_targets = [DSP.subset(s, target) for s in self.subregions]
+
for metric in self.metrics:
- run_result_shape = list((metric.run(new_refs, new_target0)).shape)
- run_result_shape.insert(0, len(self.target_datasets))
- run_result = ma.zeros(run_result_shape)
-
- for itarget, target in enumerate(self.target_datasets):
- new_target= DSP.subset(s, target)
- run_result[itarget,:] = metric.run(new_refs, new_target)
- subregion_results.append(run_result)
- results.append(subregion_results)
+ results[-1].append([])
- return results
+ for i in range(len(self.subregions)):
+ new_ref = new_refs[i]
+ new_tar = new_targets[i]
+
+ run_result = metric.run(new_ref, new_tar)
+ results[-1][-1].append(run_result)
+ return convert_evaluation_result(results, subregion=True)
def _run_no_subregion_evaluation(self):
results = []
- for metric in self.metrics:
- run_result_shape = list((metric.run(self.ref_dataset, self.target_datasets[0])).shape)
- run_result_shape.insert(0, len(self.target_datasets))
- run_result = ma.zeros(run_result_shape)
-
- for itarget, target in enumerate(self.target_datasets):
- run_result[itarget,:] = metric.run(self.ref_dataset, target)
- results.append(run_result)
- return results
+ for target in self.target_datasets:
+ results.append([])
+ for metric in self.metrics:
+ run_result = metric.run(self.ref_dataset, target)
+ results[-1].append(run_result)
+ return convert_evaluation_result(results)
def _run_unary_metric_evaluation(self):
unary_results = []
for metric in self.unary_metrics:
+ unary_results.append([])
# Unary metrics should be run over the reference Dataset also
if self.ref_dataset:
- unary_results.append(metric.run(self.ref_dataset))
+ unary_results[-1].append(metric.run(self.ref_dataset))
- unary_result_shape = list((metric.run(self.target_datasets[0])).shape)
- unary_result_shape.insert(0, len(self.target_datasets))
- unary_result = ma.zeros(unary_result_shape)
- for itarget, target in enumerate(self.target_datasets):
- unary_result[itarget,:] = metric.run(target)
- unary_results.append(unary_result)
-
- return unary_results
+ for target in self.target_datasets:
+ unary_results[-1].append(metric.run(target))
+ return convert_unary_evaluation_result(unary_results)
def _run_subregion_unary_evaluation(self):
unary_results = []
- for s in self.subregions:
- subregion_results=[]
- for metric in self.unary_metrics:
- unary_result_shape = list((metric.run(self.target_datasets[0])).shape)
- if self.ref_dataset:
- unary_result_shape.insert(0, len(self.target_datasets)+1)
- num_refs = 1
- else:
- unary_result_shape.insert(0, len(self.target_datasets))
- num_refs = 0
- unary_result = ma.zeros(unary_result_shape)
- for itarget, target in enumerate(self.target_datasets):
- new_target = DSP.subset(s, target)
- unary_result[itarget+num_refs,:] = metric.run(new_target)
+ if self.ref_dataset:
+ new_refs = [DSP.subset(s, self.ref_dataset) for s in self.subregions]
+
+ new_targets = [
+ [DSP.subset(s, t) for s in self.subregions]
+ for t in self.target_datasets
+ ]
+
+ for metric in self.unary_metrics:
+ unary_results.append([])
+
+ for i in range(len(self.subregions)):
+ unary_results[-1].append([])
+
if self.ref_dataset:
- new_refs = DSP.subset(s, self.ref_dataset)
- unary_result[0,:] = metric.run(new_refs)
-
- subregion_results.append(unary_result)
- unary_results.append(subregion_results)
- return unary_results
+ unary_results[-1][-1].append(metric.run(new_refs[i]))
+
+ for t in range(len(self.target_datasets)):
+ unary_results[-1][-1].append(metric.run(new_targets[t][i]))
+
+ return convert_unary_evaluation_result(unary_results, subregion = True)
def __str__(self):
formatted_repr = (
@@ -362,3 +351,65 @@ class Evaluation(object):
str(self.subregions)
)
+def convert_evaluation_result(evaluation_result, subregion = False):
+ if not subregion:
+ nmodel = len(evaluation_result)
+ nmetric = len(evaluation_result[0])
+ results = []
+ for imetric in range(nmetric):
+ result_shape = list(evaluation_result[0][imetric].shape)
+ result_shape.insert(0, nmodel)
+ result = ma.zeros(result_shape)
+ for imodel in range(nmodel):
+ result[imodel,:] = evaluation_result[imodel][imetric]
+ results.append(result)
+ return results
+ else:
+ nmodel = len(evaluation_result)
+ nmetric = len(evaluation_result[0])
+ nsubregion = len(evaluation_result[0][0])
+
+ results = []
+ for isubregion in range(nsubregion):
+ subregion_results = []
+ for imetric in range(nmetric):
+ result_shape = list(evaluation_result[0][imetric][isubregion].shape)
+ result_shape.insert(0, nmodel)
+ result = ma.zeros(result_shape)
+ for imodel in range(nmodel):
+ result[imodel,:] = evaluation_result[imodel][imetric][isubregion]
+ subregion_results.append(result)
+ results.append(subregion_results)
+ return results
+
+def convert_unary_evaluation_result(evaluation_result, subregion = False):
+ if not subregion:
+ nmetric = len(evaluation_result)
+ nmodel = len(evaluation_result[0])
+ results = []
+ for imetric in range(nmetric):
+ result_shape = list(evaluation_result[imetric][0].shape)
+ result_shape.insert(0, nmodel)
+ result = ma.zeros(result_shape)
+ for imodel in range(nmodel):
+ result[imodel,:] = evaluation_result[imetric][imodel]
+ results.append(result)
+ return results
+ else:
+ nmetric = len(evaluation_result)
+ nsubregion = len(evaluation_result[0])
+ nmodel = len(evaluation_result[0][0])
+
+ results = []
+ for isubregion in range(nsubregion):
+ subregion_results = []
+ for imetric in range(nmetric):
+ result_shape = list(evaluation_result[imetric][isubregion][0].shape)
+ result_shape.insert(0, nmodel)
+ result = ma.zeros(result_shape)
+ for imodel in range(nmodel):
+ result[imodel,:] = evaluation_result[imetric][isubregion][imodel]
+ subregion_results.append(result)
+ results.append(subregion_results)
+ return results
+
http://git-wip-us.apache.org/repos/asf/climate/blob/c6f5ed75/ocw/tests/test_evaluation.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_evaluation.py b/ocw/tests/test_evaluation.py
index 566de86..2d48e72 100644
--- a/ocw/tests/test_evaluation.py
+++ b/ocw/tests/test_evaluation.py
@@ -149,8 +149,8 @@ class TestEvaluation(unittest.TestCase):
# Expected result shape is
# [stddev] where stddev.shape[0] = number of datasets
- self.assertTrue(len(new_eval.unary_results) == 2)
- self.assertTrue(new_eval.unary_results[1].shape[0] == 4)
+ self.assertTrue(len(new_eval.unary_results) == 1)
+ self.assertTrue(new_eval.unary_results[0].shape[0] == 5)
def test_subregion_result_shape(self):
bound = Bounds(
@@ -186,8 +186,8 @@ class TestEvaluation(unittest.TestCase):
new_eval = Evaluation(
self.test_dataset,
[self.another_test_dataset, self.another_test_dataset],
- [TemporalStdDev()],
- [bound]
+ [TemporalStdDev(), TemporalStdDev()],
+ [bound, bound, bound, bound, bound]
)
new_eval.run()
@@ -197,9 +197,10 @@ class TestEvaluation(unittest.TestCase):
# [3, temporalstddev.run(reference).shape],
# ]
# ]
- self.assertTrue(len(new_eval.unary_results) == 1)
+ self.assertTrue(len(new_eval.unary_results) == 5) # number of subregions
+ self.assertTrue(len(new_eval.unary_results[0]) == 2) # number of metrics
self.assertTrue(type(new_eval.unary_results) == type([]))
- self.assertTrue(new_eval.unary_results[0][0].shape[0] == 3)
+ self.assertTrue(new_eval.unary_results[0][0].shape[0] == 3) # number of datasets (ref + target)
if __name__ == '__main__':
[7/7] climate git commit: CLIMATE-658 - Restructure evaluation results
Posted by hu...@apache.org.
CLIMATE-658 - Restructure evaluation results
- now the evaluation results are provided as a list of missing arrays.
- update _run_no_subregion_evaluation and _run_unary_metric_evaluation in evaluation.py
- update test_evaluation
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/77443d7a
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/77443d7a
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/77443d7a
Branch: refs/heads/master
Commit: 77443d7add59a8eed0e9a46dfaf98cbeb9f2b390
Parents: 45779fb 956c389
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Tue Aug 25 10:05:52 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Tue Aug 25 10:05:52 2015 -0700
----------------------------------------------------------------------
examples/knmi_to_cru31_full_bias.py | 1 -
ocw/evaluation.py | 72 ++++++++++++++++++++++++++++++--
ocw/tests/test_evaluation.py | 65 ++++++++--------------------
3 files changed, 86 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
[6/7] climate git commit: restore the changes in the example
Posted by hu...@apache.org.
restore the changes in the example
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/956c389a
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/956c389a
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/956c389a
Branch: refs/heads/master
Commit: 956c389a2f7a0537a7cb7f26c5eb6dbf55e68a9c
Parents: c6f5ed7
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Thu Aug 20 20:06:09 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Thu Aug 20 20:06:09 2015 -0700
----------------------------------------------------------------------
examples/knmi_to_cru31_full_bias.py | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/956c389a/examples/knmi_to_cru31_full_bias.py
----------------------------------------------------------------------
diff --git a/examples/knmi_to_cru31_full_bias.py b/examples/knmi_to_cru31_full_bias.py
index ac629e9..a241442 100644
--- a/examples/knmi_to_cru31_full_bias.py
+++ b/examples/knmi_to_cru31_full_bias.py
@@ -28,9 +28,6 @@ import ocw.dataset_processor as dsp
import ocw.evaluation as evaluation
import ocw.metrics as metrics
import ocw.plotter as plotter
-import ssl
-if hasattr(ssl, '_create_unverified_context'):
- ssl._create_default_https_context = ssl._create_unverified_context
# File URL leader
FILE_LEADER = "http://zipper.jpl.nasa.gov/dist/"
# This way we can easily adjust the time span of the retrievals