You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by hu...@apache.org on 2016/01/30 01:45:00 UTC
[1/3] climate git commit: CLIMATE-732 - Update
data_processor.temporal_rebin
Repository: climate
Updated Branches:
refs/heads/master 078316308 -> 947478539
CLIMATE-732 - Update data_processor.temporal_rebin
- efficiency of dataset_processor.temporal_rebin has been significantly improved
- the option for temporal resolution needs to be given as characters.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/3fec4826
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/3fec4826
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/3fec4826
Branch: refs/heads/master
Commit: 3fec482641694a4917652e4bd74b6d88a872cd96
Parents: 64d3668
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Fri Jan 29 15:43:32 2016 -0800
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Fri Jan 29 15:43:32 2016 -0800
----------------------------------------------------------------------
ocw/dataset_processor.py | 83 +++++++++++++++++++++++++++++++++++--------
1 file changed, 68 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/3fec4826/ocw/dataset_processor.py
----------------------------------------------------------------------
diff --git a/ocw/dataset_processor.py b/ocw/dataset_processor.py
index 9095883..40f6cc8 100755
--- a/ocw/dataset_processor.py
+++ b/ocw/dataset_processor.py
@@ -104,33 +104,22 @@ def temporal_subset(month_start, month_end, target_dataset, average_each_year=Fa
return new_dataset
-def temporal_rebin(target_dataset, temporal_resolution):
+def temporal_rebin(target_dataset, time_unit):
""" Rebin a Dataset to a new temporal resolution
:param target_dataset: Dataset object that needs temporal rebinned
:type target_dataset: :class:`dataset.Dataset`
- :param temporal_resolution: The new temporal bin size
- :type temporal_resolution: :class:`datetime.timedelta`
+ :param temporal_resolution: The new temporal resolution
+ :type temporal_resolution: :mod:`string`
:returns: A new temporally rebinned Dataset
:rtype: :class:`dataset.Dataset`
"""
# Decode the temporal resolution into a string format that
# _rcmes_calc_average_on_new_time_unit_K() can understand
- day_count = temporal_resolution.days
- time_unit = None
- if day_count == 1:
- time_unit = 'daily'
- elif day_count > 1 and day_count <= 31:
- time_unit = 'monthly'
- elif day_count > 31 and day_count <= 366:
- time_unit = 'annual'
- else:
- time_unit = 'full'
- masked_values = target_dataset.values.view(ma.MaskedArray)
- binned_values, binned_dates = _rcmes_calc_average_on_new_time_unit_K(masked_values, target_dataset.times, time_unit)
+ binned_values, binned_dates = _rcmes_calc_average_on_new_time_unit(target_dataset.values, target_dataset.times, time_unit)
binned_dates = np.array(binned_dates)
new_dataset = ds.Dataset(target_dataset.lats,
target_dataset.lons,
@@ -886,6 +875,70 @@ def _rcmes_create_mask_using_threshold(masked_array, threshold=0.5):
return mymask
+def _rcmes_calc_average_on_new_time_unit(data, dates, unit):
+ """ Rebin 3d array and list of dates using the provided unit parameter
+
+ :param data: Input data that needs to be averaged
+ :type data: 3D masked numpy array of shape (times, lats, lons)
+ :param dates: List of dates that correspond to the given data values
+ :type dates: Python datetime objects
+ :param unit: Time unit to average the data into
+ :type unit: String matching one of these values : full | annual | monthly | daily
+
+ :returns: meanstorem, newTimesList
+ :rtype: 3D numpy masked array the same shape as the input array, list of python datetime objects
+ """
+
+ # Check if the user-selected temporal grid is valid. If not, EXIT
+ acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging: EXIT'
+ return -1,-1,-1,-1
+
+ nt, ny, nx = data.shape
+ if unit == 'full':
+ new_data = ma.mean(data, axis=0)
+ new_date = [dates[size(dates)/2]]
+ if unit == 'annual':
+ years = [d.year for d in target_dataset.times]
+ years_sorted = np.unique(years)
+ new_data = ma.zeros([years_sorted.size, ny, nx])
+ it = 0
+ new_date = []
+ for year in years_sorted:
+ index = np.where(years == year)[0]
+ new_data[it,:] = ma.mean(data[index,:], axis=0)
+ new_date.append(datetime.datetime(year=year, month=7, day=2))
+ it = it+1
+ if unit == 'monthly':
+ years = [d.year for d in target_dataset.times]
+ years_sorted = np.unique(years)
+ months = [d.month for d in target_dataset.times]
+ months_sorted = np.unique(months)
+
+ new_data = ma.zeros([years_sorted.size*months_sorted.size, ny, nx])
+ it = 0
+ new_date = []
+ for year in years_sorted:
+ for month in months_sorted:
+ index = np.where((years == year) & (months == month))[0]
+ new_data[it,:] = ma.mean(data[index,:], axis=0)
+ new_date.append(datetime.datetime(year=year, month=month, day=15))
+ it = it+1
+ if unit == 'daily':
+ dates = [d.year*10000.+d.month*100.+d.day for d in target_dataset.times]
+ dates_sorted = np.unique(dates)
+
+ new_data = ma.zeros([dates_sorted.size, ny, nx])
+ it = 0
+ new_date = []
+ for date in dates_sorted:
+ index = np.where(dates == date)[0]
+ new_data[it,:] = ma.mean(data[index,:], axis=0)
+ new_date.append(datetime.datetime(year=date/10000, month=(date % 10000)/100, day=date % 100))
+ it = it+1
+
+ return new_data, np.array(new_date)
def _rcmes_calc_average_on_new_time_unit_K(data, dates, unit):
""" Rebin 3d array and list of dates using the provided unit parameter
[2/3] climate git commit: Examples that use
dataset_processor.temporal_rebin have been updated
Posted by hu...@apache.org.
Examples that use dataset_processor.temporal_rebin have been updated
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/09c01813
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/09c01813
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/09c01813
Branch: refs/heads/master
Commit: 09c018135a935f6552e8ee262bd20faed07e3d56
Parents: 3fec482
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Fri Jan 29 16:08:07 2016 -0800
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Fri Jan 29 16:08:07 2016 -0800
----------------------------------------------------------------------
RCMES/test/test.py | 6 +++---
examples/knmi_to_cru31_full_bias.py | 6 +++---
examples/model_ensemble_to_rcmed.py | 8 ++++----
examples/multi_model_taylor_diagram.py | 4 ++--
examples/simple_model_to_model_bias.py | 4 ++--
examples/taylor_diagram_example.py | 4 ++--
6 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/09c01813/RCMES/test/test.py
----------------------------------------------------------------------
diff --git a/RCMES/test/test.py b/RCMES/test/test.py
index beab16f..bbb8095 100644
--- a/RCMES/test/test.py
+++ b/RCMES/test/test.py
@@ -118,9 +118,9 @@ print("CRU31_Dataset.values shape: (times, lats, lons) - %s" % (cru31_dataset.va
print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,))
print("Temporally Rebinning the Datasets to a Single Timestep")
-# To run FULL temporal Rebinning use a timedelta > 366 days. I used 999 in this example
-knmi_dataset = dsp.temporal_rebin(knmi_dataset, datetime.timedelta(days=999))
-cru31_dataset = dsp.temporal_rebin(cru31_dataset, datetime.timedelta(days=999))
+# To run FULL temporal Rebinning,
+knmi_dataset = dsp.temporal_rebin(knmi_dataset, temporal_resolution='full')
+cru31_dataset = dsp.temporal_rebin(cru31_dataset, temporal_resolution='full')
print("KNMI_Dataset.values shape: %s" % (knmi_dataset.values.shape,))
print("CRU31_Dataset.values shape: %s \n\n" % (cru31_dataset.values.shape,))
http://git-wip-us.apache.org/repos/asf/climate/blob/09c01813/examples/knmi_to_cru31_full_bias.py
----------------------------------------------------------------------
diff --git a/examples/knmi_to_cru31_full_bias.py b/examples/knmi_to_cru31_full_bias.py
index beab16f..e37e887 100644
--- a/examples/knmi_to_cru31_full_bias.py
+++ b/examples/knmi_to_cru31_full_bias.py
@@ -118,9 +118,9 @@ print("CRU31_Dataset.values shape: (times, lats, lons) - %s" % (cru31_dataset.va
print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,))
print("Temporally Rebinning the Datasets to a Single Timestep")
-# To run FULL temporal Rebinning use a timedelta > 366 days. I used 999 in this example
-knmi_dataset = dsp.temporal_rebin(knmi_dataset, datetime.timedelta(days=999))
-cru31_dataset = dsp.temporal_rebin(cru31_dataset, datetime.timedelta(days=999))
+# To run FULL temporal Rebinning
+knmi_dataset = dsp.temporal_rebin(knmi_dataset, temporal_resolution = 'full')
+cru31_dataset = dsp.temporal_rebin(cru31_dataset, temporal_resolution = 'full')
print("KNMI_Dataset.values shape: %s" % (knmi_dataset.values.shape,))
print("CRU31_Dataset.values shape: %s \n\n" % (cru31_dataset.values.shape,))
http://git-wip-us.apache.org/repos/asf/climate/blob/09c01813/examples/model_ensemble_to_rcmed.py
----------------------------------------------------------------------
diff --git a/examples/model_ensemble_to_rcmed.py b/examples/model_ensemble_to_rcmed.py
index 45ab599..a9303dd 100644
--- a/examples/model_ensemble_to_rcmed.py
+++ b/examples/model_ensemble_to_rcmed.py
@@ -120,10 +120,10 @@ cru31_dataset = rcmed.parameter_dataset(dataset_id,
""" Step 3: Resample Datasets so they are the same shape """
print("Temporally Rebinning the Datasets to an Annual Timestep")
-# To run annual temporal Rebinning use a timedelta of 360 days.
-knmi_dataset = dsp.temporal_rebin(knmi_dataset, datetime.timedelta(days=360))
-wrf311_dataset = dsp.temporal_rebin(wrf311_dataset, datetime.timedelta(days=360))
-cru31_dataset = dsp.temporal_rebin(cru31_dataset, datetime.timedelta(days=360))
+# To run annual temporal Rebinning,
+knmi_dataset = dsp.temporal_rebin(knmi_dataset, temporal_resolution = 'annual')
+wrf311_dataset = dsp.temporal_rebin(wrf311_dataset, temporal_resolution = 'annual')
+cru31_dataset = dsp.temporal_rebin(cru31_dataset, temporal_resolution = 'annual')
# Running Temporal Rebin early helps negate the issue of datasets being on different
# days of the month (1st vs. 15th)
http://git-wip-us.apache.org/repos/asf/climate/blob/09c01813/examples/multi_model_taylor_diagram.py
----------------------------------------------------------------------
diff --git a/examples/multi_model_taylor_diagram.py b/examples/multi_model_taylor_diagram.py
index 48fc736..57dabdd 100644
--- a/examples/multi_model_taylor_diagram.py
+++ b/examples/multi_model_taylor_diagram.py
@@ -81,11 +81,11 @@ print("Resampling datasets ...")
print("... on units")
CRU31 = dsp.water_flux_unit_conversion(CRU31)
print("... temporal")
-CRU31 = dsp.temporal_rebin(CRU31, datetime.timedelta(days=30))
+CRU31 = dsp.temporal_rebin(CRU31, temporal_resolution = 'monthly')
for member, each_target_dataset in enumerate(target_datasets):
target_datasets[member] = dsp.water_flux_unit_conversion(target_datasets[member])
- target_datasets[member] = dsp.temporal_rebin(target_datasets[member], datetime.timedelta(days=30))
+ target_datasets[member] = dsp.temporal_rebin(target_datasets[member], temporal_resolution = 'monthly')
target_datasets[member] = dsp.subset(EVAL_BOUNDS, target_datasets[member])
#Regrid
http://git-wip-us.apache.org/repos/asf/climate/blob/09c01813/examples/simple_model_to_model_bias.py
----------------------------------------------------------------------
diff --git a/examples/simple_model_to_model_bias.py b/examples/simple_model_to_model_bias.py
index 635e872..44d482b 100644
--- a/examples/simple_model_to_model_bias.py
+++ b/examples/simple_model_to_model_bias.py
@@ -54,8 +54,8 @@ print("WRF_Dataset.values shape: (times, lats, lons) - %s \n" % (wrf_dataset.val
""" Step 2: Temporally Rebin the Data into an Annual Timestep """
print("Temporally Rebinning the Datasets to an Annual Timestep")
-knmi_dataset = dsp.temporal_rebin(knmi_dataset, datetime.timedelta(days=365))
-wrf_dataset = dsp.temporal_rebin(wrf_dataset, datetime.timedelta(days=365))
+knmi_dataset = dsp.temporal_rebin(knmi_dataset, temporal_resolution='annual')
+wrf_dataset = dsp.temporal_rebin(wrf_dataset, temporal_resolution='annual')
print("KNMI_Dataset.values shape: %s" % (knmi_dataset.values.shape,))
print("WRF_Dataset.values shape: %s \n\n" % (wrf_dataset.values.shape,))
http://git-wip-us.apache.org/repos/asf/climate/blob/09c01813/examples/taylor_diagram_example.py
----------------------------------------------------------------------
diff --git a/examples/taylor_diagram_example.py b/examples/taylor_diagram_example.py
index 4ed803e..90c6708 100644
--- a/examples/taylor_diagram_example.py
+++ b/examples/taylor_diagram_example.py
@@ -67,8 +67,8 @@ wrf_dataset = dsp.subset(subset, wrf_dataset)
# Temporally re-bin the data into a monthly timestep.
################################################################################
-knmi_dataset = dsp.temporal_rebin(knmi_dataset, datetime.timedelta(days=30))
-wrf_dataset = dsp.temporal_rebin(wrf_dataset, datetime.timedelta(days=30))
+knmi_dataset = dsp.temporal_rebin(knmi_dataset, temporal_resolution = 'monthly')
+wrf_dataset = dsp.temporal_rebin(wrf_dataset, temporal_resolution = 'monthly')
# Spatially regrid the datasets onto a 1 degree grid.
################################################################################
[3/3] climate git commit: CLIMATE-732 - Update
data_processor.temporal_rebin
Posted by hu...@apache.org.
CLIMATE-732 - Update data_processor.temporal_rebin
- efficiency of dataset_processor.temporal_rebin has been significantly improved
- the option for temporal resolution needs to be given as characters.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/94747853
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/94747853
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/94747853
Branch: refs/heads/master
Commit: 947478539540fd7eb1e3ee9064be74fc4d3bd484
Parents: 0783163 09c0181
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Fri Jan 29 16:44:16 2016 -0800
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Fri Jan 29 16:44:16 2016 -0800
----------------------------------------------------------------------
RCMES/test/test.py | 6 +--
examples/knmi_to_cru31_full_bias.py | 6 +--
examples/model_ensemble_to_rcmed.py | 8 +--
examples/multi_model_taylor_diagram.py | 4 +-
examples/simple_model_to_model_bias.py | 4 +-
examples/taylor_diagram_example.py | 4 +-
ocw/dataset_processor.py | 83 +++++++++++++++++++++++------
7 files changed, 84 insertions(+), 31 deletions(-)
----------------------------------------------------------------------