You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by go...@apache.org on 2013/08/20 00:13:24 UTC
svn commit: r1515640 -
/incubator/climate/branches/RefactorInput/ocw/dataset_processor.py
Author: goodale
Date: Mon Aug 19 22:13:23 2013
New Revision: 1515640
URL: http://svn.apache.org/r1515640
Log:
GENERAL WORK: Pulled the code that makes new Year, Month, Day values based on the output unit, to a function that is called now, instead of duplicate blocks of code.
Also changed it so a variable for unique_times is created once, then used in the code instead of calling np.unique(times) repeatedly.
Modified:
incubator/climate/branches/RefactorInput/ocw/dataset_processor.py
Modified: incubator/climate/branches/RefactorInput/ocw/dataset_processor.py
URL: http://svn.apache.org/viewvc/incubator/climate/branches/RefactorInput/ocw/dataset_processor.py?rev=1515640&r1=1515639&r2=1515640&view=diff
==============================================================================
--- incubator/climate/branches/RefactorInput/ocw/dataset_processor.py (original)
+++ incubator/climate/branches/RefactorInput/ocw/dataset_processor.py Mon Aug 19 22:13:23 2013
@@ -312,14 +312,17 @@ def _rcmes_calc_average_on_new_time_unit
# Year list
if unit=='annual':
timeunits = np.array([int(d.strftime("%Y")) for d in dates])
+ unique_times = np.unique(timeunits)
# YearMonth format list
if unit=='monthly':
timeunits = np.array([int(d.strftime("%Y%m")) for d in dates])
+ unique_times = np.unique(timeunits)
# YearMonthDay format list
if unit=='daily':
timeunits = np.array([int(d.strftime("%Y%m%d")) for d in dates])
+ unique_times = np.unique(timeunits)
# TODO: add pentad setting using Julian days?
@@ -331,6 +334,7 @@ def _rcmes_calc_average_on_new_time_unit
for i in np.arange(len(dates)):
timeunits.append(999) # i.e. we just want the same value for all times.
timeunits = np.array(timeunits, dtype=int)
+ unique_times = np.unique(timeunits)
# empty list to store new times
newTimesList = []
@@ -339,55 +343,35 @@ def _rcmes_calc_average_on_new_time_unit
# i.e. if data are already on required time unit then just pass data through and
# calculate and return representative datetimes.
processing_required = True
- if len(timeunits)==(len(np.unique(timeunits))):
+ if len(timeunits)==(len(unique_times)):
processing_required = False
# 1D data arrays, i.e. time series
if data.ndim==1:
# Create array to store the resulting data
- meanstore = np.zeros(len(np.unique(timeunits)))
+ meanstore = np.zeros(len(unique_times))
# Calculate the means across each unique time unit
i=0
- for myunit in np.unique(timeunits):
+ for myunit in unique_times:
if processing_required:
datam=ma.masked_array(data,timeunits!=myunit)
meanstore[i] = ma.average(datam)
# construct new times list
- smyunit = str(myunit)
- if len(smyunit)==4: # YYYY
- yyyy = int(myunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit)==6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit)==8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit)==3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dates[-1]-dates[0]
- halfway = dates[0]+(dt/2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
-
+ yyyy, mm, dd = _create_new_year_month_day(myunit, dates)
newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
i = i+1
# 3D data arrays
if data.ndim==3:
# Create array to store the resulting data
- meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
+ meanstore = np.zeros([len(unique_times),data.shape[1],data.shape[2]])
# Calculate the means across each unique time unit
i=0
datamask_store = []
- for myunit in np.unique(timeunits):
+ for myunit in unique_times:
if processing_required:
mask = np.zeros_like(data)
mask[timeunits!=myunit,:,:] = 1.0
@@ -400,26 +384,7 @@ def _rcmes_calc_average_on_new_time_unit
datam = ma.masked_array(data,np.logical_or(mask,datamask_at_this_timeunit))
meanstore[i,:,:] = ma.average(datam,axis=0)
# construct new times list
- smyunit = str(myunit)
- if len(smyunit)==4: # YYYY
- yyyy = int(smyunit[0:4])
- mm = 1
- dd = 1
- elif len(smyunit)==6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- elif len(smyunit)==8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- elif len(smyunit)==3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dates[-1]-dates[0]
- halfway = dates[0]+(dt/2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
+ yyyy, mm, dd = _create_new_year_month_day(myunit, dates)
newTimesList.append(datetime.datetime(yyyy,mm,dd))
i += 1
@@ -433,6 +398,30 @@ def _rcmes_calc_average_on_new_time_unit
return meanstorem, newTimesList
+def _create_new_year_month_day(time_unit, dates):
+ smyunit = str(time_unit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ elif len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ elif len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ elif len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dates[-1]-dates[0]
+ halfway = dates[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ return (yyyy, mm, dd)
+
def _congrid(a, newdims, method='linear', centre=False, minusone=False):
'''
This function is from http://wiki.scipy.org/Cookbook/Rebinning - Example 3