You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by hu...@apache.org on 2015/07/15 07:08:19 UTC
[06/12] climate git commit: debug
debug
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/cefd9d63
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/cefd9d63
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/cefd9d63
Branch: refs/heads/master
Commit: cefd9d632017002f1c6abb0f650e3fb8416f0eab
Parents: 69acb0e
Author: Huikyo Lee <hu...@argo.jpl.nasa.gov>
Authored: Fri Mar 20 00:41:33 2015 -0700
Committer: Huikyo Lee <hu...@argo.jpl.nasa.gov>
Committed: Fri Mar 20 00:41:33 2015 -0700
----------------------------------------------------------------------
ocw/data_source/local.py | 178 ------------------------------------------
ocw/dataset.py | 22 +++---
ocw/utils.py | 4 -
3 files changed, 11 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index 3fe6497..2de03f5 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -26,7 +26,6 @@ import ocw.utils as utils
import netCDF4
import numpy
import numpy.ma as ma
-import glob
LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes']
LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes']
@@ -212,182 +211,6 @@ def load_file(file_path,
else:
values = values [:,:,:,elevation_index]
-<<<<<<< HEAD
- return Dataset(lats, lons, times, values, variable_name, name=name)
-
-def load_files(file_path,
- filename_pattern,
- variable_name,
- elevation_index=0,
- name='',
- lat_name=None,
- lon_name=None,
- time_name=None,
- latitude_range=None,
- longitude_range=None):
- ''' Load multiple NetCDF files whose file names have common patterns into a Dataset.
- The dataset can be spatially subset.
-
- :param file_path: Directory to the NetCDF file to load.
- :type file_path: :mod:`string`
-
- :param filename_pattern: Path to the NetCDF file to load.
- :type filename_pattern: :list:`string`
-
- :param variable_name: The variable name to load from the NetCDF file.
- :type variable_name: :mod:`string`
-
- :param elevation_index: (Optional) The elevation index for which data should
- be returned. Climate data is often times 4 dimensional data. Some
- datasets will have readins at different height/elevation levels. OCW
- expects 3D data so a single layer needs to be stripped out when loading.
- By default, the first elevation layer is used. If desired you may
- specify the elevation value to use.
- :type elevation_index: :class:`int`
-
- :param name: (Optional) A name for the loaded dataset.
- :type name: :mod:`string`
-
- :param lat_name: (Optional) The latitude variable name to extract from the
- dataset.
- :type lat_name: :mod:`string`
-
- :param lon_name: (Optional) The longitude variable name to extract from the
- dataset.
- :type lon_name: :mod:`string`
-
- :param time_name: (Optional) The time variable name to extract from the
- dataset.
- :type time_name: :mod:`string`
-
- :param latitude_range: (Optional) southern and northern boundary of the sub-region
- :type latitude_range: :list:float
-
- :param longitude_range: (Optional) western and eastern boundary of the sub-region
- :type longitude_range: :list:float
-
- :returns: An OCW Dataset object with the requested variable's data from
- the NetCDF file.
- :rtype: :class:`dataset.Dataset`
-
- :raises ValueError: When the specified file path cannot be loaded by ndfCDF4
- or when the lat/lon/time variable name cannot be determined
- automatically.
- '''
-
- netcdf_files= []
- for pattern in filename_pattern:
- netcdf_files.extend(glob.glob(file_path+pattern))
- netcdf_files.sort()
-
- try:
- netcdf = netCDF4.Dataset(netcdf_files[0], mode='r')
- except RuntimeError:
- err = "Dataset filepath is invalid. Please ensure it is correct."
- raise ValueError(err)
- except:
- err = (
- "The given file cannot be loaded. Please ensure that it is a valid "
- "NetCDF file. If problems persist, report them to the project's "
- "mailing list."
- )
- raise ValueError(err)
-
- if not lat_name:
- lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
- if not lon_name:
- lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
- if not time_name:
- time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
-
- lats = netcdf.variables[lat_name][:]
- lons = netcdf.variables[lon_name][:]
-
- if latitude_range and longitude_range:
- if lats.ndim == 1:
- x_index = numpy.where((lons>=numpy.min(longitude_range)) & (lons<=numpy.max(longitude_range)))[0]
- y_index = numpy.where((lats>=numpy.min(latitude_range)) & (lats<=numpy.max(latitude_range)))[0]
- lats = lats[y_index]
- lons = lons[x_index]
- else:
- y_index,x_index = numpy.where((lons>=numpy.min(longitude_range)) & (lons<=numpy.max(longitude_range)) & (lats>=numpy.min(latitude_range)) & (lats<=numpy.max(latitude_range)))
- lats = lats[y_index, x_index]
- lons = lons[y_index, x_index]
- else:
- y_index = np.arange(lats.shape[0])
- x_index = np.arange(lons.shape[-1])
-
- time_raw_values = netcdf.variables[time_name]
- for attr, value in time_raw_values.__dict__.iteritems():
- if 'unit' in attr.lower():
- time_unit = value
- times = netCDF4.num2date(time_raw_values[:], units = time_unit)
- times = numpy.array(times)
-
- # check the variable structure before reading data from the open file
- variable = netcdf.variables[variable_name]
- # If the values are 4D then we need to strip out the elevation index
- if len(variable.shape) == 4:
- # Determine the set of possible elevation dimension names excluding
- # the list of names that are used for the lat, lon, and time values.
- dims = netcdf.variables[variable_name].dimensions
- dimension_names = [dim_name.encode() for dim_name in dims]
- lat_lon_time_var_names = [lat_name, lon_name, time_name]
-
- elev_names = set(dimension_names) - set(lat_lon_time_var_names)
-
- # Grab the index value for the elevation values
- level_index = dimension_names.index(elev_names.pop())
-
- # Strip out the elevation values so we're left with a 3D array.
- if level_index == 0:
- values = variable[elevation_index,:,y_index,x_index]
- elif level_index == 1:
- values = variable[:,elevation_index,y_index,x_index]
- else:
- raise ValueError('The structure of this variable does not follow the community standard')
- if len(netcdf_files) >1:
- for netcdf_file in netcdf_files[1:]:
- netcdf.close()
- netcdf = netCDF4.Dataset(netcdf_file, mode='r')
- time_raw_values = netcdf.variables[time_name]
- for attr, value in time_raw_values.__dict__.iteritems():
- if 'unit' in attr.lower():
- time_unit = value
- times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units = time_unit))
- if level_index == 0:
- values = numpy.concatenate((values, netcdf.variables[variable_name][elevation_index,:,y_index,x_index]), axis=0)
- elif level_index == 1:
- values = numpy.concatenate((values, netcdf.variables[variable_name][:,elevation_index,y_index,x_index]), axis=0)
-
- elif len(variable.shape) == 3:
- values = variable[:,y_index,x_index]
-
- if len(netcdf_files) >1:
- for netcdf_file in netcdf_files[1:]:
- netcdf.close()
- netcdf = netCDF4.Dataset(netcdf_file, mode='r')
- time_raw_values = netcdf.variables[time_name]
- for attr, value in time_raw_values.__dict__.iteritems():
- if 'unit' in attr.lower():
- time_unit = value
- times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units=time_unit))
- values = numpy.concatenate((values, netcdf.variables[variable_name][:,y_index,x_index]), axis=0)
- elif len(variable.shape) == 2:
- values = (variable[y_index,x_index]).reshape((1,y_index.size,x_index.size))
- if len(netcdf_files) >1:
- for netcdf_file in netcdf_files[1:]:
- netcdf.close()
- netcdf = netCDF4.Dataset(netcdf_file, mode='r')
- time_raw_values = netcdf.variables[time_name]
- for attr, value in time_raw_values.__dict__.iteritems():
- if 'unit' in attr.lower():
- time_unit = value
- times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units=time_unit))
- values = numpy.concatenate((values, (netcdf.variables[variable_name][y_index,x_index]).reshape((1,y_index.size,x_index.size))), axis=0)
- return Dataset(lats, lons, times, values, variable_name, name=name)
-
-=======
origin = {
'source': 'local',
'path': file_path,
@@ -399,4 +222,3 @@ def load_files(file_path,
return Dataset(lats, lons, times, values, variable=variable_name,
units=variable_unit, name=name, origin=origin)
->>>>>>> upstream/master
http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/dataset.py
----------------------------------------------------------------------
diff --git a/ocw/dataset.py b/ocw/dataset.py
index 48e7a22..ee86532 100644
--- a/ocw/dataset.py
+++ b/ocw/dataset.py
@@ -69,7 +69,7 @@ class Dataset:
:raises: ValueError
'''
self._validate_inputs(lats, lons, times, values)
- #lats, lons, values = utils.normalize_lat_lon_values(lats, lons, values)
+ lats, lons, values = utils.normalize_lat_lon_values(lats, lons, values)
self.lats = lats
self.lons = lons
@@ -180,17 +180,17 @@ class Dataset:
err_msg = "Longitude Array should be 1 dimensional. %s dimensions found." % lon_dim
elif time_dim != 1:
err_msg = "Time Array should be 1 dimensional. %s dimensions found." % time_dim
- #elif value_dim != 3:
- # err_msg = "Value Array should be 3 dimensional. %s dimensions found." % value_dim
+ elif value_dim != 3:
+ err_msg = "Value Array should be 3 dimensional. %s dimensions found." % value_dim
# Finally check that the Values array conforms to the proper shape
- #elif values.shape != (time_count, lat_count, lon_count):
- # err_msg = """Value Array must be of shape (times, lats, lons).
-#Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count,
-# lat_count,
-# lon_count,
-# values.shape[0],
-# values.shape[1],
-# values.shape[2])
+ elif values.shape != (time_count, lat_count, lon_count):
+ err_msg = """Value Array must be of shape (times, lats, lons).
+Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count,
+ lat_count,
+ lon_count,
+ values.shape[0],
+ values.shape[1],
+ values.shape[2])
if err_msg:
logger.error(err_msg)
raise ValueError(err_msg)
http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/utils.py
----------------------------------------------------------------------
diff --git a/ocw/utils.py b/ocw/utils.py
index e3d3311..e778d40 100755
--- a/ocw/utils.py
+++ b/ocw/utils.py
@@ -289,7 +289,6 @@ def calc_climatology_year(dataset):
return annually_mean, total_mean
-<<<<<<< HEAD
def calc_climatology_season(month_start, month_end, dataset):
''' Calculate seasonal mean and time series for given months.
@@ -345,6 +344,3 @@ def calc_climatology_monthly(dataset):
raise ValueError(error)
else:
return reshape_monthly_to_annually(dataset).mean(axis=0)
-
-=======
->>>>>>> cfb120e5eaea004e850b884e49d41f8a0c269b75