You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by jo...@apache.org on 2014/11/07 22:39:38 UTC
[1/6] climate git commit: CLIMATE-552 - Drop unused private functions
from local.py
Repository: climate
Updated Branches:
refs/heads/master cff5b5459 -> 227b5de93
CLIMATE-552 - Drop unused private functions from local.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/99b57325
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/99b57325
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/99b57325
Branch: refs/heads/master
Commit: 99b57325dbe9750f237f5b3ff32fabc92ab3e9f0
Parents: cff5b54
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Nov 7 11:06:20 2014 -0800
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Nov 7 11:06:20 2014 -0800
----------------------------------------------------------------------
ocw/data_source/local.py | 107 ------------------------------------------
1 file changed, 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/99b57325/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index 260029e..b72cd70 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -92,107 +92,6 @@ def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
)
raise ValueError(error)
-
-def _get_lat_name(variable_names):
- '''Find the latitude variable name
-
- :param variable_names: List of netCDF variables' name
- :type variable_names: List
-
- :returns: Latitude variable name
- :rtype: String
- '''
-
- common_name = set(['lat', 'lats', 'latitude', 'latitudes']).intersection(variable_names)
- if len(common_name) !=1:
- err = "Unable to autodetect latitude variable name."
- raise ValueError(err)
- else:
- lat_variable_name = common_name.pop()
-
- return lat_variable_name
-
-
-def _get_lon_name(variable_names):
- '''Find the longitude variable name
-
- :param variable_names: List of netCDF variables' name
- :type variable_names: List
-
- :returns: Longitude variable name
- :rtype: String
- '''
-
- common_name = set(['lon', 'lons', 'longitude', 'longitudes']).intersection(variable_names)
- if len(common_name) !=1:
- err = "Unable to autodetect longitude variable name."
- raise ValueError(err)
- else:
- lon_variable_name = common_name.pop()
-
- return lon_variable_name
-
-
-def _get_time_name(variable_names):
- '''Find the time variable name.
-
- :param: variableNameList: List of netCDF variables' name
- :type variable_names: List
-
- :returns: Time variable name
- :rtype: String
- '''
-
- common_name = set(['time', 'times', 'date', 'dates', 'julian']).intersection(variable_names)
-
- if len(common_name) !=1:
- err = "Unable to autodetect time variable name. These option(s) found: {0} ".format([each for each in common_name])
- raise ValueError(err)
- else:
- time_variable_name = common_name.pop()
-
- return time_variable_name
-
-
-def _get_level_name(variable_names):
- '''Find the level variable name.
-
- :param variable_names: List of netCDF variables' name
- type variable_names: List
-
- :returns: Level variable name
- :rtype: String
- '''
-
- level_variable_name = None
- common_name = set(['lev', 'level', 'levels', 'height', 'heights', 'elev', 'elevation', 'elevations']).intersection(variable_names)
- if len(common_name) !=1:
- pass
- else:
- level_variable_name = common_name.pop()
-
- return level_variable_name
-
-
-def _get_value_name(possible_value_name):
- '''Find the value variable name.
-
- :param possible_value_name: List of all value variable names
- :type possible_value_name: List
-
- :returns: Value variable name
- :rtype: String
- '''
-
- if len(possible_value_name) == 1:
- value_variable_name = possible_value_name[0]
- else:
- err = "The given value variable name does not match with existing variables name."
- raise ValueError(err)
-
- return value_variable_name
-
-
def load_file(file_path, variable_name, elevation_index=0):
'''Load netCDF file, get the all variables name and get the data.
@@ -217,12 +116,6 @@ def load_file(file_path, variable_name, elevation_index=0):
lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
- #lat_variable_name = _get_lat_name(variable_names)
- #lon_variable_name = _get_lon_name(variable_names)
- #time_variable_name = _get_time_name(variable_names)
- #level_variable_name = _get_level_name(variable_names)
-
-
# Check returned variable dimensions. lats, lons, and times should be 1D
#
# Check dimensions of the values
[5/6] climate git commit: CLIMATE-552 - Clean up imports in local.py
Posted by jo...@apache.org.
CLIMATE-552 - Clean up imports in local.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/55ac1779
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/55ac1779
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/55ac1779
Branch: refs/heads/master
Commit: 55ac17798c071a923ce807cc6fc2e0894e972a8e
Parents: f3badc9
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Nov 7 13:29:14 2014 -0800
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Nov 7 13:29:14 2014 -0800
----------------------------------------------------------------------
ocw/data_source/local.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/55ac1779/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index e5ac7da..0cbecad 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -15,17 +15,18 @@
# specific language governing permissions and limitations
# under the License.
-import re
-import netCDF4
-from ocw.dataset import Dataset
-import numpy
-import numpy.ma as ma
-from datetime import timedelta ,datetime
import calendar
+from datetime import timedelta ,datetime
+import re
import string
+from ocw.dataset import Dataset
import ocw.utils as utils
+import netCDF4
+import numpy
+import numpy.ma as ma
+
LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes']
LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes']
TIME_NAMES = ['time', 'times', 'date', 'dates', 'julian']
[6/6] climate git commit: Resolve CLIMATE-552. Merge PR #130.
Posted by jo...@apache.org.
Resolve CLIMATE-552. Merge PR #130.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/227b5de9
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/227b5de9
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/227b5de9
Branch: refs/heads/master
Commit: 227b5de93bc542accfc5a8e1cd9fa3efeea4314d
Parents: cff5b54 55ac177
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Nov 7 13:39:10 2014 -0800
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Nov 7 13:39:10 2014 -0800
----------------------------------------------------------------------
ocw/data_source/local.py | 228 +++++++++++++-----------------------------
1 file changed, 69 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
[2/6] climate git commit: CLIMATE-552 - Remove dead code from local.py
Posted by jo...@apache.org.
CLIMATE-552 - Remove dead code from local.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/3992e0c6
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/3992e0c6
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/3992e0c6
Branch: refs/heads/master
Commit: 3992e0c61058770df7d4df90347e0e910db4dddc
Parents: 99b5732
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Nov 7 11:09:07 2014 -0800
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Nov 7 11:09:07 2014 -0800
----------------------------------------------------------------------
ocw/data_source/local.py | 21 ---------------------
1 file changed, 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/3992e0c6/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index b72cd70..85e27b3 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -116,26 +116,6 @@ def load_file(file_path, variable_name, elevation_index=0):
lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
- # Check returned variable dimensions. lats, lons, and times should be 1D
- #
- # Check dimensions of the values
- # if != 3
- # find the indices for lat, lon, time
- # strip out everything else by select 1st of possible options
- #
- # Check the order of the variables
- # if not correct order (times, lats, lons)
- # reorder as appropriate
- #
- # Make new dataset object
-
- '''
- if variable_name in variable_names:
- value_variable_name = variable_name
- else:
- possible_value_name = list(set(variable_names) - set([lat_variable_name, lon_variable_name, time_variable_name, level_variable_name]))
- value_variable_name = _get_value_name(possible_value_name)
- '''
lats = netcdf.variables[lat_name][:]
lons = netcdf.variables[lon_name][:]
time_raw_values = netcdf.variables[time_name][:]
@@ -145,7 +125,6 @@ def load_file(file_path, variable_name, elevation_index=0):
if len(values.shape) == 4:
- #value_dimensions_names = list(netcdf.variables[variable_name].dimensions)
value_dimensions_names = [dim_name.encode() for dim_name in netcdf.variables[variable_name].dimensions]
lat_lon_time_var_names = [lat_name, lon_name, time_name]
level_index = value_dimensions_names.index(list(set(value_dimensions_names) - set(lat_lon_time_var_names))[0])
[4/6] climate git commit: CLIMATE-552 - Line length fixes and
readability fixes
Posted by jo...@apache.org.
CLIMATE-552 - Line length fixes and readability fixes
- Correct PEP-8 incompatible line lengths.
- Adjust variable name and add additional comments to load_file for
clarity.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/f3badc99
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/f3badc99
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/f3badc99
Branch: refs/heads/master
Commit: f3badc99981dc6dff3a74e67433b048c3e5aab57
Parents: 32a9bd1
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Nov 7 13:22:26 2014 -0800
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Nov 7 13:22:26 2014 -0800
----------------------------------------------------------------------
ocw/data_source/local.py | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/f3badc99/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index 0875d0c..e5ac7da 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -134,7 +134,10 @@ def load_file(file_path, variable_name, elevation_index=0):
try:
netcdf = netCDF4.Dataset(file_path, mode='r')
except:
- err = "The given file cannot be loaded (Only netCDF file can be supported)."
+ err = (
+ "The given file cannot be loaded "
+ "(Only NetCDF file can be supported)."
+ )
raise ValueError(err)
lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
@@ -148,10 +151,20 @@ def load_file(file_path, variable_name, elevation_index=0):
times = numpy.array(times)
values = ma.array(netcdf.variables[variable_name][:])
+ # If the values are 4D then we need to strip out the elevation index
if len(values.shape) == 4:
- value_dimensions_names = [dim_name.encode() for dim_name in netcdf.variables[variable_name].dimensions]
+ # Determine the set of possible elevation dimension names excluding
+ # the list of names that are used for the lat, lon, and time values.
+ dims = netcdf.variables[variable_name].dimensions
+ dimension_names = [dim_name.encode() for dim_name in dims]
lat_lon_time_var_names = [lat_name, lon_name, time_name]
- level_index = value_dimensions_names.index(list(set(value_dimensions_names) - set(lat_lon_time_var_names))[0])
+
+ elev_names = set(dimension_names) - set(lat_lon_time_var_names)
+
+ # Grab the index value for the elevation values
+ level_index = dimension_names.index(elev_names.pop())
+
+ # Strip out the elevation values so we're left with a 3D array.
if level_index == 0:
values = values [elevation_index,:,:,:]
elif level_index == 1:
[3/6] climate git commit: CLIMATE-552 - Comments and minor code
cleanup
Posted by jo...@apache.org.
CLIMATE-552 - Comments and minor code cleanup
- Cleanup doc strings and in-line documentation.
- Remove unneeded list conversions of sets.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/32a9bd19
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/32a9bd19
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/32a9bd19
Branch: refs/heads/master
Commit: 32a9bd192dcd7e96f7e71061bffe86f20db3d563
Parents: 3992e0c
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Nov 7 12:00:08 2014 -0800
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Nov 7 12:00:08 2014 -0800
----------------------------------------------------------------------
ocw/data_source/local.py | 68 +++++++++++++++++++++++++++++--------------
1 file changed, 46 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/32a9bd19/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index 85e27b3..0875d0c 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -32,46 +32,61 @@ TIME_NAMES = ['time', 'times', 'date', 'dates', 'julian']
def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
- '''Return valid variable from given netCDF object.
+ ''' Determine if one of a set of variable names are in a NetCDF Dataset.
- Looks for an occurrence of a valid_var_name in the netcdf variable data.
- If multiple possible matches are found a ValueError is raised. If no
- matching variable names are found a Value is raised.
+ Looks for an occurrence of a valid_var_name in the NetCDF variable data.
+ This is useful for automatically determining the names of the lat, lon,
+ and time variable names inside of a dataset object.
:param valid_var_names: The possible variable names to search for in
the netCDF object.
:type valid_var_names: List of Strings
- :param netcdf: The netCDF object in which to check for valid_var_names.
+ :param netcdf: The netCDF Dataset object in which to check for
+ valid_var_names.
:type netcdf: netcdf4.Dataset
:param netcdf_var: The relevant variable name to search over in the
- netcdf object.
+ netcdf object. This is used to narrow down the search for valid
+ variable names by first checking the desired variable's dimension
+ values for one or more of the valid variable names.
:returns: The variable from valid_var_names that it locates in
the netCDF object.
- :raises: ValueError
+ :raises ValueError: When unable to locate a single matching variable
+ name in the NetCDF Dataset from the supplied list of valid variable
+ names.
'''
- # Check for valid variable names in netCDF value variable dimensions
+ # Check for valid variable names in netCDF variable dimensions
dimensions = netcdf.variables[netcdf_var].dimensions
dims_lower = [dim.encode().lower() for dim in dimensions]
- intersect = list(set(valid_var_names).intersection(dims_lower))
+ intersect = set(valid_var_names).intersection(dims_lower)
if len(intersect) == 1:
- index = dims_lower.index(intersect[0])
+ # Retrieve the name of the dimension where we found the matching
+ # variable name
+ index = dims_lower.index(intersect.pop())
dimension_name = dimensions[index].encode()
+ # Locate all of the variables that share the dimension that we matched
+ # earlier. If the dimension's name matches then that variable is
+ # potentially what we want to return to the user.
possible_vars = []
for var in netcdf.variables.keys():
var_dimensions = netcdf.variables[var].dimensions
+ # Skip any dimensions are > 1D
if len(var_dimensions) != 1:
continue
if var_dimensions[0].encode() == dimension_name:
possible_vars.append(var)
+ # If there are multiple variables with matching dimension names then we
+ # aren't able to determining the correct variable name using the
+ # variable dimensions. We need to try a different approach. Otherwise,
+ # we're done!
if len(possible_vars) == 1:
return possible_vars[0]
@@ -79,31 +94,41 @@ def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
variables = netcdf.variables.keys()
vars_lower = [var.encode().lower() for var in variables]
- intersect = list(set(valid_var_names).intersection(vars_lower))
+ intersect = set(valid_var_names).intersection(vars_lower)
if len(intersect) == 1:
- index = vars_lower.index(intersect[0])
+ index = vars_lower.index(intersect.pop())
return variables[index]
- # If we couldn't find a single matching valid variable name, we're
- # unable to load the file properly.
+ # If we couldn't locate a single matching valid variable then we're unable
+ # to automatically determine the variable names for the user.
error = (
- "Unable to locate a single matching variable name in NetCDF object. "
+ "Unable to locate a single matching variable name from the "
+ "supplied list of valid variable names. "
)
raise ValueError(error)
def load_file(file_path, variable_name, elevation_index=0):
- '''Load netCDF file, get the all variables name and get the data.
+ ''' Load a NetCDF file into a Dataset.
- :param file_path: NetCDF directory with file name
+ :param file_path: Path to the NetCDF file to load.
:type file_path: String
- :param variable_name: The given (by user) value variable name
+ :param variable_name: The variable name to load from the NetCDF file.
:type variable_name: String
-
- :returns: An OCW Dataset object containing the requested parameter data.
+ :param elevation_index: The elevation index for which data should be
+ returned. Climate data is often times 4 dimensional data. Some datasets
+ will have readins at different height/elevation levels. OCW expects
+ 3D data so a single layer needs to be stripped out when loading. By
+ default, the first elevation layer is used. If desired you may specify
+ the elevation value to use.
+
+ :returns: An OCW Dataset object with the requested variable's data from
+ the NetCDF file.
:rtype: ocw.dataset.Dataset object
- :raises: ValueError
+ :raises ValueError: When the specified file path cannot be loaded by ndfCDF4
+ or when the lat/lon/time variable name cannot be determined
+ automatically.
'''
try:
@@ -123,7 +148,6 @@ def load_file(file_path, variable_name, elevation_index=0):
times = numpy.array(times)
values = ma.array(netcdf.variables[variable_name][:])
-
if len(values.shape) == 4:
value_dimensions_names = [dim_name.encode() for dim_name in netcdf.variables[variable_name].dimensions]
lat_lon_time_var_names = [lat_name, lon_name, time_name]