You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by hu...@apache.org on 2015/09/25 17:55:19 UTC
[1/3] climate git commit: CLIMATE-672 - Update the input validation
function for OCW datasets
Repository: climate
Updated Branches:
refs/heads/master 4ad37eb93 -> 0f3e8860e
CLIMATE-672 - Update the input validation function for OCW datasets
- ocw.dataset._validate_inputs considers datasets on curvilinear grids
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/ea203b2e
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/ea203b2e
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/ea203b2e
Branch: refs/heads/master
Commit: ea203b2ef11e16bf18fe0aab7490ed8fb0cb5e16
Parents: 9eac1f6
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Mon Sep 21 17:05:29 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Mon Sep 21 17:05:29 2015 -0700
----------------------------------------------------------------------
ocw/dataset.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/ea203b2e/ocw/dataset.py
----------------------------------------------------------------------
diff --git a/ocw/dataset.py b/ocw/dataset.py
index e432928..d3d2cb2 100644
--- a/ocw/dataset.py
+++ b/ocw/dataset.py
@@ -182,15 +182,16 @@ class Dataset:
elif value_dim < 2:
err_msg = "Value Array should be at least 2 dimensional. %s dimensions found." % value_dim
# Finally check that the Values array conforms to the proper shape
- if value_dim == 2 and values.shape != (lat_count, lon_count):
- err_msg = """Value Array must be of shape (times, lats, lons).
- Expected shape (%s, %s) but received (%s, %s)""" % (lat_count,
+ if value_dim == 2:
+ if values.shape[0] != time_count and values.shape != (lat_count, lon_count):
+ err_msg = """Value Array must be of shape (lats, lons) or (times, locations).
+ Expected shape (%s, %s) but received (%s, %s)""" % (lat_count,
lon_count,
values.shape[0],
values.shape[1])
if value_dim == 3 and values.shape != (time_count, lat_count, lon_count):
err_msg = """Value Array must be of shape (times, lats, lons).
- Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count,
+ Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count,
lat_count,
lon_count,
values.shape[0],
[2/3] climate git commit: the changes in CLIMATE-671 have been
separated
Posted by hu...@apache.org.
the changes in CLIMATE-671 have been separated
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/c596a5d1
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/c596a5d1
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/c596a5d1
Branch: refs/heads/master
Commit: c596a5d152933da1287f147463d2fac8b9c7c644
Parents: ea203b2
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Mon Sep 21 17:13:47 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Mon Sep 21 17:13:47 2015 -0700
----------------------------------------------------------------------
ocw/dataset_processor.py | 147 ++++++++++++++++--------------------------
1 file changed, 54 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/c596a5d1/ocw/dataset_processor.py
----------------------------------------------------------------------
diff --git a/ocw/dataset_processor.py b/ocw/dataset_processor.py
index cb92171..d7b1a3f 100755
--- a/ocw/dataset_processor.py
+++ b/ocw/dataset_processor.py
@@ -257,75 +257,48 @@ def subset(subregion, target_dataset, subregion_name=None):
subregion_name = target_dataset.name
# Slice the values array with our calculated slice indices
- if target_dataset.lats.ndim ==1 and target_dataset.lons.ndim ==1:
- if target_dataset.values.ndim == 2:
- subset_values = ma.zeros([len(target_dataset.values[
+ if target_dataset.values.ndim == 2:
+ subset_values = ma.zeros([len(target_dataset.values[
+ dataset_slices["lat_start"]:dataset_slices["lat_end"]]),
+ len(target_dataset.values[
+ dataset_slices["lon_start"]:dataset_slices["lon_end"]])])
+
+ subset_values = target_dataset.values[
+ dataset_slices["lat_start"]:dataset_slices["lat_end"] + 1,
+ dataset_slices["lon_start"]:dataset_slices["lon_end"] + 1]
+
+ elif target_dataset.values.ndim == 3:
+ subset_values = ma.zeros([len(target_dataset.values[
+ dataset_slices["time_start"]:dataset_slices["time_end"]]),
+ len(target_dataset.values[
dataset_slices["lat_start"]:dataset_slices["lat_end"]]),
- len(target_dataset.values[
- dataset_slices["lon_start"]:dataset_slices["lon_end"]])])
-
- subset_values = target_dataset.values[
- dataset_slices["lat_start"]:dataset_slices["lat_end"] + 1,
- dataset_slices["lon_start"]:dataset_slices["lon_end"] + 1]
-
- elif target_dataset.values.ndim == 3:
- subset_values = ma.zeros([len(target_dataset.values[
- dataset_slices["time_start"]:dataset_slices["time_end"]]),
- len(target_dataset.values[
- dataset_slices["lat_start"]:dataset_slices["lat_end"]]),
- len(target_dataset.values[
- dataset_slices["lon_start"]:dataset_slices["lon_end"]])])
+ len(target_dataset.values[
+ dataset_slices["lon_start"]:dataset_slices["lon_end"]])])
- subset_values = target_dataset.values[
- dataset_slices["time_start"]:dataset_slices["time_end"] + 1,
- dataset_slices["lat_start"]:dataset_slices["lat_end"] + 1,
- dataset_slices["lon_start"]:dataset_slices["lon_end"] + 1]
-
- # Build new dataset with subset information
- return ds.Dataset(
- # Slice the lats array with our calculated slice indices
- target_dataset.lats[dataset_slices["lat_start"]:
- dataset_slices["lat_end"] + 1],
- # Slice the lons array with our calculated slice indices
- target_dataset.lons[dataset_slices["lon_start"]:
- dataset_slices["lon_end"] + 1],
- # Slice the times array with our calculated slice indices
- target_dataset.times[dataset_slices["time_start"]:
- dataset_slices["time_end"]+ 1],
- # Slice the values array with our calculated slice indices
- subset_values,
- variable=target_dataset.variable,
- units=target_dataset.units,
- name=subregion_name,
- origin=target_dataset.origin
- )
- elif target_dataset.lats.ndim ==2 and target_dataset.lons.ndim ==2:
- y_index = dataset_slices["y_index"]
- x_index = dataset_slices["x_index"]
- if target_dataset.values.ndim == 2:
- subset_values = target_dataset.values[y_index, x_index]
-
- elif target_dataset.values.ndim == 3:
- nt = dataset_slices["time_end"] - dataset_slices["time_start"] +1
- subset_values = ma.zeros([nt, len(y_index)])
- for it in np.arange(nt):
- subset_values[it,:] = target_dataset.values[dataset_slices["time_start"]+it, y_index, x_index]
- return ds.Dataset(
- # Slice the lats array with our calculated slice indices
- target_dataset.lats[y_index, x_index],
- # Slice the lons array with our calculated slice indices
- target_dataset.lons[y_index, x_index],
- # Slice the times array with our calculated slice indices
- target_dataset.times[dataset_slices["time_start"]:
- dataset_slices["time_end"]+ 1],
- # Slice the values array with our calculated slice indices
- subset_values,
- variable=target_dataset.variable,
- units=target_dataset.units,
- name=subregion_name,
- origin=target_dataset.origin
- )
+ subset_values = target_dataset.values[
+ dataset_slices["time_start"]:dataset_slices["time_end"] + 1,
+ dataset_slices["lat_start"]:dataset_slices["lat_end"] + 1,
+ dataset_slices["lon_start"]:dataset_slices["lon_end"] + 1]
+ # Build new dataset with subset information
+ return ds.Dataset(
+ # Slice the lats array with our calculated slice indices
+ target_dataset.lats[dataset_slices["lat_start"]:
+ dataset_slices["lat_end"] + 1],
+ # Slice the lons array with our calculated slice indices
+ target_dataset.lons[dataset_slices["lon_start"]:
+ dataset_slices["lon_end"] + 1],
+ # Slice the times array with our calculated slice indices
+ target_dataset.times[dataset_slices["time_start"]:
+ dataset_slices["time_end"]+ 1],
+ # Slice the values array with our calculated slice indices
+ subset_values,
+ variable=target_dataset.variable,
+ units=target_dataset.units,
+ name=subregion_name,
+ origin=target_dataset.origin
+ )
+
def safe_subset(subregion, target_dataset, subregion_name=None):
'''Safely subset given dataset with subregion information
@@ -1119,34 +1092,22 @@ def _get_subregion_slice_indices(subregion, target_dataset):
:returns: The indices to slice the Datasets arrays as a Dictionary.
'''
+ latStart = min(np.nonzero(target_dataset.lats >= subregion.lat_min)[0])
+ latEnd = max(np.nonzero(target_dataset.lats <= subregion.lat_max)[0])
+
+ lonStart = min(np.nonzero(target_dataset.lons >= subregion.lon_min)[0])
+ lonEnd = max(np.nonzero(target_dataset.lons <= subregion.lon_max)[0])
+
+
timeStart = min(np.nonzero(target_dataset.times >= subregion.start)[0])
timeEnd = max(np.nonzero(target_dataset.times <= subregion.end)[0])
- if target_dataset.lats.ndim ==1 and target_dataset.lons.ndim ==1:
- latStart = min(np.nonzero(target_dataset.lats >= subregion.lat_min)[0])
- latEnd = max(np.nonzero(target_dataset.lats <= subregion.lat_max)[0])
-
- lonStart = min(np.nonzero(target_dataset.lons >= subregion.lon_min)[0])
- lonEnd = max(np.nonzero(target_dataset.lons <= subregion.lon_max)[0])
-
-
- return {
- "lat_start" : latStart,
- "lat_end" : latEnd,
- "lon_start" : lonStart,
- "lon_end" : lonEnd,
- "time_start" : timeStart,
- "time_end" : timeEnd
- }
- elif target_dataset.lats.ndim ==2 and target_dataset.lons.ndim ==2:
- y_index, x_index = np.where((target_dataset.lats >= subregion.lat_min) &
- (target_dataset.lats <= subregion.lat_max) &
- (target_dataset.lons >= subregion.lon_min) &
- (target_dataset.lons <= subregion.lon_max))
- return {
- "y_index" : y_index,
- "x_index" : x_index,
- "time_start" : timeStart,
- "time_end" : timeEnd
- }
+ return {
+ "lat_start" : latStart,
+ "lat_end" : latEnd,
+ "lon_start" : lonStart,
+ "lon_end" : lonEnd,
+ "time_start" : timeStart,
+ "time_end" : timeEnd
+ }
[3/3] climate git commit: CLIMATE-672 - Update the input validation
function for OCW datasets
Posted by hu...@apache.org.
CLIMATE-672 - Update the input validation function for OCW datasets
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/0f3e8860
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/0f3e8860
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/0f3e8860
Branch: refs/heads/master
Commit: 0f3e8860e0bc44892c647dfbfeb54e2b5a11826f
Parents: 4ad37eb c596a5d
Author: huikyole <hu...@argo.jpl.nasa.gov>
Authored: Fri Sep 25 08:55:02 2015 -0700
Committer: huikyole <hu...@argo.jpl.nasa.gov>
Committed: Fri Sep 25 08:55:02 2015 -0700
----------------------------------------------------------------------
ocw/dataset.py | 9 +--
ocw/dataset_processor.py | 147 ++++++++++++++++--------------------------
2 files changed, 59 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/0f3e8860/ocw/dataset.py
----------------------------------------------------------------------