You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by le...@apache.org on 2016/10/21 02:46:41 UTC
climate git commit: CLIMATE-875 Upgrade to Podaacpy 1.4.0
Repository: climate
Updated Branches:
refs/heads/master 8cf223f2c -> 088706d82
CLIMATE-875 Upgrade to Podaacpy 1.4.0
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/088706d8
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/088706d8
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/088706d8
Branch: refs/heads/master
Commit: 088706d8209204869fbaf24d80c7cc7b7b9aeda3
Parents: 8cf223f
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Thu Oct 20 19:08:23 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Oct 20 19:08:23 2016 -0700
----------------------------------------------------------------------
examples/podaac_integration_example.py | 4 ++--
ocw/data_source/podaac_datasource.py | 36 ++++++++++++++++++++++++++---
ocw/dataset_loader.py | 2 +-
ocw/tests/subset.json | 12 ++++++++++
ocw/tests/test_podaac.py | 12 +++++++++-
5 files changed, 59 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/examples/podaac_integration_example.py
----------------------------------------------------------------------
diff --git a/examples/podaac_integration_example.py b/examples/podaac_integration_example.py
index da90373..a06fc94 100644
--- a/examples/podaac_integration_example.py
+++ b/examples/podaac_integration_example.py
@@ -25,8 +25,8 @@ variable = 'uwnd'
name = 'PO.DAAC_test_dataset'
OUTPUT_PLOT = "ccmp_temporal_std"
""" Step 1: Load Local NetCDF Files into OCW Dataset Objects """
-print("Loading %s dataset into an OCW dataset objects" % datasetShortName)
-ccmp_dataset = podaac.load_dataset(
+print("Loading %s dataset short name into a OCW dataset object." % datasetId)
+ccmp_dataset = podaac.load_level4_granule(
variable=variable, datasetId=datasetId, name=name)
print("CCMP_Dataset.values shape: (times, lats, lons) - %s \n" %
(ccmp_dataset.values.shape,))
http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/data_source/podaac_datasource.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/podaac_datasource.py b/ocw/data_source/podaac_datasource.py
index 4a08046..6feb02b 100644
--- a/ocw/data_source/podaac_datasource.py
+++ b/ocw/data_source/podaac_datasource.py
@@ -16,6 +16,7 @@
# under the License.
from podaac.podaac import Podaac
+from podaac.podaac_utils import PodaacUtils
import numpy as np
from ocw.dataset import Dataset
from netCDF4 import Dataset as netcdf_dataset
@@ -42,9 +43,38 @@ def convert_times_to_datetime(time):
return [parsed_time.num2date(x) for x in time[:]]
-def load_dataset(variable, datasetId='', name=''):
- '''Loads a Dataset from PODAAC
+def list_available_extract_granule_dataset_ids():
+ '''Convenience function which returns an up-to-date \
+ list of available granule dataset id's which can be \
+ used in the granule extraction service.
+ :returns: a comma-seperated list of granule dataset id's.
+ '''
+ podaac_utils = PodaacUtils()
+ return podaac_utils.list_available_extract_granule_dataset_ids()
+
+def subset_granule(input_file_path=''):
+ '''Subset Granule service allows users to Submit subset jobs. \
+ Use of this service should be preceded by a Granule Search in \
+ order to identify and generate a list of granules to be subsetted.
+
+ :param input_file_path: path to a json file which contains the \
+ the request that you want to send to PO.DAAC
+ :type input_file_path: :mod:`string`
+
+ :returns: a token on successful request reception. This can be \
+ further used to check the status of the request.
+
+ '''
+ podaac = Podaac()
+ status = podaac.subset_status(podaac.granule_subset(input_file_path))
+ print("Granule subsetting initiated with request tracking token '%s'." % status)
+ while status is not "done":
+ print('...')
+ return status
+
+def load_level4_granule(variable, datasetId='', name=''):
+ '''Loads a Level4 gridded Dataset from PODAAC
:param variable: The name of the variable to read from the dataset.
:type variable: :mod:`string`
@@ -80,7 +110,7 @@ def load_dataset(variable, datasetId='', name=''):
# but conventions aren't always followed and all dimensions aren't always present so
# see if we can make some educated deductions before defaulting to just pulling the first three
# columns.
- temp_dimensions = map(lambda x: x.lower(), dataset.dimensions)
+ temp_dimensions = list(map(lambda x: x.lower(), dataset.dimensions))
dataset_dimensions = dataset.dimensions
time = dataset_dimensions[temp_dimensions.index(
'time') if 'time' in temp_dimensions else 0]
http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/dataset_loader.py
----------------------------------------------------------------------
diff --git a/ocw/dataset_loader.py b/ocw/dataset_loader.py
index fdcbb1b..5a7f6ea 100644
--- a/ocw/dataset_loader.py
+++ b/ocw/dataset_loader.py
@@ -94,7 +94,7 @@ class DatasetLoader:
'local': local.load_multiple_files,
'local_split': local.load_dataset_from_multiple_netcdf_files,
'rcmed': rcmed.parameter_dataset,
- 'podaac': podaac.load_dataset
+ 'podaac': podaac.load_level4_granule
}
# Exclude esgf and dap for python 3 until they are compatible
http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/tests/subset.json
----------------------------------------------------------------------
diff --git a/ocw/tests/subset.json b/ocw/tests/subset.json
new file mode 100644
index 0000000..c00891a
--- /dev/null
+++ b/ocw/tests/subset.json
@@ -0,0 +1,12 @@
+{
+ "email":"yournamem@emailaddress.com",
+ "query":[
+ {
+ "datasetId":"PODAAC-ASOP2-25X01",
+ "granuleIds":["ascat_20160409_113000_metopa_49153_eps_o_250_2401_ovw.l2.nc"],
+ "bbox":"-180,-90,180,90",
+ "variables":["wvc_index","model_speed","model_dir","ice_prob","ice_age","wvc_quality_flag","wind_speed","wind_dir","bs_distance","lat","lon","time"],
+ "compact":false
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/tests/test_podaac.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_podaac.py b/ocw/tests/test_podaac.py
index 66afcb8..89bc143 100644
--- a/ocw/tests/test_podaac.py
+++ b/ocw/tests/test_podaac.py
@@ -32,8 +32,13 @@ class TestPodaacDataSource(unittest.TestCase):
cls.name = 'PO.DAAC_test_dataset'
cls.file_path = os.path.dirname(os.path.abspath(__file__))
cls.format = '.nc'
- cls.dataset = podaac.load_dataset(
+ cls.dataset = podaac.load_level4_granule(
cls.variable, cls.datasetId, cls.name)
+ #Until we can retrieve the subset data download link programmatically,
+ #we will need to skip this test. More information can be see at
+ #https://podaac.jpl.nasa.gov/forum/viewtopic.php?f=53&t=424&p=790
+ #cls.json = 'subset.json'
+ #cls.granule_subset = podaac.subset_granule(cls.json)
def test_is_dataset(self):
self.assertTrue(isinstance(self.dataset, Dataset))
@@ -61,6 +66,11 @@ class TestPodaacDataSource(unittest.TestCase):
def test_custom_name(self):
self.assertEquals(self.dataset.name, self.name)
+ def test_granule_subset(self):
+ #Until we can retrieve the subset data download link programmatically,
+ #we will need to skip this test. More information can be see at
+ #https://podaac.jpl.nasa.gov/forum/viewtopic.php?f=53&t=424&p=790
+ pass
if __name__ == '__main__':
unittest.main()