You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by jo...@apache.org on 2014/10/27 20:19:51 UTC
[01/11] git commit: Update gitignore so setup.py develop artifacts
are ignored
Repository: climate
Updated Branches:
refs/heads/master 335512307 -> 15a93c833
Update gitignore so setup.py develop artifacts are ignored
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/22192f5e
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/22192f5e
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/22192f5e
Branch: refs/heads/master
Commit: 22192f5effb56aa36d7613d70c43992b799d5dcc
Parents: 26b6c91
Author: Michael Joyce <jo...@apache.org>
Authored: Mon Sep 15 07:24:43 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Sun Oct 19 14:16:54 2014 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/22192f5e/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index caa89c7..a5062d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,4 +10,4 @@ examples/*.png
build/
docs/build/
-
+Apache_Open_Climate_Workbench.egg-info/
[02/11] git commit: CLIMATE-519 - Rework ESGF integration module
structuring
Posted by jo...@apache.org.
CLIMATE-519 - Rework ESGF integration module structuring
The ESGF module is now part of the OCW package.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/7d3c237c
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/7d3c237c
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/7d3c237c
Branch: refs/heads/master
Commit: 7d3c237c57305629e588e84bd2f5c46e6dc87edd
Parents: 22192f5
Author: Michael Joyce <jo...@apache.org>
Authored: Mon Sep 15 08:01:22 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Sun Oct 19 14:17:08 2014 -0700
----------------------------------------------------------------------
esgf/README | 17 -----
esgf/src/esgf/__init__.py | 18 ------
esgf/src/esgf/rcmes/__init__.py | 18 ------
esgf/src/esgf/rcmes/constants.py | 35 ----------
esgf/src/esgf/rcmes/download.py | 67 --------------------
esgf/src/esgf/rcmes/logon.py | 44 -------------
esgf/src/esgf/rcmes/logon2.py | 43 -------------
esgf/src/esgf/rcmes/main.py | 116 ----------------------------------
esgf/src/esgf/rcmes/search.py | 89 --------------------------
ocw/esgf/__init__.py | 18 ++++++
ocw/esgf/constants.py | 35 ++++++++++
ocw/esgf/download.py | 67 ++++++++++++++++++++
ocw/esgf/logon.py | 44 +++++++++++++
ocw/esgf/logon2.py | 43 +++++++++++++
ocw/esgf/main.py | 116 ++++++++++++++++++++++++++++++++++
ocw/esgf/search.py | 89 ++++++++++++++++++++++++++
16 files changed, 412 insertions(+), 447 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/README
----------------------------------------------------------------------
diff --git a/esgf/README b/esgf/README
deleted file mode 100644
index 019eb24..0000000
--- a/esgf/README
+++ /dev/null
@@ -1,17 +0,0 @@
-PROTOTYPE PYTHON PACKAGE FOR ESGF-RCMES INTEGRATION
-
-Python dependencies:
- - Python 2.7
- - PyOpenSSL (sudo easy_install PyOpenSSL==0.10 - NOTE: latest PyOpenSSL 0.13 needs latest SSL installed)
- - myproxyclient (sudo easy_install MyProxyClient)
- - esgf-pyclient (sudo easy_install esgf-pyclient)
-
-Pre-requisites:
- - user must be registered with ESGF and be assigned an OpenID to logon
- - user must have been granted membership in group CMIP5 Research to be able to download CMIP5 datasets
-
-Usage:
- - install package under some directory: <INSTALL_DIR>
- - export PYTHONPATH=<INSTALL_DIR>/src
- - edit <INSTALL_DIR>/src/esgf/rcmes/main.py to configure openid, password, output directory
- - python <INSTALL_DIR>/src/esgf/rcmes/main.py
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/__init__.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/__init__.py b/esgf/src/esgf/__init__.py
deleted file mode 100644
index 2872cad..0000000
--- a/esgf/src/esgf/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/rcmes/__init__.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/rcmes/__init__.py b/esgf/src/esgf/rcmes/__init__.py
deleted file mode 100644
index 2872cad..0000000
--- a/esgf/src/esgf/rcmes/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/rcmes/constants.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/rcmes/constants.py b/esgf/src/esgf/rcmes/constants.py
deleted file mode 100644
index a6baf9f..0000000
--- a/esgf/src/esgf/rcmes/constants.py
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-'''Module containing constant parameters for ESGF RCMES integration.'''
-
-# default location of ESGF user credentials
-ESGF_CREDENTIALS = "~/.esg/credentials.pem"
-
-# URL for ESGF certificate service
-#CERT_SERVICE_URL = "https://localhost:8443/esgf-idp/idp/getcert.htm"
-CERT_SERVICE_URL = "https://esg-datanode.jpl.nasa.gov/esgf-idp/idp/getcert.htm"
-
-# Basic authentication realm
-REALM = "ESGF"
-
-# DN of JPL MyProxy server (needs to be explicitely set somtimes)
-JPL_MYPROXY_SERVER_DN = "/O=ESGF/OU=esg-datanode.jpl.nasa.gov/CN=host/esg-vm.jpl.nasa.gov"
-
-# URL of ESGF search service to contact
-JPL_SEARCH_SERVICE_URL = "http://esg-datanode.jpl.nasa.gov/esg-search/search"
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/rcmes/download.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/rcmes/download.py b/esgf/src/esgf/rcmes/download.py
deleted file mode 100644
index 389b982..0000000
--- a/esgf/src/esgf/rcmes/download.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-'''
-RCMES module to download a file from ESGF.
-
-'''
-
-import urllib2, httplib
-from os.path import expanduser, join
-
-from esgf.rcmes.constants import ESGF_CREDENTIALS
-
-class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
- '''
- HTTP handler that transmits an X509 certificate as part of the request
- '''
-
- def __init__(self, key, cert):
- urllib2.HTTPSHandler.__init__(self)
- self.key = key
- self.cert = cert
- def https_open(self, req):
- return self.do_open(self.getConnection, req)
- def getConnection(self, host, timeout=300):
- return httplib.HTTPSConnection(host, key_file=self.key, cert_file=self.cert)
-
-def download(url, toDirectory="/tmp"):
- '''
- Function to download a single file from ESGF.
-
- :param url: the URL of the file to download
- :param toDirectory: target directory where the file will be written
- '''
-
- # setup HTTP handler
- certFile = expanduser(ESGF_CREDENTIALS)
- opener = urllib2.build_opener(HTTPSClientAuthHandler(certFile,certFile))
- opener.add_handler(urllib2.HTTPCookieProcessor())
-
- # download file
- localFilePath = join(toDirectory,url.split('/')[-1])
- print "\nDownloading url: %s to local path: %s ..." % (url, localFilePath)
- localFile=open( localFilePath, 'w')
- webFile=opener.open(url)
- localFile.write(webFile.read())
-
- # cleanup
- localFile.close()
- webFile.close()
- opener.close()
- print "... done"
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/rcmes/logon.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/rcmes/logon.py b/esgf/src/esgf/rcmes/logon.py
deleted file mode 100644
index 0a6f4f7..0000000
--- a/esgf/src/esgf/rcmes/logon.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-'''
-RCMES module to logon onto the ESGF.
-'''
-
-from pyesgf.logon import LogonManager
-import os
-
-from esgf.rcmes.constants import JPL_MYPROXY_SERVER_DN
-
-def logon(openid, password):
- '''
- Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
- The certificate is written in the location ~/.esg/credentials.pem.
- The trusted CA certificates are written in the directory ~/.esg/certificates.
- '''
-
- # Must configure the DN of the JPL MyProxy server if using a JPL openid
- if "esg-datanode.jpl.nasa.gov" in openid:
- os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN
-
- lm = LogonManager()
- lm.logon_with_openid(openid,password)
- return lm.is_logged_on()
-
-
-
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/rcmes/logon2.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/rcmes/logon2.py b/esgf/src/esgf/rcmes/logon2.py
deleted file mode 100644
index 11a808a..0000000
--- a/esgf/src/esgf/rcmes/logon2.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-'''
-RCMES module to logon onto the ESGF by contacting the IdP RESTful service.
-'''
-
-from esgf.rcmes.constants import ESGF_CREDENTIALS, CERT_SERVICE_URL, REALM
-
-import urllib2
-from os.path import expanduser
-
-def logon2(openid, password):
- '''
- Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
- The certificate is written in the location specified by ESGF_CREDENTIALS.
- '''
-
- password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
- password_mgr.add_password(REALM, CERT_SERVICE_URL, openid, password)
- handler = urllib2.HTTPBasicAuthHandler(password_mgr)
- opener = urllib2.build_opener(urllib2.HTTPHandler, handler)
- request = opener.open(CERT_SERVICE_URL)
- #print request.read()
-
- localFilePath = expanduser(ESGF_CREDENTIALS)
- certFile=open(localFilePath, 'w')
- certFile.write(request.read())
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/rcmes/main.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/rcmes/main.py b/esgf/src/esgf/rcmes/main.py
deleted file mode 100644
index 4ffadcc..0000000
--- a/esgf/src/esgf/rcmes/main.py
+++ /dev/null
@@ -1,116 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-'''
-Example main program for ESGF-RCMES integration.
-
-'''
-
-# constant parameters
-USER_OPENID = "https://esg-datanode.jpl.nasa.gov/esgf-idp/openid/lucacinquini"
-USER_PASSWORD = "*****"
-DATA_DIRECTORY = "/tmp"
-
-from esgf.rcmes.logon import logon
-from esgf.rcmes.logon2 import logon2
-from esgf.rcmes.search import SearchClient
-from esgf.rcmes.download import download
-
-def main():
- '''Example driver program'''
-
- # step 1: obtain short-term certificate
- print 'Retrieving ESGF certificate...'
- # logon using client-side MyProxy libraries
- #if logon(USER_OPENID, USER_PASSWORD):
- # print "...done."
- # logon through server-side MyProxy service
- if logon2(USER_OPENID, USER_PASSWORD):
- print "...done"
-
- # step 2: execute faceted search for files
- urls = main_obs4mips()
- #urls = main_cmip5()
-
- # step 3: download file(s)
- for i, url in enumerate(urls):
- if i>=1:
- break
- download(url, toDirectory=DATA_DIRECTORY)
-
-
-def main_cmip5():
- '''
- Example workflow to search for CMIP5 files
- '''
-
- searchClient = SearchClient(searchServiceUrl="http://pcmdi9.llnl.gov/esg-search/search", distrib=False)
-
- print '\nAvailable projects=%s' % searchClient.getFacets('project')
- searchClient.setConstraint(project='CMIP5')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- print '\nAvailable models=%s' % searchClient.getFacets('model')
- searchClient.setConstraint(model='INM-CM4')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- print '\nAvailable experiments=%s' % searchClient.getFacets('experiment')
- searchClient.setConstraint(experiment='historical')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- print '\nAvailable time frequencies=%s' % searchClient.getFacets('time_frequency')
- searchClient.setConstraint(time_frequency='mon')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- print '\nAvailable CF standard names=%s' % searchClient.getFacets('cf_standard_name')
- searchClient.setConstraint(cf_standard_name='air_temperature')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- urls = searchClient.getFiles()
- return urls
-
-
-def main_obs4mips():
- '''
- Example workflow to search for obs4MIPs files.
- '''
-
- searchClient = SearchClient(distrib=False)
-
- # obs4MIPs
- print '\nAvailable projects=%s' % searchClient.getFacets('project')
- searchClient.setConstraint(project='obs4MIPs')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- print '\nAvailable variables=%s' % searchClient.getFacets('variable')
- searchClient.setConstraint(variable='hus')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- print '\nAvailable time frequencies=%s' % searchClient.getFacets('time_frequency')
- searchClient.setConstraint(time_frequency='mon')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- print '\nAvailable models=%s' % searchClient.getFacets('model')
- searchClient.setConstraint(model='Obs-MLS')
- print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
-
- urls = searchClient.getFiles()
- return urls
-
-if __name__ == '__main__':
- main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/esgf/src/esgf/rcmes/search.py
----------------------------------------------------------------------
diff --git a/esgf/src/esgf/rcmes/search.py b/esgf/src/esgf/rcmes/search.py
deleted file mode 100644
index 3a81f24..0000000
--- a/esgf/src/esgf/rcmes/search.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-'''
-RCMES module to execute a faceted search for ESGF files.
-
-'''
-
-from pyesgf.search import SearchConnection
-
-from esgf.rcmes.constants import JPL_SEARCH_SERVICE_URL
-
-class SearchClient():
- """
- Simple ESGF search client for RCMES.
- This class is a thin layer on top of the esgfpy-client package.
- Note: this class always searches for latest versions, no replicas.
- """
-
- def __init__(self, searchServiceUrl=JPL_SEARCH_SERVICE_URL, distrib=True):
- """
- :param searchServiceUrl: URL of ESGF search service to query
- :param distrib: True to execute a federation-wide search,
- False to search only the specified search service
- """
- connection = SearchConnection(searchServiceUrl, distrib=distrib)
-
- # dictionary of query constraints
- self.constraints = { "latest":True, "replica":False, "distrib":distrib }
-
- # initial search context
- self.context = connection.new_context( **self.constraints )
-
-
- def setConstraint(self, **constraints):
- """
- Sets one or more facet constraints.
- :param constraints: dictionary of (facet name, facet value) constraints.
- """
- for key in constraints:
- print 'Setting constraint: %s=%s' % (key, constraints[key])
- self.constraints[key] = constraints[key]
- self.context = self.context.constrain(**constraints)
-
- def getNumberOfDatasets(self):
- """
- :return: the number of datasets matching the current constraints.
- """
- return self.context.hit_count
-
- def getFacets(self, facet):
- """
- :return: a dictionary of (facet value, facet count) for the specified facet and current constraints.
- Example (for facet='project'): {u'COUND': 4, u'CMIP5': 2657, u'obs4MIPs': 7}
- """
- return self.context.facet_counts[facet]
-
- def getFiles(self):
- """
- Executes a search for files with the current constraints.
- :return: list of file download URLs.
- """
- datasets = self.context.search()
- urls = []
- for dataset in datasets:
- print "\nSearching files for dataset=%s with constraints: %s" % (dataset.dataset_id, self.constraints)
- files = dataset.file_context().search(**self.constraints)
- for file in files:
- print 'Found file=%s' % file.download_url
- urls.append(file.download_url)
- return urls
-
-
-
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/ocw/esgf/__init__.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/__init__.py b/ocw/esgf/__init__.py
new file mode 100644
index 0000000..2872cad
--- /dev/null
+++ b/ocw/esgf/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/ocw/esgf/constants.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/constants.py b/ocw/esgf/constants.py
new file mode 100644
index 0000000..a6baf9f
--- /dev/null
+++ b/ocw/esgf/constants.py
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+'''Module containing constant parameters for ESGF RCMES integration.'''
+
+# default location of ESGF user credentials
+ESGF_CREDENTIALS = "~/.esg/credentials.pem"
+
+# URL for ESGF certificate service
+#CERT_SERVICE_URL = "https://localhost:8443/esgf-idp/idp/getcert.htm"
+CERT_SERVICE_URL = "https://esg-datanode.jpl.nasa.gov/esgf-idp/idp/getcert.htm"
+
+# Basic authentication realm
+REALM = "ESGF"
+
+# DN of JPL MyProxy server (needs to be explicitely set somtimes)
+JPL_MYPROXY_SERVER_DN = "/O=ESGF/OU=esg-datanode.jpl.nasa.gov/CN=host/esg-vm.jpl.nasa.gov"
+
+# URL of ESGF search service to contact
+JPL_SEARCH_SERVICE_URL = "http://esg-datanode.jpl.nasa.gov/esg-search/search"
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/ocw/esgf/download.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/download.py b/ocw/esgf/download.py
new file mode 100644
index 0000000..389b982
--- /dev/null
+++ b/ocw/esgf/download.py
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+'''
+RCMES module to download a file from ESGF.
+
+'''
+
+import urllib2, httplib
+from os.path import expanduser, join
+
+from esgf.rcmes.constants import ESGF_CREDENTIALS
+
+class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
+ '''
+ HTTP handler that transmits an X509 certificate as part of the request
+ '''
+
+ def __init__(self, key, cert):
+ urllib2.HTTPSHandler.__init__(self)
+ self.key = key
+ self.cert = cert
+ def https_open(self, req):
+ return self.do_open(self.getConnection, req)
+ def getConnection(self, host, timeout=300):
+ return httplib.HTTPSConnection(host, key_file=self.key, cert_file=self.cert)
+
+def download(url, toDirectory="/tmp"):
+ '''
+ Function to download a single file from ESGF.
+
+ :param url: the URL of the file to download
+ :param toDirectory: target directory where the file will be written
+ '''
+
+ # setup HTTP handler
+ certFile = expanduser(ESGF_CREDENTIALS)
+ opener = urllib2.build_opener(HTTPSClientAuthHandler(certFile,certFile))
+ opener.add_handler(urllib2.HTTPCookieProcessor())
+
+ # download file
+ localFilePath = join(toDirectory,url.split('/')[-1])
+ print "\nDownloading url: %s to local path: %s ..." % (url, localFilePath)
+ localFile=open( localFilePath, 'w')
+ webFile=opener.open(url)
+ localFile.write(webFile.read())
+
+ # cleanup
+ localFile.close()
+ webFile.close()
+ opener.close()
+ print "... done"
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/ocw/esgf/logon.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/logon.py b/ocw/esgf/logon.py
new file mode 100644
index 0000000..0a6f4f7
--- /dev/null
+++ b/ocw/esgf/logon.py
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+'''
+RCMES module to logon onto the ESGF.
+'''
+
+from pyesgf.logon import LogonManager
+import os
+
+from esgf.rcmes.constants import JPL_MYPROXY_SERVER_DN
+
+def logon(openid, password):
+ '''
+ Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
+ The certificate is written in the location ~/.esg/credentials.pem.
+ The trusted CA certificates are written in the directory ~/.esg/certificates.
+ '''
+
+ # Must configure the DN of the JPL MyProxy server if using a JPL openid
+ if "esg-datanode.jpl.nasa.gov" in openid:
+ os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN
+
+ lm = LogonManager()
+ lm.logon_with_openid(openid,password)
+ return lm.is_logged_on()
+
+
+
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/ocw/esgf/logon2.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/logon2.py b/ocw/esgf/logon2.py
new file mode 100644
index 0000000..11a808a
--- /dev/null
+++ b/ocw/esgf/logon2.py
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+'''
+RCMES module to logon onto the ESGF by contacting the IdP RESTful service.
+'''
+
+from esgf.rcmes.constants import ESGF_CREDENTIALS, CERT_SERVICE_URL, REALM
+
+import urllib2
+from os.path import expanduser
+
+def logon2(openid, password):
+ '''
+ Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
+ The certificate is written in the location specified by ESGF_CREDENTIALS.
+ '''
+
+ password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ password_mgr.add_password(REALM, CERT_SERVICE_URL, openid, password)
+ handler = urllib2.HTTPBasicAuthHandler(password_mgr)
+ opener = urllib2.build_opener(urllib2.HTTPHandler, handler)
+ request = opener.open(CERT_SERVICE_URL)
+ #print request.read()
+
+ localFilePath = expanduser(ESGF_CREDENTIALS)
+ certFile=open(localFilePath, 'w')
+ certFile.write(request.read())
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/ocw/esgf/main.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/main.py b/ocw/esgf/main.py
new file mode 100644
index 0000000..4ffadcc
--- /dev/null
+++ b/ocw/esgf/main.py
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+'''
+Example main program for ESGF-RCMES integration.
+
+'''
+
+# constant parameters
+USER_OPENID = "https://esg-datanode.jpl.nasa.gov/esgf-idp/openid/lucacinquini"
+USER_PASSWORD = "*****"
+DATA_DIRECTORY = "/tmp"
+
+from esgf.rcmes.logon import logon
+from esgf.rcmes.logon2 import logon2
+from esgf.rcmes.search import SearchClient
+from esgf.rcmes.download import download
+
+def main():
+ '''Example driver program'''
+
+ # step 1: obtain short-term certificate
+ print 'Retrieving ESGF certificate...'
+ # logon using client-side MyProxy libraries
+ #if logon(USER_OPENID, USER_PASSWORD):
+ # print "...done."
+ # logon through server-side MyProxy service
+ if logon2(USER_OPENID, USER_PASSWORD):
+ print "...done"
+
+ # step 2: execute faceted search for files
+ urls = main_obs4mips()
+ #urls = main_cmip5()
+
+ # step 3: download file(s)
+ for i, url in enumerate(urls):
+ if i>=1:
+ break
+ download(url, toDirectory=DATA_DIRECTORY)
+
+
+def main_cmip5():
+ '''
+ Example workflow to search for CMIP5 files
+ '''
+
+ searchClient = SearchClient(searchServiceUrl="http://pcmdi9.llnl.gov/esg-search/search", distrib=False)
+
+ print '\nAvailable projects=%s' % searchClient.getFacets('project')
+ searchClient.setConstraint(project='CMIP5')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ print '\nAvailable models=%s' % searchClient.getFacets('model')
+ searchClient.setConstraint(model='INM-CM4')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ print '\nAvailable experiments=%s' % searchClient.getFacets('experiment')
+ searchClient.setConstraint(experiment='historical')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ print '\nAvailable time frequencies=%s' % searchClient.getFacets('time_frequency')
+ searchClient.setConstraint(time_frequency='mon')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ print '\nAvailable CF standard names=%s' % searchClient.getFacets('cf_standard_name')
+ searchClient.setConstraint(cf_standard_name='air_temperature')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ urls = searchClient.getFiles()
+ return urls
+
+
+def main_obs4mips():
+ '''
+ Example workflow to search for obs4MIPs files.
+ '''
+
+ searchClient = SearchClient(distrib=False)
+
+ # obs4MIPs
+ print '\nAvailable projects=%s' % searchClient.getFacets('project')
+ searchClient.setConstraint(project='obs4MIPs')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ print '\nAvailable variables=%s' % searchClient.getFacets('variable')
+ searchClient.setConstraint(variable='hus')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ print '\nAvailable time frequencies=%s' % searchClient.getFacets('time_frequency')
+ searchClient.setConstraint(time_frequency='mon')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ print '\nAvailable models=%s' % searchClient.getFacets('model')
+ searchClient.setConstraint(model='Obs-MLS')
+ print "Number of Datasets=%d" % searchClient.getNumberOfDatasets()
+
+ urls = searchClient.getFiles()
+ return urls
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/7d3c237c/ocw/esgf/search.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/search.py b/ocw/esgf/search.py
new file mode 100644
index 0000000..3a81f24
--- /dev/null
+++ b/ocw/esgf/search.py
@@ -0,0 +1,89 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+'''
+RCMES module to execute a faceted search for ESGF files.
+
+'''
+
+from pyesgf.search import SearchConnection
+
+from esgf.rcmes.constants import JPL_SEARCH_SERVICE_URL
+
+class SearchClient():
+ """
+ Simple ESGF search client for RCMES.
+ This class is a thin layer on top of the esgfpy-client package.
+ Note: this class always searches for latest versions, no replicas.
+ """
+
+ def __init__(self, searchServiceUrl=JPL_SEARCH_SERVICE_URL, distrib=True):
+ """
+ :param searchServiceUrl: URL of ESGF search service to query
+ :param distrib: True to execute a federation-wide search,
+ False to search only the specified search service
+ """
+ connection = SearchConnection(searchServiceUrl, distrib=distrib)
+
+ # dictionary of query constraints
+ self.constraints = { "latest":True, "replica":False, "distrib":distrib }
+
+ # initial search context
+ self.context = connection.new_context( **self.constraints )
+
+
+ def setConstraint(self, **constraints):
+ """
+ Sets one or more facet constraints.
+ :param constraints: dictionary of (facet name, facet value) constraints.
+ """
+ for key in constraints:
+ print 'Setting constraint: %s=%s' % (key, constraints[key])
+ self.constraints[key] = constraints[key]
+ self.context = self.context.constrain(**constraints)
+
+ def getNumberOfDatasets(self):
+ """
+ :return: the number of datasets matching the current constraints.
+ """
+ return self.context.hit_count
+
+ def getFacets(self, facet):
+ """
+ :return: a dictionary of (facet value, facet count) for the specified facet and current constraints.
+ Example (for facet='project'): {u'COUND': 4, u'CMIP5': 2657, u'obs4MIPs': 7}
+ """
+ return self.context.facet_counts[facet]
+
+ def getFiles(self):
+ """
+ Executes a search for files with the current constraints.
+ :return: list of file download URLs.
+ """
+ datasets = self.context.search()
+ urls = []
+ for dataset in datasets:
+ print "\nSearching files for dataset=%s with constraints: %s" % (dataset.dataset_id, self.constraints)
+ files = dataset.file_context().search(**self.constraints)
+ for file in files:
+ print 'Found file=%s' % file.download_url
+ urls.append(file.download_url)
+ return urls
+
+
+
[03/11] git commit: CLIMATE-519 - Rework ESGF helper module import
structuring
Posted by jo...@apache.org.
CLIMATE-519 - Rework ESGF helper module import structuring
Update ESGF module imports to reflect new package/module structuring.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/79464c74
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/79464c74
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/79464c74
Branch: refs/heads/master
Commit: 79464c74823c7234eb44c6af1b4d8bf29a8e4cd9
Parents: 7d3c237
Author: Michael Joyce <jo...@apache.org>
Authored: Mon Sep 15 08:06:04 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Sun Oct 19 14:18:05 2014 -0700
----------------------------------------------------------------------
ocw/esgf/download.py | 2 +-
ocw/esgf/logon.py | 2 +-
ocw/esgf/logon2.py | 4 ++--
ocw/esgf/main.py | 10 +++++-----
ocw/esgf/search.py | 2 +-
5 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/79464c74/ocw/esgf/download.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/download.py b/ocw/esgf/download.py
index 389b982..d643de8 100644
--- a/ocw/esgf/download.py
+++ b/ocw/esgf/download.py
@@ -24,7 +24,7 @@ RCMES module to download a file from ESGF.
import urllib2, httplib
from os.path import expanduser, join
-from esgf.rcmes.constants import ESGF_CREDENTIALS
+from ocw.esgf.constants import ESGF_CREDENTIALS
class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
'''
http://git-wip-us.apache.org/repos/asf/climate/blob/79464c74/ocw/esgf/logon.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/logon.py b/ocw/esgf/logon.py
index 0a6f4f7..c7f3478 100644
--- a/ocw/esgf/logon.py
+++ b/ocw/esgf/logon.py
@@ -23,7 +23,7 @@ RCMES module to logon onto the ESGF.
from pyesgf.logon import LogonManager
import os
-from esgf.rcmes.constants import JPL_MYPROXY_SERVER_DN
+from ocw.esgf.constants import JPL_MYPROXY_SERVER_DN
def logon(openid, password):
'''
http://git-wip-us.apache.org/repos/asf/climate/blob/79464c74/ocw/esgf/logon2.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/logon2.py b/ocw/esgf/logon2.py
index 11a808a..46724f7 100644
--- a/ocw/esgf/logon2.py
+++ b/ocw/esgf/logon2.py
@@ -20,7 +20,7 @@
RCMES module to logon onto the ESGF by contacting the IdP RESTful service.
'''
-from esgf.rcmes.constants import ESGF_CREDENTIALS, CERT_SERVICE_URL, REALM
+from ocw.esgf.constants import ESGF_CREDENTIALS, CERT_SERVICE_URL, REALM
import urllib2
from os.path import expanduser
@@ -40,4 +40,4 @@ def logon2(openid, password):
localFilePath = expanduser(ESGF_CREDENTIALS)
certFile=open(localFilePath, 'w')
- certFile.write(request.read())
\ No newline at end of file
+ certFile.write(request.read())
http://git-wip-us.apache.org/repos/asf/climate/blob/79464c74/ocw/esgf/main.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/main.py b/ocw/esgf/main.py
index 4ffadcc..65c2a1c 100644
--- a/ocw/esgf/main.py
+++ b/ocw/esgf/main.py
@@ -26,10 +26,10 @@ USER_OPENID = "https://esg-datanode.jpl.nasa.gov/esgf-idp/openid/lucacinquini"
USER_PASSWORD = "*****"
DATA_DIRECTORY = "/tmp"
-from esgf.rcmes.logon import logon
-from esgf.rcmes.logon2 import logon2
-from esgf.rcmes.search import SearchClient
-from esgf.rcmes.download import download
+from ocw.esgf.logon import logon
+from ocw.esgf.logon2 import logon2
+from ocw.esgf.search import SearchClient
+from ocw.esgf.download import download
def main():
'''Example driver program'''
@@ -113,4 +113,4 @@ def main_obs4mips():
return urls
if __name__ == '__main__':
- main()
\ No newline at end of file
+ main()
http://git-wip-us.apache.org/repos/asf/climate/blob/79464c74/ocw/esgf/search.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/search.py b/ocw/esgf/search.py
index 3a81f24..a8790b3 100644
--- a/ocw/esgf/search.py
+++ b/ocw/esgf/search.py
@@ -23,7 +23,7 @@ RCMES module to execute a faceted search for ESGF files.
from pyesgf.search import SearchConnection
-from esgf.rcmes.constants import JPL_SEARCH_SERVICE_URL
+from ocw.esgf.constants import JPL_SEARCH_SERVICE_URL
class SearchClient():
"""
[11/11] git commit: Resolve CLIMATE-518. Merge PR #115.
Posted by jo...@apache.org.
Resolve CLIMATE-518. Merge PR #115.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/15a93c83
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/15a93c83
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/15a93c83
Branch: refs/heads/master
Commit: 15a93c833af5f828899d33299a679632e6fb8036
Parents: 3355123 28d9293
Author: Michael Joyce <jo...@apache.org>
Authored: Mon Oct 27 12:19:03 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Mon Oct 27 12:19:03 2014 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
docs/source/data_source/data_sources.rst | 5 ++
esgf/README | 17 ----
esgf/src/esgf/__init__.py | 18 ----
esgf/src/esgf/rcmes/__init__.py | 18 ----
esgf/src/esgf/rcmes/constants.py | 35 --------
esgf/src/esgf/rcmes/download.py | 67 ---------------
esgf/src/esgf/rcmes/logon.py | 44 ----------
esgf/src/esgf/rcmes/logon2.py | 43 ----------
esgf/src/esgf/rcmes/main.py | 116 --------------------------
esgf/src/esgf/rcmes/search.py | 89 --------------------
ocw/data_source/esgf.py | 111 ++++++++++++++++++++++++
ocw/esgf/__init__.py | 18 ++++
ocw/esgf/constants.py | 36 ++++++++
ocw/esgf/download.py | 67 +++++++++++++++
ocw/esgf/logon.py | 44 ++++++++++
ocw/esgf/logon2.py | 43 ++++++++++
ocw/esgf/main.py | 116 ++++++++++++++++++++++++++
ocw/esgf/search.py | 89 ++++++++++++++++++++
setup.py | 4 +-
20 files changed, 532 insertions(+), 450 deletions(-)
----------------------------------------------------------------------
[10/11] git commit: CLIMATE-518 - Documentation Clarification
Posted by jo...@apache.org.
CLIMATE-518 - Documentation Clarification
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/28d9293c
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/28d9293c
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/28d9293c
Branch: refs/heads/master
Commit: 28d9293cf42de66b86a3228564a6dece2efbc4b5
Parents: 08b780d
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Oct 24 08:21:00 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Oct 24 08:21:00 2014 -0700
----------------------------------------------------------------------
ocw/data_source/esgf.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/28d9293c/ocw/data_source/esgf.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/esgf.py b/ocw/data_source/esgf.py
index f138367..084a73d 100644
--- a/ocw/data_source/esgf.py
+++ b/ocw/data_source/esgf.py
@@ -54,7 +54,9 @@ def load_dataset(dataset_id,
pass as constraints to the search wrapper. These can be anything found
on the ESGF metadata page for a dataset.
- :returns: An ocw.dataset.Dataset object contained the requested dataset.
+ :returns: A list of ocw.dataset.Dataset objects contained the requested
+ dataset. If the dataset is stored in multiple files each will be loaded
+ into a separate ocw.dataset.Dataset object.
:raises ValueError: If no dataset can be found for the supplied ID and
variable, or if the requested dataset is a multi-file dataset.
[08/11] git commit: CLIMATE-518 - Add descriptive default search URL
constant
Posted by jo...@apache.org.
CLIMATE-518 - Add descriptive default search URL constant
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/1094b18a
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/1094b18a
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/1094b18a
Branch: refs/heads/master
Commit: 1094b18a9259223350c15ebe5892b39289e53286
Parents: 2cc0401
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Oct 24 08:02:33 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Oct 24 08:02:33 2014 -0700
----------------------------------------------------------------------
ocw/esgf/constants.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/1094b18a/ocw/esgf/constants.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/constants.py b/ocw/esgf/constants.py
index a6baf9f..860a489 100644
--- a/ocw/esgf/constants.py
+++ b/ocw/esgf/constants.py
@@ -33,3 +33,4 @@ JPL_MYPROXY_SERVER_DN = "/O=ESGF/OU=esg-datanode.jpl.nasa.gov/CN=host/esg-vm.jpl
# URL of ESGF search service to contact
JPL_SEARCH_SERVICE_URL = "http://esg-datanode.jpl.nasa.gov/esg-search/search"
+DEFAULT_ESGF_SEARCH = "http://esg-datanode.jpl.nasa.gov/esg-search/search"
[06/11] git commit: CLIMATE-518 - Add documentation to ESGF
data_source
Posted by jo...@apache.org.
CLIMATE-518 - Add documentation to ESGF data_source
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/f625cf82
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/f625cf82
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/f625cf82
Branch: refs/heads/master
Commit: f625cf82e4f0165bbbcfe484221e3a565b88f2ce
Parents: 51a019c
Author: Michael Joyce <jo...@apache.org>
Authored: Thu Oct 9 09:15:38 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Sun Oct 19 14:20:21 2014 -0700
----------------------------------------------------------------------
ocw/data_source/esgf.py | 31 ++++++++++++++++++++++++++++---
1 file changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/f625cf82/ocw/data_source/esgf.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/esgf.py b/ocw/data_source/esgf.py
index 7f5b6a6..a072b21 100644
--- a/ocw/data_source/esgf.py
+++ b/ocw/data_source/esgf.py
@@ -31,7 +31,32 @@ def load_dataset(dataset_id,
esgf_password,
search_url=JPL_SEARCH_SERVICE_URL,
**additional_constraints):
- ''''''
+ ''' Load an ESGF dataset.
+
+ .. note:
+ Currently, multi-file datasets aren't supported. This functionality
+ will be added soon!
+
+ :param dataset_id: The ESGF ID of the dataset to load.
+ :type dataset_id: String
+ :param variable: The variable to load.
+ :type variable: String
+ :param esgf_username: ESGF OpenID value to use for authentication.
+ :type esgf_username: String
+ :param esgf_password: ESGF Password to use for authentication.
+ :type esgf_password: String
+ :param search_url: (Optional) The ESGF node to use for searching. Defaults
+ to the Jet Propulsion Laboratory node.
+ :type search_url: String
+ :param additional_constraints: (Optional) Additional key,value pairs to
+ pass as constraints to the search wrapper. These can be anything found
+ on the ESGF metadata page for a dataset.
+
+ :returns: An ocw.dataset.Dataset object contained the requested dataset.
+
+ :raises ValueError: If no dataset can be found for the supplied ID and
+ variable, or if the requested dataset is a multi-file dataset.
+ '''
urls = _get_file_urls(url=search_url,
id=dataset_id,
variable=variable,
@@ -56,6 +81,8 @@ def load_dataset(dataset_id,
def _get_file_urls(**constraints):
''''''
+ # Allow the user to optionally specify the URL for a search node to use.
+ # Default to the Jet Propulsion Laboratory node if nothing is specified.
if 'url' in constraints:
url = constraints['url']
constraints.pop('url', None)
@@ -68,8 +95,6 @@ def _get_file_urls(**constraints):
def _download_files(file_urls, username, password, download_directory='/tmp'):
''''''
- username = 'https://esg-datanode.jpl.nasa.gov/esgf-idp/openid/mjjoyce'
- password = '?pu}8gHM{2T2EB]jRX}TmKx6kCGEwPqfaP8aN8jGq8r4HDf82Y'
try:
logon2(username, password)
except urllib2.HTTPError:
[05/11] git commit: CLIMATE-518 - Add initial ESGF data_source
Posted by jo...@apache.org.
CLIMATE-518 - Add initial ESGF data_source
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/51a019c3
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/51a019c3
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/51a019c3
Branch: refs/heads/master
Commit: 51a019c344f039346bcf524df4a381659059318e
Parents: 3b30be7
Author: Michael Joyce <jo...@apache.org>
Authored: Sun Sep 21 04:38:59 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Sun Oct 19 14:19:35 2014 -0700
----------------------------------------------------------------------
ocw/data_source/esgf.py | 79 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 79 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/51a019c3/ocw/data_source/esgf.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/esgf.py b/ocw/data_source/esgf.py
new file mode 100644
index 0000000..7f5b6a6
--- /dev/null
+++ b/ocw/data_source/esgf.py
@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import urllib2
+
+from ocw.esgf.constants import JPL_SEARCH_SERVICE_URL
+from ocw.esgf.download import download
+from ocw.esgf.logon2 import logon2
+from ocw.esgf.search import SearchClient
+import ocw.data_source.local as local
+
+def load_dataset(dataset_id,
+ variable,
+ esgf_username,
+ esgf_password,
+ search_url=JPL_SEARCH_SERVICE_URL,
+ **additional_constraints):
+ ''''''
+ urls = _get_file_urls(url=search_url,
+ id=dataset_id,
+ variable=variable,
+ **additional_constraints)
+
+ if len(urls) > 1:
+ err = (
+ "esgf.load_dataset: Unable to handle multi-file datasets. "
+ "Feature coming soon ..."
+ )
+ raise ValueError(err)
+ elif len(urls) == 0:
+ err = (
+ "esgf.load_dataset: No files found for specified dataset."
+ )
+ raise ValueError(err)
+
+ # TODO: In the future, we need to combine multi-file datasets into a single
+ # file and then load it. For now we're only handling a single file use case.
+ _download_files(urls, esgf_username, esgf_password)
+ return local.load_file('/tmp/' + urls[0].split('/')[-1], variable)
+
+def _get_file_urls(**constraints):
+ ''''''
+ if 'url' in constraints:
+ url = constraints['url']
+ constraints.pop('url', None)
+ else:
+ url = ocw.esgf.constants.JPL_SEARCH_SERVICE_URL
+
+ sc = SearchClient(searchServiceUrl=url, distrib=False)
+ sc.setConstraint(**constraints)
+ return sc.getFiles()
+
+def _download_files(file_urls, username, password, download_directory='/tmp'):
+ ''''''
+ username = 'https://esg-datanode.jpl.nasa.gov/esgf-idp/openid/mjjoyce'
+ password = '?pu}8gHM{2T2EB]jRX}TmKx6kCGEwPqfaP8aN8jGq8r4HDf82Y'
+ try:
+ logon2(username, password)
+ except urllib2.HTTPError:
+ raise ValueError('esgf._download_files: Invalid login credentials')
+
+ for url in file_urls:
+ download(url, toDirectory=download_directory)
[04/11] git commit: CLIMATE-520 - Add ESGF module to setup.py
Posted by jo...@apache.org.
CLIMATE-520 - Add ESGF module to setup.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/3b30be75
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/3b30be75
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/3b30be75
Branch: refs/heads/master
Commit: 3b30be75530baaa0a69d3686b527a402795169ff
Parents: 79464c7
Author: Michael Joyce <jo...@apache.org>
Authored: Mon Sep 15 08:12:01 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Sun Oct 19 14:18:50 2014 -0700
----------------------------------------------------------------------
setup.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/3b30be75/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index edceca9..3874385 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from distutils.core import setup
+from setuptools import setup
setup(
name='Apache Open Climate Workbench',
@@ -23,7 +23,7 @@ setup(
url='http://climate.apache.org/index.html',
author='Apache Open Climate Workbench',
author_email='dev@climate.apache.org',
- packages=['ocw', 'ocw.data_source', 'ocw.tests'],
+ packages=['ocw', 'ocw.data_source', 'ocw.tests', 'ocw.esgf'],
license='Apache License, Version 2.0',
long_description='The Apache Open Climate Workbench provides tools for the evaluation and analysis of climate models.'
)
[07/11] git commit: CLIMATE-518 - Add ESGF data_source to Sphinx build
Posted by jo...@apache.org.
CLIMATE-518 - Add ESGF data_source to Sphinx build
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/2cc0401e
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/2cc0401e
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/2cc0401e
Branch: refs/heads/master
Commit: 2cc0401e178e9d40153860a46a1395b1f834f6ec
Parents: f625cf8
Author: Michael Joyce <jo...@apache.org>
Authored: Thu Oct 9 09:19:25 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Sun Oct 19 14:20:51 2014 -0700
----------------------------------------------------------------------
docs/source/data_source/data_sources.rst | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/2cc0401e/docs/source/data_source/data_sources.rst
----------------------------------------------------------------------
diff --git a/docs/source/data_source/data_sources.rst b/docs/source/data_source/data_sources.rst
index c368d86..19f9293 100644
--- a/docs/source/data_source/data_sources.rst
+++ b/docs/source/data_source/data_sources.rst
@@ -15,3 +15,8 @@ DAP Module
==========
.. automodule:: dap
:members:
+
+ESGF Module
+===========
+.. automodule:: esgf
+ :members:
[09/11] git commit: CLIMATE-518 - Add multi-dataset handling
Posted by jo...@apache.org.
CLIMATE-518 - Add multi-dataset handling
- Add multi-dataset handling to the ESGF data source. Loaded dataset(s)
are returned as a list now.
- Add support for elevation layer selection when loading a dataset.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/08b780d7
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/08b780d7
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/08b780d7
Branch: refs/heads/master
Commit: 08b780d7004dc0ef254c5da595fd2152960485f9
Parents: 1094b18
Author: Michael Joyce <jo...@apache.org>
Authored: Fri Oct 24 08:15:28 2014 -0700
Committer: Michael Joyce <jo...@apache.org>
Committed: Fri Oct 24 08:15:28 2014 -0700
----------------------------------------------------------------------
ocw/data_source/esgf.py | 81 +++++++++++++++++++++++---------------------
1 file changed, 43 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/08b780d7/ocw/data_source/esgf.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/esgf.py b/ocw/data_source/esgf.py
index a072b21..f138367 100644
--- a/ocw/data_source/esgf.py
+++ b/ocw/data_source/esgf.py
@@ -19,24 +19,24 @@
import urllib2
-from ocw.esgf.constants import JPL_SEARCH_SERVICE_URL
+from ocw.esgf.constants import DEFAULT_ESGF_SEARCH
from ocw.esgf.download import download
from ocw.esgf.logon2 import logon2
from ocw.esgf.search import SearchClient
import ocw.data_source.local as local
+from bs4 import BeautifulSoup
+import requests
+
def load_dataset(dataset_id,
variable,
esgf_username,
esgf_password,
- search_url=JPL_SEARCH_SERVICE_URL,
+ search_url=DEFAULT_ESGF_SEARCH,
+ elevation_index=0,
**additional_constraints):
''' Load an ESGF dataset.
- .. note:
- Currently, multi-file datasets aren't supported. This functionality
- will be added soon!
-
:param dataset_id: The ESGF ID of the dataset to load.
:type dataset_id: String
:param variable: The variable to load.
@@ -48,6 +48,8 @@ def load_dataset(dataset_id,
:param search_url: (Optional) The ESGF node to use for searching. Defaults
to the Jet Propulsion Laboratory node.
:type search_url: String
+ :param elevation_index: (Optional) The elevation level to strip out when
+ loading the dataset using ocw.data_source.local.
:param additional_constraints: (Optional) Additional key,value pairs to
pass as constraints to the search wrapper. These can be anything found
on the ESGF metadata page for a dataset.
@@ -57,41 +59,44 @@ def load_dataset(dataset_id,
:raises ValueError: If no dataset can be found for the supplied ID and
variable, or if the requested dataset is a multi-file dataset.
'''
- urls = _get_file_urls(url=search_url,
- id=dataset_id,
- variable=variable,
- **additional_constraints)
-
- if len(urls) > 1:
- err = (
- "esgf.load_dataset: Unable to handle multi-file datasets. "
- "Feature coming soon ..."
- )
- raise ValueError(err)
- elif len(urls) == 0:
- err = (
- "esgf.load_dataset: No files found for specified dataset."
- )
- raise ValueError(err)
+ download_data = _get_file_download_data(url=search_url,
+ dataset_id=dataset_id,
+ variable=variable)
+
+ datasets = []
+ for url, var in download_data:
+ _download_files([url], esgf_username, esgf_password)
+ datasets.append(local.load_file('/tmp/' + url.split('/')[-1],
+ var,
+ elevation_index=elevation_index))
- # TODO: In the future, we need to combine multi-file datasets into a single
- # file and then load it. For now we're only handling a single file use case.
- _download_files(urls, esgf_username, esgf_password)
- return local.load_file('/tmp/' + urls[0].split('/')[-1], variable)
+ return datasets
-def _get_file_urls(**constraints):
+def _get_file_download_data(dataset_id, variable, url=DEFAULT_ESGF_SEARCH):
''''''
- # Allow the user to optionally specify the URL for a search node to use.
- # Default to the Jet Propulsion Laboratory node if nothing is specified.
- if 'url' in constraints:
- url = constraints['url']
- constraints.pop('url', None)
- else:
- url = ocw.esgf.constants.JPL_SEARCH_SERVICE_URL
-
- sc = SearchClient(searchServiceUrl=url, distrib=False)
- sc.setConstraint(**constraints)
- return sc.getFiles()
+ url += '?distrib=false&type=File&dataset_id={}&variable={}'
+ url = url.format(dataset_id, variable)
+
+ r = requests.get(url)
+ xml = BeautifulSoup(r.content)
+
+ dont_have_results = not bool(xml.response.result['numfound'])
+
+ if dont_have_results:
+ err = "esgf.load_dataset: No files found for specified dataset."
+ raise ValueError(err)
+
+ # Split out URLs for dataset download along with variable names for each
+ # of those files.
+ url_groups = xml.response.result.findAll('arr', {'name': 'url'})
+ variable_groups = xml.response.result.findAll('arr', {'name': 'variable'})
+
+ urls = [group.findAll('str')[0].string.split('|')[0]
+ for group in url_groups]
+ variables = [group.findAll('str')[0].string
+ for group in variable_groups]
+
+ return zip(urls, variables)
def _download_files(file_urls, username, password, download_directory='/tmp'):
''''''