You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by pr...@apache.org on 2013/08/27 07:35:49 UTC
svn commit: r1517753 [2/33] - in /incubator/climate/branches/rcmet-2.1.1: ./
src/ src/main/ src/main/python/ src/main/python/bin/ src/main/python/docs/
src/main/python/docs/_static/ src/main/python/docs/_templates/
src/main/python/rcmes/ src/main/pytho...
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/make.bat
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/make.bat?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/make.bat (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/make.bat Tue Aug 27 05:35:42 2013
@@ -0,0 +1,190 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+ set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^<target^>` where ^<target^> is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. singlehtml to make a single large HTML file
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. devhelp to make HTML files and a Devhelp project
+ echo. epub to make an epub
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. text to make text files
+ echo. man to make manual pages
+ echo. texinfo to make Texinfo files
+ echo. gettext to make PO message catalogs
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "singlehtml" (
+ %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\rcmes.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\rcmes.ghc
+ goto end
+)
+
+if "%1" == "devhelp" (
+ %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished.
+ goto end
+)
+
+if "%1" == "epub" (
+ %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub file is in %BUILDDIR%/epub.
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "text" (
+ %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The text files are in %BUILDDIR%/text.
+ goto end
+)
+
+if "%1" == "man" (
+ %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The manual pages are in %BUILDDIR%/man.
+ goto end
+)
+
+if "%1" == "texinfo" (
+ %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+ goto end
+)
+
+if "%1" == "gettext" (
+ %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+:end
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/make.bat
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.cli.rst
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.cli.rst?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.cli.rst (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.cli.rst Tue Aug 27 05:35:42 2013
@@ -0,0 +1,19 @@
+cli Package
+===========
+
+:mod:`do_rcmes_processing_sub` Module
+-------------------------------------
+
+.. automodule:: rcmes.cli.do_rcmes_processing_sub
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`rcmet_ui` Module
+----------------------
+
+.. automodule:: rcmes.cli.rcmet_ui
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.cli.rst
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.rst
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.rst?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.rst (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.rst Tue Aug 27 05:35:42 2013
@@ -0,0 +1,22 @@
+rcmes Package
+=============
+
+:mod:`rcmes` Package
+--------------------
+
+.. automodule:: rcmes.__init__
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ rcmes.cli
+ rcmes.services
+ rcmes.storage
+ rcmes.toolkit
+ rcmes.utils
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.rst
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.services.rst
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.services.rst?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.services.rst (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.services.rst Tue Aug 27 05:35:42 2013
@@ -0,0 +1,51 @@
+services Package
+================
+
+:mod:`decode_model_times` Module
+--------------------------------
+
+.. automodule:: rcmes.services.decode_model_times
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`find_latlon_var` Module
+-----------------------------
+
+.. automodule:: rcmes.services.find_latlon_var
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`find_time_var` Module
+---------------------------
+
+.. automodule:: rcmes.services.find_time_var
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`list_vars_in_file` Module
+-------------------------------
+
+.. automodule:: rcmes.services.list_vars_in_file
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`main_ws` Module
+---------------------
+
+.. automodule:: rcmes.services.main_ws
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`run_rcmes_processing` Module
+----------------------------------
+
+.. automodule:: rcmes.services.run_rcmes_processing
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.services.rst
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.storage.rst
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.storage.rst?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.storage.rst (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.storage.rst Tue Aug 27 05:35:42 2013
@@ -0,0 +1,27 @@
+storage Package
+===============
+
+:mod:`db` Module
+----------------
+
+.. automodule:: rcmes.storage.db
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`files` Module
+-------------------
+
+.. automodule:: rcmes.storage.files
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`rcmed` Module
+-------------------
+
+.. automodule:: rcmes.storage.rcmed
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.storage.rst
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.toolkit.rst
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.toolkit.rst?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.toolkit.rst (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.toolkit.rst Tue Aug 27 05:35:42 2013
@@ -0,0 +1,27 @@
+toolkit Package
+===============
+
+:mod:`metrics` Module
+---------------------
+
+.. automodule:: rcmes.toolkit.metrics
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`plots` Module
+-------------------
+
+.. automodule:: rcmes.toolkit.plots
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`process` Module
+---------------------
+
+.. automodule:: rcmes.toolkit.process
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.toolkit.rst
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.utils.rst
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.utils.rst?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.utils.rst (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.utils.rst Tue Aug 27 05:35:42 2013
@@ -0,0 +1,19 @@
+utils Package
+=============
+
+:mod:`fortranfile` Module
+-------------------------
+
+.. automodule:: rcmes.utils.fortranfile
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`misc` Module
+------------------
+
+.. automodule:: rcmes.utils.misc
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/docs/rcmes.utils.rst
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/README
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/README?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/README (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/README Tue Aug 27 05:35:42 2013
@@ -0,0 +1,8 @@
+Regional Climate Model Evaluation System
+
+Installation instructions:
+ 1) For python to discover this package you need to set the PYTHONPATH environment variable to point to this directory that the rcmes directory sits in.
+
+Usage instructions:
+ 1) import rcmes
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/README
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/__init__.py
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/__init__.py?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/__init__.py (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/__init__.py Tue Aug 27 05:35:42 2013
@@ -0,0 +1 @@
+""" Regional Climate Model Evaluation System"""
\ No newline at end of file
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/__init__.py
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/classes.py
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/classes.py?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/classes.py (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/classes.py Tue Aug 27 05:35:42 2013
@@ -0,0 +1,211 @@
+import calendar
+import os
+from datetime import datetime as datetime
+import urllib
+
+import storage.files as files
+import toolkit.process as process
+import utils.misc
+
+class BoundingBox(object):
+
+ def __init__(self, latMin, lonMin, latMax, lonMax):
+ self.latMin = latMin
+ self.lonMin = lonMin
+ self.latMax = latMax
+ self.lonMax = lonMax
+
+
+class SubRegion(BoundingBox):
+
+ def __init__(self, name, latMin, lonMin, latMax, lonMax):
+ BoundingBox.__init__(self, latMin, lonMin, latMax, lonMax)
+ self.name = name
+
+class GridBox(BoundingBox):
+
+ def __init__(self, latMin, lonMin, latMax, lonMax, lonStep, latStep):
+ BoundingBox.__init__(self, latMin, lonMin, latMax, lonMax)
+ self.lonStep = lonStep
+ self.latStep = latStep
+ self.lonCount = int((self.lonMax - self.lonMin) / self.lonStep) + 1
+ self.latCount = int((self.latMax - self.latMin) / self.latStep) + 1
+
+class JobProperties(object):
+
+ def __init__(self, workDir, cacheDir, spatialGrid, temporalGrid, gridLonStep=None, gridLatStep=None, outputFile='false',
+ latMin=None, latMax=None, lonMin=None, lonMax=None, startDate=None, endDate=None):
+ self.workDir = os.path.abspath(workDir)
+ self.cacheDir = os.path.abspath(cacheDir)
+ self.spatialGrid = spatialGrid
+ self.temporalGrid = temporalGrid
+
+ if gridLonStep and gridLatStep:
+ self.gridLonStep = float(gridLonStep)
+ self.gridLatStep = float(gridLatStep)
+ else:
+ self.gridLonStep = None
+ self.gridLatStep = None
+
+ # Support for both User provided Dates, and Interactive Date Collection
+ if startDate and endDate:
+ self.startDate = datetime.strptime(startDate, '%Y%m%d')
+ self.endDate = datetime.strptime(endDate, '%Y%m%d')
+ else:
+ self.startDate = None
+ self.endDate = None
+
+ if outputFile.lower() == 'false':
+ self.writeOutFile = 'no'
+ elif outputFile.lower() == 'netcdf':
+ self.writeOutFile = 'nc'
+ else:
+ self.writeOutFile = False
+
+ if self.spatialGrid.lower() == 'user':
+ self.latMin = float(latMin)
+ self.latMax = float(latMax)
+ self.lonMin = float(lonMin)
+ self.lonMax = float(lonMax)
+
+ def obsDatasetCount(self):
+ self.parameterList = self.obsParamId.split(',')
+ count = len(self.paramterList)
+ return count
+
+class Model(object):
+
+ def __init__(self, *args, **kwargs):
+ """
+ This Class can be instantiated with either a single filename or a dictionary of parameters.
+ If a single filename is used, then the class will use internal methods to parse the file
+ and set the needed attributes.
+
+ Input (single file)::
+ filename - Full path to a local model file
+
+ Input (keyword dictionary)::
+ filename - Full path to a local model file
+ latVariable - the name of the latitude variable within the file
+ lonVariable - the name of the longitude variable within the file
+ timeVariable - the name of the time variable within the file
+ timeStep - description of the time between readings ['hourly','daily','monthly','annual']
+ varName - name of the variable to analyze within the model
+ precipFlag - boolean telling if this is precipitation data
+
+ Output::
+ Model object
+ """
+ if len(args) == 1:
+ self.filename = args[0]
+ self.name = os.path.basename(self.filename)
+ self.processModelFile()
+ self.precipFlag = False
+ if len(kwargs) == 7:
+ self.filename = kwargs['filename']
+ self.name = os.path.basename(self.filename)
+ self.latVariable = kwargs['latVariable']
+ self.lonVariable = kwargs['lonVariable']
+ self.timeVariable = kwargs['timeVariable']
+ self.timeStep = kwargs['timeStep']
+ self.varName = kwargs['varName']
+ self.times, _ = process.getModelTimes(self.filename, self.timeVariable)
+ self.minTime = min(self.times)
+ self.maxTime = max(self.times)
+ self.setLatitudeRange()
+ self.setLongitudeRange()
+
+ if kwargs['precipFlag'] == 'True':
+ self.precipFlag = True
+ else:
+ self.precipFlag = False
+
+
+ def setLatitudeRange(self):
+ self.latMin, self.latMax = files.getVariableRange(self.filename, self.latVariable)
+
+ def setLongitudeRange(self):
+ self.lonMin, self.lonMax = files.getVariableRange(self.filename, self.lonVariable)
+
+ def processModelFile(self):
+ """ This series of steps should be consolidated to merely pass around a PyNIO object
+ Until then we will be opening the same file repeatedly. And clearly wasting I/O
+ """
+ self.latVariable, self.lonVariable, self.latMin, self.latMax, self.lonMin, self.lonMax = files.findLatLonVarFromFile(self.filename)
+ self.timeVariable, modelVariables = files.findTimeVariable(self.filename)
+ self.times, self.timeStep = process.getModelTimes(self.filename, self.timeVariable)
+ self.varName = utils.misc.askUserForVariableName(modelVariables, "analysis")
+ self.minTime = min(self.times)
+ self.maxTime = max(self.times)
+
+class Parameter(object):
+ def __init__(self, param_id, shortName=None, description=None, endDate=None ):
+ pass
+
+class RCMED(object):
+ def __init__(self):
+ pass
+
+ def retriveData(self, parameter, dataBounds, timeStep=None):
+ # Returns the data
+ pass
+
+ @classmethod
+ def isoDateString(self, pythonDate):
+ isoDate = pythonDate.strftime("%Y%m%dT%H%MZ")
+ return isoDate
+
+ @classmethod
+ def jplUrl(self, datasetID, paramID, latMin, latMax, lonMin, lonMax, startTime, endTime, cachedir, timestep):
+ """ This will create a valid RCMED Query URL used to contact the JPL RCMED Instance"""
+ JPL_RCMED_URL = 'http://rcmes.jpl.nasa.gov/query-api/query.php?'
+
+ """This block will expand the Time Range based on the timestep to ensure complete temporal range coverage"""
+ expanded = False
+ if timestep.lower() == 'monthly':
+ if startTime.day != 1:
+ # Clean the startTime
+ startTimeString = startTime.strftime('%Y%m%d')
+ normalInputDatetimeString = startTimeString[:6] + '01'
+ startTime = datetime.strptime(normalInputDatetimeString, '%Y%m%d')
+ expanded = True
+
+ lastDayOfMonth = calendar.monthrange(endTime.year, endTime.month)[1]
+ if endTime.day != lastDayOfMonth:
+ # Clean the endTime
+ endTimeString = endTime.strftime('%Y%m%d')
+ endTimeString = endTimeString[:6] + str(lastDayOfMonth)
+ endTime = datetime.strptime(endTimeString, '%Y%m%d')
+ expanded = True
+
+ elif timestep.lower() == 'daily':
+ if startTime.hour != 0 or startTime.minute != 0 or startTime.second != 0:
+ datetimeString = startTime.strftime('%Y%m%d%H%M%S')
+ normalDatetimeString = datetimeString[:8] + '000000'
+ startTime = datetime.strptime(normalDatetimeString, '%Y%m%d%H%M%S')
+ expanded = True
+
+ endTimeString = endTime.strftime('%Y%m%d%H%M%S')
+ endTimeString = endTimeString[:8] + '235959'
+ endTime = datetime.strptime(endTimeString, '%Y%m%d%H%M%S')
+
+ if expanded:
+ print "Your date range selection has been expanded to ensure we return the largest number of available readings."
+ print "The new date Range is: %s through %s" % (startTime.strftime('%Y-%m-%d %H:%M:%S'), endTime.strftime('%Y-%m-%d %H:%M:%S'))
+
+
+ timeStart = self.isoDateString(startTime)
+ timeEnd = self.isoDateString(endTime)
+ query = [('datasetId',datasetID), ('parameterId',paramID), ('latMin',latMin), ('latMax',latMax),
+ ('lonMin', lonMin), ('lonMax',lonMax), ('timeStart', timeStart), ('timeEnd', timeEnd)]
+ queryURL = urllib.urlencode(query)
+ urlRequest = JPL_RCMED_URL+queryURL
+
+ return urlRequest
+
+class RCMEDDataset(object):
+
+ def __init__(self, obsDatasetId, obsParamId, obsTimeStep):
+ self.datasetId = obsDatasetId
+ self.parameterId = obsParamId
+ self.timeStep = obsTimeStep
\ No newline at end of file
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/classes.py
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/__init__.py
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/__init__.py?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/__init__.py (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/__init__.py Tue Aug 27 05:35:42 2013
@@ -0,0 +1,11 @@
+"""This is the Command Line Interface Package
+
+Here we have a collection of modules that can be used to interact with the RCMES
+from the command line.
+
+Example:
+------------
+
+>>> ./rcmet_ui.py
+
+"""
\ No newline at end of file
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/__init__.py
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/do_rcmes_processing_sub.py
URL: http://svn.apache.org/viewvc/incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/do_rcmes_processing_sub.py?rev=1517753&view=auto
==============================================================================
--- incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/do_rcmes_processing_sub.py (added)
+++ incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/do_rcmes_processing_sub.py Tue Aug 27 05:35:42 2013
@@ -0,0 +1,730 @@
+#!/usr/local/bin/python
+"""
+ PENDING DEPRICATION - YOU SHOULD INSTEAD USE THE rcmet.py within the bin dir
+
+ Module that is used to lauch the rcmes processing from the rcmet_ui.py
+ script.
+"""
+
+import sys
+import datetime
+import numpy
+import numpy.ma as ma
+import toolkit.plots as plots
+
+import storage.db as db
+import storage.files as files
+import toolkit.process as process
+import toolkit.metrics as metrics
+
+
+def do_rcmes(settings, params, model, mask, options):
+ '''
+ Routine to perform full end-to-end RCMET processing.
+
+ i) retrieve observations from the database
+ ii) load in model data
+ iii) temporal regridding
+ iv) spatial regridding
+ v) area-averaging
+ vi) seasonal cycle compositing
+ vii) metric calculation
+ viii) plot production
+
+ Input:
+ 5 dictionaries which contain a huge argument list with all of the user options
+ (which can be collected from the GUI)
+
+ settings - dictionary of rcmes run settings::
+
+ settings = {"cacheDir": string describing directory path,
+ "workDir": string describing directory path,
+ "fileList": string describing model file name + path }
+
+ params - dictionary of rcmes run parameters::
+
+ params = {"obsDatasetId": int( db dataset id ),
+ "obsParamId": int( db parameter id ),
+ "startTime": datetime object (needs to change to string + decode),
+ "endTime": datetime object (needs to change to string + decode),
+ "latMin": float,
+ "latMax": float,
+ "lonMin": float,
+ "lonMax": float }
+
+ model - dictionary of model parameters::
+
+ model = {"varName": string describing name of variable to evaluate (as written in model file),
+ "timeVariable": string describing name of time variable (as written in model file),
+ "latVariable": string describing name of latitude variable (as written in model file),
+ "lonVariable": string describing name of longitude variable (as written in model file) }
+
+ mask - dictionary of mask specific options (only used if options['mask']=True)::
+
+ mask = {"latMin": float,
+ "latMax": float,
+ "lonMin": float,
+ "lonMax": float}
+
+ options - dictionary full of different user supplied options::
+
+ options = {"regrid": str( 'obs' | 'model' | 'regular' ),
+ "timeRegrid": str( 'full' | 'annual' | 'monthly' | 'daily' ),
+ "seasonalCycle": Boolean,
+ "metric": str('bias'|'mae'|'acc'|'pdf'|'patcor'|'rms'|'diff'),
+ "plotTitle": string describing title to use in plot graphic,
+ "plotFilename": basename of file to use for plot graphic i.e. {plotFilename}.png,
+ "mask": Boolean,
+ "precip": Boolean }
+
+ Output: image files of plots + possibly data
+ '''
+
+ # check the number of model data files
+ if len(settings['fileList']) < 1: # no input data file
+ print 'No input model data file. EXIT'
+ sys.exit()
+ # assign parameters that must be preserved throughout the process
+ if options['mask'] == True:
+ options['seasonalCycle'] = True
+
+ ###########################################################################
+ # Part 1: retrieve observation data from the database
+ # NB. automatically uses local cache if already retrieved.
+ ###########################################################################
+ rcmedData = getDataFromRCMED( params, settings, options )
+
+ ###########################################################################
+ # Part 2: load in model data from file(s)
+ ###########################################################################
+ modelData = getDataFromModel( model, settings )
+
+ ###########################################################################
+ # Deal with some precipitation specific options
+ # i.e. adjust units of model data and set plot color bars suitable for precip
+ ###########################################################################
+ colorbar = 'rainbow'
+ if options['precip'] == True:
+ modelData['data'] = modelData['data']*86400. # convert from kgm-2s-1 into mm/day
+ colorbar = 'precip2_17lev'
+
+ # set color bar suitable for MODIS cloud data
+ if params['obsParamId'] == 31:
+ colorbar = 'gsdtol'
+
+ ##################################################################################################################
+ # Extract sub-selection of model data for required time range.
+ # e.g. a single model file may contain data for 20 years,
+ # but the user may have selected to only analyse data between 2003 and 2004.
+ ##################################################################################################################
+
+ # Make list of indices where modelData['times'] are between params['startTime'] and params['endTime']
+ modelTimeOverlap = numpy.logical_and((numpy.array(modelData['times'])>=params['startTime']),
+ (numpy.array(modelData['times'])<=params['endTime']))
+
+ # Make subset of modelData['times'] using full list of times and indices calculated above
+ modelData['times'] = list(numpy.array(modelData['times'])[modelTimeOverlap])
+
+ # Make subset of modelData['data'] using full model data and indices calculated above
+ modelData['data'] = modelData['data'][modelTimeOverlap, :, :]
+
+ ##################################################################################################################
+ # Part 3: Temporal regridding
+ # i.e. model data may be monthly, and observation data may be daily.
+ # We need to compare like with like so the User Interface asks what time unit the user wants to work with
+ # e.g. the user may select that they would like to regrid everything to 'monthly' data
+ # in which case, the daily observational data will be averaged onto monthly data
+ # so that it can be compared directly with the monthly model data.
+ ##################################################################################################################
+ print 'Temporal Regridding Started'
+
+ if(options['timeRegrid']):
+ # Run both obs and model data through temporal regridding routine.
+ # NB. if regridding not required (e.g. monthly time units selected and model data is already monthly),
+ # then subroutine detects this and returns data untouched.
+ rcmedData['data'], newObsTimes = process.calc_average_on_new_time_unit(rcmedData['data'],
+ rcmedData['times'],
+ unit=options['timeRegrid'])
+
+ modelData['data'], newModelTimes = process.calc_average_on_new_time_unit(modelData['data'],
+ modelData['times'],
+ unit=options['timeRegrid'])
+
+ # Set a new 'times' list which describes the common times used for both model and obs after the regrid.
+ if newObsTimes == newModelTimes:
+ times = newObsTimes
+
+ ###########################################################################
+ # Catch situations where after temporal regridding the times in model and obs don't match.
+ # If this occurs, take subset of data from times common to both model and obs only.
+ # e.g. imagine you are looking at monthly model data,
+ # the model times are set to the 15th of each month.
+ # + you are comparing against daily obs data.
+ # If you set the start date as Jan 1st, 1995 and the end date as Jan 1st, 1996
+ # -then system will load all model data in this range with the last date as Dec 15th, 1995
+ # loading the daily obs data from the database will have a last data item as Jan 1st, 1996.
+ # If you then do temporal regridding of the obs data from daily -> monthly (to match the model)
+ # Then there will be data for Jan 96 in the obs, but only up to Dec 95 for the model.
+ # This section of code deals with this situation by only looking at data
+ # from the common times between model and obs after temporal regridding.
+ ###########################################################################
+ if newObsTimes != newModelTimes:
+ print 'Warning: after temporal regridding, times from observations and model do not match'
+ print 'Check if this is unexpected.'
+ print 'Proceeding with data from times common in both model and obs.'
+
+ # Create empty lists ready to store data
+ times = []
+ tempModelData = []
+ tempObsData = []
+
+ # Loop through each time that is common in both model and obs
+ for commonTime in numpy.intersect1d(newObsTimes, newModelTimes):
+ # build up lists of times, and model and obs data for each common time
+ # NB. use lists for data for convenience (then convert to masked arrays at the end)
+ times.append(newObsTimes[numpy.where(numpy.array(newObsTimes) == commonTime)[0][0]])
+ tempModelData.append(modelData['data'][numpy.where(numpy.array(newModelTimes) == commonTime)[0][0], :, :])
+ tempObsData.append(rcmedData['data'][numpy.where(numpy.array(newObsTimes) == commonTime)[0][0], :, :])
+
+ # Convert data arrays from list back into full 3d arrays.
+ modelData['data'] = ma.array(tempModelData)
+ rcmedData['data'] = ma.array(tempObsData)
+
+ # Reset all time lists so representative of the data actually used.
+ newObsTimes = times
+ newModelTimes = times
+ rcmedData['times'] = times
+ modelData['times'] = times
+
+ ##################################################################################################################
+ # Part 4: spatial regridding
+ # The model and obs are rarely on the same grid.
+ # To compare the two, you need them to be on the same grid.
+ # The User Interface asked the user if they'd like to regrid everything to the model grid or the obs grid.
+ # Alternatively, they could chose to regrid both model and obs onto a third regular lat/lon grid as defined
+ # by parameters that they enter.
+ #
+ # NB. from this point on in the code, the 'lats' and 'lons' arrays are common to
+ # both rcmedData['data'] and modelData['data'].
+ ##################################################################################################################
+
+ ##################################################################################################################
+ # either i) Regrid obs data to model grid.
+ ##################################################################################################################
+ if options['regrid'] == 'model':
+ # User chose to regrid observations to the model grid
+ modelData['data'], rcmedData['data'], lats, lons = process.regrid_wrapper('0', rcmedData['data'],
+ rcmedData['lats'],
+ rcmedData['lons'],
+ modelData['data'],
+ modelData['lats'],
+ modelData['lons'])
+
+ ##################################################################################################################
+ # or ii) Regrid model data to obs grid.
+ ##################################################################################################################
+ if options['regrid'] == 'obs':
+ # User chose to regrid model data to the observation grid
+
+ modelData['data'], rcmedData['data'], lats, lons = process.regrid_wrapper('1', rcmedData['data'],
+ rcmedData['lats'],
+ rcmedData['lons'],
+ modelData['data'],
+ modelData['lats'],
+ modelData['lons'])
+
+ ##################################################################################################################
+ # or iii) Regrid both model data and obs data to new regular lat/lon grid.
+ ##################################################################################################################
+ if options['regrid'] == 'regular':
+ # User chose to regrid both model and obs data onto a newly defined regular lat/lon grid
+ # Construct lats, lons from grid parameters
+
+ # Create 1d lat and lon arrays
+ lat = numpy.arange(nLats)*dLat+Lat0
+ lon = numpy.arange(nLons)*dLon+Lon0
+
+ # Combine 1d lat and lon arrays into 2d arrays of lats and lons
+ lons, lats = numpy.meshgrid(lon, lat)
+
+ ###########################################################################################################
+ # Regrid model data for every time
+ # NB. store new data in a list and convert back to an array at the end.
+ ###########################################################################################################
+ tmpModelData = []
+
+ timeCount = modelData['data'].shape[0]
+ for t in numpy.arange(timeCount):
+ tmpModelData.append(process.do_regrid(modelData['data'][t, :, :],
+ modelData['lats'][:, :],
+ modelData['lons'][:, :],
+ rcmedData['lats'][:, :],
+ rcmedData['lons'][:, :]))
+
+ # Convert list back into a masked array
+ modelData['data'] = ma.array(tmpModelData)
+
+ ###########################################################################################################
+ # Regrid obs data for every time
+ # NB. store new data in a list and convert back to an array at the end.
+ ###########################################################################################################
+ tempObsData = []
+ timeCount = rcmedData['data'].shape[0]
+ for t in numpy.arange(timeCount):
+ tempObsData.append(process.do_regrid(rcmedData['data'][t, :, :],
+ rcmedData['lats'][:, :],
+ rcmedData['lons'][:, :],
+ modelData['lats'][:, :], modelData['lons'][:, :]))
+
+ # Convert list back into a masked array
+ rcmedData['data'] = ma.array(tempObsData)
+
+ ##################################################################################################################
+ # (Optional) Part 5: area-averaging
+ #
+ # RCMET has the ability to either calculate metrics at every grid point,
+ # or to calculate metrics for quantities area-averaged over a defined (masked) region.
+ #
+ # If the user has selected to perform area-averaging,
+ # then they have also selected how they want to define
+ # the area to average over.
+ # The options were:
+ # -define masked region using regular lat/lon bounding box parameters
+ # -read in masked region from file
+ #
+ # either i) Load in the mask file (if required)
+ # or ii) Create the mask using latlonbox
+ # then iii) Do the area-averaging
+ #
+ ###############################################################################################################
+ if options['mask'] == True: # i.e. define regular lat/lon box for area-averaging
+ print 'Using Latitude/Longitude Mask for Area Averaging'
+
+ ###############################################################################################################
+ # Define mask using regular lat/lon box specified by users (i.e. ignore regions where mask = True)
+ ###############################################################################################################
+ mask = numpy.logical_or(numpy.logical_or(lats<=mask['latMin'], lats>=mask['latMax']),
+ numpy.logical_or(lons<=mask['lonMin'], lons>=mask['lonMax']))
+
+ ######################m########################################################################################
+ # Calculate area-weighted averages within this region and store in new lists
+ ###############################################################################################################
+ modelStore = []
+ timeCount = modelData['data'].shape[0]
+ for t in numpy.arange(timeCount):
+ modelStore.append(process.calc_area_mean(modelData['data'][t, :, :], lats, lons, mymask=mask))
+
+ obsStore = []
+ timeCount = rcmedData['data'].shape[0]
+ for t in numpy.arange(timeCount):
+ obsStore.append(process.calc_area_mean(rcmedData['data'][t, :, :], lats, lons, mymask=mask))
+
+ ###############################################################################################################
+ # Now overwrite data arrays with the area-averaged values
+ ###############################################################################################################
+ modelData['data'] = ma.array(modelStore)
+ rcmedData['data'] = ma.array(obsStore)
+
+ ###############################################################################################################
+ # Free-up some memory by overwriting big variables
+ ###############################################################################################################
+ obsStore = 0
+ modelStore = 0
+
+ ##############################################################################################################
+ # NB. if area-averaging has been performed then the dimensions of the data arrays will have changed from 3D to 1D
+ # i.e. only one value per time.
+ ##############################################################################################################
+
+ ##############################################################################################################
+ # (Optional) Part 6: seasonal cycle compositing
+ #
+ # RCMET has the ability to calculate seasonal average values from a long time series of data.
+ #
+ # e.g. for monthly data going from Jan 1980 - Dec 2010
+ # If the user selects to do seasonal cycle compositing,
+ # this section calculates the mean of all Januarys, mean of all Februarys, mean of all Marchs etc
+ # -result has 12 times.
+ #
+ # NB. this works with incoming 3D data or 1D data (e.g. time series after avea-averaging).
+ #
+ # If no area-averaging has been performed in Section 5,
+ # then the incoming data is 3D, and the outgoing data will also be 3D,
+ # but with the number of times reduced to 12
+ # i.e. you will get 12 map plots each one showing the average values for a month. (all Jans, all Febs etc)
+ #
+ #
+ # If area-averaging has been performed in Section 5,
+ # then the incoming data is 1D, and the outgoing data will also be 1D,
+ # but with the number of times reduced to 12
+ # i.e. you will get a time series of 12 data points
+ # each one showing the average values for a month. (all Jans, all Febs etc).
+ #
+ ##################################################################################################################
+ if options['seasonalCycle'] == True:
+ print 'Compositing data to calculate seasonal cycle'
+
+ modelData['data'] = metrics.calc_annual_cycle_means(modelData['data'], modelData['times'])
+ rcmedData['data'] = metrics.calc_annual_cycle_means(rcmedData['data'], modelData['times'])
+
+ ##################################################################################################################
+ # Part 7: metric calculation
+ # Calculate performance metrics comparing rcmedData['data'] and modelData['data'].
+ # All output is stored in metricData regardless of what metric was calculated.
+ #
+ # NB. the dimensions of metricData will vary depending on the dimensions of the incoming data
+ # *and* on the type of metric being calculated.
+ #
+ # e.g. bias between incoming 1D model and 1D obs data (after area-averaging) will be a single number.
+ # bias between incoming 3D model and 3D obs data will be 2D, i.e. a map of mean bias.
+ # correlation coefficient between incoming 3D model and 3D obs data will be 1D time series.
+ #
+ ##################################################################################################################
+
+ if options['metric'] == 'bias':
+ metricData = metrics.calc_bias(modelData['data'], rcmedData['data'])
+ metricTitle = 'Bias'
+
+ if options['metric'] == 'mae':
+ metricData = metrics.calc_mae(modelData['data'], rcmedData['data'])
+ metricTitle = 'Mean Absolute Error'
+
+ if options['metric'] == 'rms':
+ metricData = metrics.calc_rms(modelData['data'], rcmedData['data'])
+ metricTitle = 'RMS error'
+
+ if options['metric'] == 'difference':
+ metricData = metrics.calc_difference(modelData['data'], rcmedData['data'])
+ metricTitle = 'Difference'
+
+ #if options['metric'] == 'patcor':
+ #metricData = metrics.calc_pat_cor2D(modelData['data'], rcmedData['data'])
+ #metricTitle = 'Pattern Correlation'
+
+ if options['metric'] == 'nacc':
+ metricData = metrics.calc_anom_corn(modelData['data'], rcmedData['data'])
+ metricTitle = 'Anomaly Correlation'
+
+ if options['metric'] == 'pdf':
+ metricData = metrics.calc_pdf(modelData['data'], rcmedData['data'])
+ metricTitle = 'Probability Distribution Function'
+
+ if options['metric'] == 'coe':
+ metricData = metrics.calc_nash_sutcliff(modelData['data'], rcmedData['data'])
+ metricTitle = 'Coefficient of Efficiency'
+
+ if options['metric'] == 'stddev':
+ metricData = metrics.calc_stdev(modelData['data'])
+ data2 = metrics.calc_stdev(rcmedData['data'])
+ metricTitle = 'Standard Deviation'
+
+ ##################################################################################################################
+ # Part 8: Plot production
+ #
+ # Produce plots of metrics and obs, model data.
+ # Type of plot produced depends on dimensions of incoming data.
+ # e.g. 1D data is plotted as a time series.
+ # 2D data is plotted as a map.
+ # 3D data is plotted as a sequence of maps.
+ #
+ ##################################################################################################################
+
+ ##################################################################################################################
+ # 1 dimensional data, e.g. Time series plots
+ ##################################################################################################################
+ if metricData.ndim == 1:
+ print 'Producing time series plots ****'
+ print metricData
+ yearLabels = True
+ # mytitle = 'Area-average model v obs'
+
+ ################################################################################################################
+ # If producing seasonal cycle plots, don't want to put year labels on the time series plots.
+ ################################################################################################################
+ if options['seasonalCycle'] == True:
+ yearLabels = False
+ mytitle = 'Annual cycle: area-average model v obs'
+ # Create a list of datetimes to represent the annual cycle, one per month.
+ times = []
+ for m in xrange(12):
+ times.append(datetime.datetime(2000, m+1, 1, 0, 0, 0, 0))
+
+ ###############################################################################################
+ # Special case for pattern correlation plots. TODO: think of a cleaner way of doing this.
+ # Only produce these plots if the metric is NOT pattern correlation.
+ ###############################################################################################
+
+ # TODO - Clean up this if statement. We can use a list of values then ask if not in LIST...
+ #KDW: change the if statement to if else to accommodate the 2D timeseries plots
+ if (options['metric'] != 'patcor')&(options['metric'] != 'acc')&(options['metric'] != 'nacc')&(options['metric'] != 'coe')&(options['metric'] != 'pdf'):
+ # for anomaly and pattern correlation,
+ # can't plot time series of model, obs as these are 3d fields
+ # ^^ This is the reason modelData['data'] has been swapped for metricData in
+ # the following function
+ # TODO: think of a cleaner way of dealing with this.
+
+ ###########################################################################################
+ # Produce the time series plots with two lines: obs and model
+ ###########################################################################################
+ print 'two line timeseries'
+ # mytitle = options['plotTitle']
+ mytitle = 'Area-average model v obs'
+ if options['plotTitle'] == 'default':
+ mytitle = metricTitle+' model & obs'
+ #plots.draw_time_series_plot(modelData['data'],times,options['plotFilename']+'both',
+ # settings['workDir'],data2=rcmedData['data'],mytitle=mytitle,
+ # ytitle='Y',xtitle='time',
+ # year_labels=yearLabels)
+ plots.draw_time_series_plot(metricData, times, options['plotFilename']+'both',
+ settings['workDir'], data2, mytitle=mytitle,
+ ytitle='Y', xtitle='time',
+ year_labels=yearLabels)
+
+ else:
+ ###############################################################################################
+ # Produce the metric time series plot (one line only)
+ ###############################################################################################
+ mytitle = options['plotTitle']
+ if options['plotTitle'] == 'default':
+ mytitle = metricTitle+' model v obs'
+ print 'one line timeseries'
+ plots.draw_time_series_plot(metricData, times, options['plotFilename'],
+ settings['workDir'], mytitle=mytitle, ytitle='Y', xtitle='time',
+ year_labels=yearLabels)
+
+ ###############################################################################################
+ # 2 dimensional data, e.g. Maps
+ ###############################################################################################
+ if metricData.ndim == 2:
+
+ ###########################################################################################
+ # Calculate color bar ranges for data such that same range is used in obs and model plots
+ # for like-with-like comparison.
+ ###########################################################################################
+ mymax = max(rcmedData['data'].mean(axis=0).max(), modelData['data'].mean(axis=0).max())
+ mymin = min(rcmedData['data'].mean(axis=0).min(), modelData['data'].mean(axis=0).min())
+
+ ###########################################################################################
+ # Time title labels need their format adjusting depending on the temporal regridding used,
+ # e.g. if data are averaged to monthly,
+ # then want to write 'Jan 2002', 'Feb 2002', etc instead of 'Jan 1st, 2002', 'Feb 1st, 2002'
+ #
+ # Also, if doing seasonal cycle compositing
+ # then want to write 'Jan','Feb','Mar' instead of 'Jan 2002','Feb 2002','Mar 2002' etc
+ # as data are representative of all Jans, all Febs etc.
+ ###########################################################################################
+ if(options['timeRegrid'] == 'daily'):
+ timeFormat = "%b %d, %Y"
+ if(options['timeRegrid'] == 'monthly'):
+ timeFormat = "%b %Y"
+ if(options['timeRegrid'] == 'annual'):
+ timeFormat = "%Y"
+ if(options['timeRegrid'] == 'full'):
+ timeFormat = "%b %d, %Y"
+
+ ###########################################################################################
+ # Special case: when plotting bias data, we also like to plot the mean obs and mean model data.
+ # In this case, we need to calculate new time mean values for both obs and model.
+ # When doing this time averaging, we also need to deal with missing data appropriately.
+ #
+ # Classify missing data resulting from multiple times (using threshold data requirment)
+ # i.e. if the working time unit is monthly data, and we are dealing with multiple months of data
+ # then when we show mean of several months, we need to decide what threshold of missing data we tolerate
+ # before classifying a data point as missing data.
+ ###########################################################################################
+
+ ###########################################################################################
+ # Calculate time means of model and obs data
+ ###########################################################################################
+ modelDataMean = modelData['data'].mean(axis=0)
+ obsDataMean = rcmedData['data'].mean(axis=0)
+
+ ###########################################################################################
+ # Calculate missing data masks using tolerance threshold of missing data going into calculations
+ ###########################################################################################
+ obsDataMask = process.create_mask_using_threshold(rcmedData['data'], threshold=0.75)
+ modelDataMask = process.create_mask_using_threshold(modelData['data'], threshold=0.75)
+
+ ###########################################################################################
+ # Combine data and masks into masked arrays suitable for plotting.
+ ###########################################################################################
+ modelDataMean = ma.masked_array(modelDataMean, modelDataMask)
+ obsDataMean = ma.masked_array(obsDataMean, obsDataMask)
+
+ ###########################################################################################
+ # Plot model data
+ ###########################################################################################
+ mytitle = 'Model data: mean between %s and %s' % ( modelData['times'][0].strftime(timeFormat),
+ modelData['times'][-1].strftime(timeFormat) )
+ plots.draw_map_color_filled(modelDataMean, lats, lons, options['plotFilename']+'model',
+ settings['workDir'], mytitle=mytitle, rangeMax=mymax,
+ rangeMin=mymin, colorTable=colorbar, niceValues=True)
+
+ ###########################################################################################
+ # Plot obs data
+ ###########################################################################################
+ mytitle = 'Obs data: mean between %s and %s' % ( rcmedData['times'][0].strftime(timeFormat),
+ rcmedData['times'][-1].strftime(timeFormat) )
+ plots.draw_map_color_filled(obsDataMean, lats, lons, options['plotFilename']+'obs',
+ settings['workDir'], mytitle=mytitle, rangeMax=mymax,
+ rangeMin=mymin, colorTable=colorbar, niceValues=True)
+
+ ###########################################################################################
+ # Plot metric
+ ###########################################################################################
+ mymax = metricData.max()
+ mymin = metricData.min()
+
+ mytitle = options['plotTitle']
+
+ if options['plotTitle'] == 'default':
+ mytitle = metricTitle+' model v obs %s to %s' % ( rcmedData['times'][0].strftime(timeFormat),
+ rcmedData['times'][-1].strftime(timeFormat) )
+
+ plots.draw_map_color_filled(metricData, lats, lons, options['plotFilename'],
+ settings['workDir'], mytitle=mytitle,
+ rangeMax=mymax, rangeMin=mymin, diff=True,
+ niceValues=True, nsteps=24)
+
+ ###############################################################################################
+ # 3 dimensional data, e.g. sequence of maps
+ ###############################################################################################
+ if metricData.ndim == 3:
+ print 'Generating series of map plots, each for a different time.'
+ for t in numpy.arange(rcmedData['data'].shape[0]):
+
+ #######################################################################################
+ # Calculate color bar ranges for data such that same range is used in obs and model plots
+ # for like-with-like comparison.
+ #######################################################################################
+ colorRangeMax = max(rcmedData['data'][t, :, :].max(), modelData['data'][t, :, :].max())
+ colorRangeMin = min(rcmedData['data'][t, :, :].min(), modelData['data'][t, :, :].min())
+
+ # Setup the timeTitle
+ timeSlice = times[t]
+ timeTitle = createTimeTitle( options, timeSlice, rcmedData, modelData )
+
+ #######################################################################################
+ # Plot model data
+ #######################################################################################
+ mytitle = 'Model data: mean '+timeTitle
+ plots.draw_map_color_filled(modelData['data'][t, :, :], lats, lons,
+ options['plotFilename']+'model'+str(t),
+ settings['workDir'], mytitle=mytitle,
+ rangeMax=colorRangeMax, rangeMin=colorRangeMin,
+ colorTable=colorbar, niceValues=True)
+
+ #######################################################################################
+ # Plot obs data
+ #######################################################################################
+ mytitle = 'Obs data: mean '+timeTitle
+ plots.draw_map_color_filled(rcmedData['data'][t, :, :], lats, lons,
+ options['plotFilename']+'obs'+str(t),
+ settings['workDir'], mytitle=mytitle,
+ rangeMax=colorRangeMax, rangeMin=colorRangeMin,
+ colorTable=colorbar, niceValues=True)
+
+ #######################################################################################
+ # Plot metric
+ #######################################################################################
+ mytitle = options['plotTitle']
+
+ if options['plotTitle'] == 'default':
+ mytitle = metricTitle +' model v obs : '+timeTitle
+
+ colorRangeMax = metricData.max()
+ colorRangeMin = metricData.min()
+
+ plots.draw_map_color_filled(metricData[t, :, :], lats, lons,
+ options['plotFilename']+str(t), settings['workDir'],
+ mytitle=mytitle, rangeMax=colorRangeMax, rangeMin=colorRangeMin, diff=True,
+ niceValues=True, nsteps=24)
+
+
+def getDataFromRCMED( params, settings, options ):
+ """
+ This function takes in the params, settings, and options dictionaries and will return an rcmedData dictionary.
+
+ return:
+ rcmedData = {"lats": 1-d numpy array of latitudes,
+ "lons": 1-d numpy array of longitudes,
+ "levels": 1-d numpy array of height/pressure levels (surface based data will have length == 1),
+ "times": list of python datetime objects,
+ "data": masked numpy arrays of data values}
+ """
+ rcmedData = {}
+ obsLats, obsLons, obsLevs, obsTimes, obsData = db.extractData(params['obsDatasetId'],
+ params['obsParamId'],
+ params['latMin'],
+ params['latMax'],
+ params['lonMin'],
+ params['lonMax'],
+ params['startTime'],
+ params['endTime'],
+ settings['cacheDir'],
+ options['timeRegrid'])
+ rcmedData['lats'] = obsLats
+ rcmedData['lons'] = obsLons
+ rcmedData['levels'] = obsLevs
+ rcmedData['times'] = obsTimes
+ rcmedData['data'] = obsData
+
+ return rcmedData
+
+def getDataFromModel( model, settings ):
+ """
+ This function takes in the model and settings dictionaries and will return a model data dictionary.
+
+ return:
+ model = {"lats": 1-d numpy array of latitudes,
+ "lons": 1-d numpy array of longitudes,
+ "times": list of python datetime objects,
+ "data": numpy array containing data from all files}
+ """
+ model = files.read_data_from_file_list(settings['fileList'],
+ model['varName'],
+ model['timeVariable'],
+ model['latVariable'],
+ model['lonVariable'])
+ return model
+
+##################################################################################################################
+# Processing complete
+##################################################################################################################
+
+def createTimeTitle( options, timeSlice, rcmedData, modelData ):
+ """
+ Function that takes in the options dictionary and a specific timeSlice.
+
+ Return: string timeTitle properly formatted based on the 'timeRegrid' and 'seasonalCycle' options value.
+
+ Time title labels need their format adjusting depending on the temporal regridding used
+
+ e.g. if data are averaged to monthly, then want to write 'Jan 2002',
+ 'Feb 2002', etc instead of 'Jan 1st, 2002', 'Feb 1st, 2002'
+
+ Also, if doing seasonal cycle compositing then want to write 'Jan','Feb',
+ 'Mar' instead of 'Jan 2002', 'Feb 2002','Mar 2002' etc as data are
+ representative of all Jans, all Febs etc.
+ """
+ if(options['timeRegrid'] == 'daily'):
+ timeTitle = timeSlice.strftime("%b %d, %Y")
+ if options['seasonalCycle'] == True:
+ timeTitle = timeSlice.strftime("%b %d (all years)")
+
+ if(options['timeRegrid'] == 'monthly'):
+ timeTitle = timeSlice.strftime("%b %Y")
+ if options['seasonalCycle'] == True:
+ timeTitle = timeSlice.strftime("%b (all years)")
+
+ if(options['timeRegrid'] == 'annual'):
+ timeTitle = timeSlice.strftime("%Y")
+
+ if(options['timeRegrid'] == 'full'):
+ minTime = min(min(rcmedData['times']), min(modelData['times']))
+ maxTime = max(max(rcmedData['times']), max(modelData['times']))
+ timeTitle = minTime.strftime("%b %d, %Y")+' to '+maxTime.strftime("%b %d, %Y")
+
+ return timeTitle
+
+
Propchange: incubator/climate/branches/rcmet-2.1.1/src/main/python/rcmes/cli/do_rcmes_processing_sub.py
------------------------------------------------------------------------------
svn:executable = *