You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@climate.apache.org by "Mattmann, Chris A (3980)" <ch...@jpl.nasa.gov> on 2014/10/03 20:41:18 UTC
Re: git commit: Add Resources sheet into excel spreadsheet. It is
no longer necessary to have ".rc" file Add -x option in obs4Mips.py in the
"file_template" it is now possible to specify the variable to pass a a
parameter. "trmm_%s.lst, years" where ye...
woot thanks Denis!
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Chris Mattmann, Ph.D.
Chief Architect
Instrument Software and Science Data Systems Section (398)
NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA
Office: 168-519, Mailstop: 168-527
Email: chris.a.mattmann@nasa.gov
WWW: http://sunset.usc.edu/~mattmann/
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Adjunct Associate Professor, Computer Science Department
University of Southern California, Los Angeles, CA 90089 USA
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-----Original Message-----
From: "dnadeau@apache.org" <dn...@apache.org>
Reply-To: "dev@climate.apache.org" <de...@climate.apache.org>
Date: Tuesday, September 30, 2014 at 12:51 PM
To: "commits@climate.apache.org" <co...@climate.apache.org>
Subject: git commit: Add Resources sheet into excel spreadsheet. It is no
longer necessary to have ".rc" file Add -x option in obs4Mips.py in the
"file_template" it is now possible to specify the variable to pass a a
parameter. "trmm_%s.lst, years" where years c
>Repository: climate
>Updated Branches:
> refs/heads/master 8469bcacb -> 7a0a7e2d4
>
>
>Add Resources sheet into excel spreadsheet. It is no longer necessary to
>have ".rc" file
>Add -x option in obs4Mips.py
>in the "file_template" it is now possible to specify the variable to pass
>a a parameter. "trmm_%s.lst,years" where years contain the values to
>pass into the format "%s".
>
>
>Project: http://git-wip-us.apache.org/repos/asf/climate/repo
>Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/7a0a7e2d
>Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/7a0a7e2d
>Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/7a0a7e2d
>
>Branch: refs/heads/master
>Commit: 7a0a7e2d4b52ab7573787425351999f7ab6d90ce
>Parents: 8469bca
>Author: Nadeau <dn...@GSSLA40018857.nccs.nasa.gov>
>Authored: Tue Sep 30 15:43:30 2014 -0400
>Committer: Nadeau <dn...@GSSLA40018857.nccs.nasa.gov>
>Committed: Tue Sep 30 15:43:30 2014 -0400
>
>----------------------------------------------------------------------
> obs4MIPs/INSTALLATION | 3 +
> obs4MIPs/README | 2 +
> obs4MIPs/Toolbox/ESGFexcel.py | 125 +++++++++++++++++++
> .../examples/ECMWF/ecmwf_table_obs4MIPs.xls | Bin 10752 -> 55296
>bytes
> obs4MIPs/obs4MIPs_process.py | 29 +++--
> 5 files changed, 152 insertions(+), 7 deletions(-)
>----------------------------------------------------------------------
>
>
>http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/INST
>ALLATION
>----------------------------------------------------------------------
>diff --git a/obs4MIPs/INSTALLATION b/obs4MIPs/INSTALLATION
>index 082b787..e647d55 100644
>--- a/obs4MIPs/INSTALLATION
>+++ b/obs4MIPs/INSTALLATION
>@@ -54,6 +54,9 @@ http://code.google.com/p/netcdf4-python/
>
> RESSOURCE file documentation
> =============================
>+**********************
>+*** NOTE: The Resource file can now be including in the Excel
>spreadsheet as a new sheet called: Resources. See
>"ecmwf_table_obs4MIPs.xls" for details.
>+**********************
>
> In order to run osb4MIPs, the first thing to do is to create a resource
>file giving information about your original data. The resource files
>contain all Global Attributes and information needed to convert a
>variable to a CMIP5 variable.
>
>
>http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/READ
>ME
>----------------------------------------------------------------------
>diff --git a/obs4MIPs/README b/obs4MIPs/README
>index f9983b8..c0cc05f 100644
>--- a/obs4MIPs/README
>+++ b/obs4MIPs/README
>@@ -12,6 +12,8 @@ obs4MIPs.py -r trmm.rc
>
> Convert trmm data to CMIP5 format using excel spreadsheet
> obs4MIPs.py -r ecmwf.rc
>+or
>+obs4MIPs.py -x ecmwf_table_obs4MIPs.xls
>
> --------------
> Installation
>
>http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/Tool
>box/ESGFexcel.py
>----------------------------------------------------------------------
>diff --git a/obs4MIPs/Toolbox/ESGFexcel.py b/obs4MIPs/Toolbox/ESGFexcel.py
>new file mode 100644
>index 0000000..92fd4df
>--- /dev/null
>+++ b/obs4MIPs/Toolbox/ESGFexcel.py
>@@ -0,0 +1,125 @@
>+# Licensed to the Apache Software Foundation (ASF) under one
>+# or more contributor license agreements. See the NOTICE file
>+# distributed with this work for additional information
>+# regarding copyright ownership. The ASF licenses this file
>+# to you under the Apache License, Version 2.0 (the
>+# "License"); you may not use this file except in compliance
>+# with the License. You may obtain a copy of the License at
>+#
>+# http://www.apache.org/licenses/LICENSE-2.0
>+#
>+# Unless required by applicable law or agreed to in writing,
>+# software distributed under the License is distributed on an
>+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
>+# KIND, either express or implied. See the License for the
>+# specific language governing permissions and limitations
>+# under the License.
>+
>+
>+import pdb
>+import xlrd
>+import shutil
>+import os,sys
>+from numpy import arange
>+from Toolbox.CMORresources import CMORAttributes
>+
>+# ********************************************************************
>+# ESGFexcel()
>+#
>+# ********************************************************************
>+class ESGFexcel:
>+ '''
>+ Create dictionary based on a file using key=value standard from Excel
>+ Sheet resource Key is in Column A, Value is in Column B.
>+ '''
>+ def __init__(self,excelFile):
>+ '''
>+ '''
>+ self.xcl = excelFile
>+ if( os.path.isfile(self.xcl) ):
>+ wb=xlrd.open_workbook(self.xcl)
>+ else:
>+ print "****** Could not find "+self.xcl+" file ****"
>+ print "****** Please check excel file name ****"
>+ raise NameError(self.xcl)
>+
>+ sheet=wb.sheet_by_name('Resources')
>+
>+ for i in arange(sheet.nrows-1) + 1:
>+ key = sheet.row(i)[0]
>+ value = sheet.row(i)[1]
>+
>+ self.resources = dict( [( key, value, )
>+ for( key, value ) in
>+ [ (sheet.row(i)[0].value,
>str(sheet.row(i)[1].value))
>+ for i in arange(sheet.nrows-1) + 1] ] )
>+ pdb.set_trace()
>+ self.ReadXCL()
>+ print self.resources.keys()
>+
>+ def ReadXCL(self):
>+ '''
>+ Read Excel Table and fill rc variable related field.
>+ '''
>+ try:
>+ import xlrd
>+ except:
>+ print "****** Could not find xlrd Python Package ****"
>+ print "****** Please install xlrd package to read excel files
>****"
>+
>+ if( os.path.isfile(self.xcl) ):
>+ wb=xlrd.open_workbook(self.xcl)
>+ else:
>+ print "****** Could not find "+self.xcl+" file ****"
>+ print "****** Please check excel file name ****"
>+ raise NameError(self.xcl)
>+
>+ sheet=wb.sheet_by_name('Variables')
>+
>+ self.resources['cmor_var'] = [ sheet.row( i )[ 0 ].value.\
>+ encode('ascii','ignore') \
>+ for i in arange(sheet.nrows-2)
>+ 2 ]
>+
>+ self.resources['original_var'] = [ sheet.row( i )[ 1 ].value.\
>+ encode('ascii','ignore') \
>+ for i in
>arange(sheet.nrows-2) + 2 ]
>+
>+ self.resources['original_units']=[ sheet.row( i )[ 2 ].value.\
>+ encode('ascii','ignore') \
>+ for i in
>arange(sheet.nrows-2) + 2 ]
>+
>+ self.resources['level'] =[ sheet.row( i )[ 3 ].value.\
>+ encode('ascii','ignore') \
>+ for i in
>arange(sheet.nrows-2) + 2]
>+
>+ self.resources['equation'] =[ sheet.row( i )[ 6 ].value.\
>+ encode('ascii','ignore') \
>+ for i in
>arange(sheet.nrows-2) + 2 ]
>+
>+ #
>-----------------------------------------------------------------
>+ # Make sure it is a string. The main program will call eval on
>it.
>+ #
>-----------------------------------------------------------------
>+ self.resources['cmor_var'] =
>str(self.resources['cmor_var'] )
>+ self.resources['original_var'] =
>str(self.resources['original_var'])
>+ self.resources['original_units']
>=str(self.resources['original_units'])
>+ self.resources['equation'] =str(self.resources['equation'])
>+ self.resources['level'] =str(self.resources['level'])
>+ return 1
>+
>+ def __getitem__(self,key):
>+ '''
>+ Retreive item from resource dictionary
>+ '''
>+ return self.resources[key]
>+
>+ def __setitem__(self,key,value):
>+ '''
>+ '''
>+ self.resources[key]=value
>+
>+ def __delete__(self,key):
>+ '''
>+ '''
>+ del self.resources[key]
>+
>+
>
>http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/exam
>ples/ECMWF/ecmwf_table_obs4MIPs.xls
>----------------------------------------------------------------------
>diff --git a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls
>b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls
>index 271c2a7..d2d12d7 100644
>Binary files a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls and
>b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls differ
>
>http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/obs4
>MIPs_process.py
>----------------------------------------------------------------------
>diff --git a/obs4MIPs/obs4MIPs_process.py b/obs4MIPs/obs4MIPs_process.py
>index bb6d238..3370100 100755
>--- a/obs4MIPs/obs4MIPs_process.py
>+++ b/obs4MIPs/obs4MIPs_process.py
>@@ -26,6 +26,7 @@ import factory
> import numpy
> from factory.formats import import_equation
> from Toolbox.ESGFresources import *
>+from Toolbox.ESGFexcel import *
> from Toolbox.CMORresources import CMORTable
>
>
>@@ -38,11 +39,18 @@ def process( rc ):
> '''
> Convert netcdf/matlab/grads files into CMIP5 format.
> '''
>- #pdb.set_trace()
>+ pdb.set_trace()
> # ----------------------------
> # Loop yearly on file list.
> # ----------------------------
>- for year in rc['years'].split(","):
>+ file_template = rc['file_template'].split(",");
>+ if( len(file_template) == 2 ):
>+ template_parameter = file_template[1]
>+ rc['file_template'] = file_template[0]
>+ else:
>+ template_parameter = 'years'
>+
>+ for year in rc[template_parameter].split(","):
> if(year == ''):
> files= os.popen( "ls " + rc['file_template'] ).readlines()
> else:
>@@ -419,9 +427,10 @@ def usage(message):
> def main():
> '''
> '''
>+ pdb.set_trace()
> try:
>- opts, args = getopt.getopt(sys.argv[1:], "hy:r:",
>- ["help" ,"year=","resource="])
>+ opts, args = getopt.getopt(sys.argv[1:], "hy:r:x:",
>+ ["help"
>,"year=","resource=","excel="])
> except getopt.GetoptError, err:
> usage(str(err))# will print something like "option -a not
>recognized"
> return(2)
>@@ -431,9 +440,12 @@ def main():
> # --------------------------
> year = -1
> resource = None
>+ excel = None
> for o, a in opts:
> if o in ("-r", "--resource"):
> resource = a
>+ elif o in ("-x", "--excel"):
>+ excel = a
> elif o in ("-h", "--help"):
> usage()
> return(0)
>@@ -445,14 +457,17 @@ def main():
> # ------------------------------
> # Does the resource file exist?
> # ------------------------------
>- if( resource == None ) or ( not os.path.isfile( resource ) ) :
>- usage("bad Input Resource File")
>+ if( ((resource == None ) or ( not os.path.isfile( resource ) )) and
>(( excel == None ) or ( not os.path.isfile( excel ) )) ):
>+ usage("bad Input Resource/Excel File")
> return 1
>
> # -----------------------
> # Read in "rc" file
> # -----------------------
>- rc=ESGFresources(resource)
>+ if( resource ):
>+ rc = ESGFresources( resource )
>+ if( excel ):
>+ rc = ESGFexcel( excel )
>
> # --------------------------------
> # Extract CMIP5 Table information
>