You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by jo...@apache.org on 2014/07/01 16:50:02 UTC
[38/56] [partial] gh-pages clean up
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/Toolbox/CMORresources.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/Toolbox/CMORresources.py b/obs4MIPs/Toolbox/CMORresources.py
deleted file mode 100644
index d20f971..0000000
--- a/obs4MIPs/Toolbox/CMORresources.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import netCDF4
-import re
-import pdb
-
-# ********************************************************************
-# CMORTable()
-#
-# ********************************************************************
-class CMORTable:
- '''
- Create dictionary based on a file using key:value standard
- '''
- def __init__( self, inpath, table, entry=None ):
- '''
- Read CMIP5 Table and convert into a dictionary
- '''
- f=open( inpath + '/' + table, 'r')
-
- if( f == None ):
- print "Table file %s does not exist " % (inpath + "/" + table )
-
- lines = f.readlines()
-
- startParse=[0]
- stopParse=[len(lines)]
-
- # --------------------------------
- # extract user specified entry
- # -------------------------------
- if( entry != None ):
- startParse = [ i for i in range(len(lines)) \
- if re.match( ".*entry.*"+entry+".*", lines[i] ) != None ]
- stopParse = [ i for i in range(startParse[0]+2, len(lines)) \
- if re.match( ".*==========.*", lines[i] ) != None ]
-
- self.dico = dict([ tuple(lines[i].split(":"))
- for i in range(startParse[0], stopParse[0])
- if len(lines[i].split(':')) == 2 ] )
-
- # --------------------------------
- # extract levels
- # -------------------------------
- if 'requested' in self.dico.keys():
-
- self.dico['levels'] = self.dico['requested'].split('!')[0].split()
- # --------------
- # Convert to int
- # ---------------
- self.dico['levels'] = [ int( float( self.dico['levels'][i] ) )
- for i in range( len( self.dico['levels'] ) )]
- self.dico['levels'].sort()
-
- def __getitem__( self, key ):
- '''
- Get rid of end of line comments and strip new lines CR "/n"
- '''
- return self.dico[key].split("!")[0].strip()
-
- def __setitem__( self,key,value ):
- '''
- '''
- self.dico[key]=value
-
- def __delete__( self, key ):
- '''
- '''
- del self.dico[key]
-
-
-
-# ********************************************************************
-# Global Attributes
-#
-# Manage Global Attributes in a cmor file
-# ********************************************************************
-class CMORAttributes:
- '''
- Manage Global Attributes.
- '''
- def __init__( self, file ):
- '''
- Open Cmor file
- '''
- self.filename = file
- self.f = netCDF4.Dataset( self.filename, 'r+' )
-
- def GlbDel( self, attribute ):
- '''
- Delete attribute
- '''
- self.f.__delattr__(attribute)
-
- def GlbSet( self, attribute, value):
- '''
- Set attribute
- '''
- self.f.__setattr__(attribute, value)
-
- def VarDel( self, variable,attribute ):
- '''
- Delete Variable attribute
- '''
- var=self.f.variables[variable]
- var.__delatttr__(attribute)
-
- def VarSet( self, variable, attribute, value):
- '''
- Set Variable attribute
- '''
- var=self.variables[variable]
- var.__setattr__(attribute,value)
- def close(self):
- '''
- '''
- self.f.close()
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/Toolbox/ESGFresources.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/Toolbox/ESGFresources.py b/obs4MIPs/Toolbox/ESGFresources.py
deleted file mode 100644
index ddbfe60..0000000
--- a/obs4MIPs/Toolbox/ESGFresources.py
+++ /dev/null
@@ -1,234 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-import pdb
-import shutil
-import os,sys
-from numpy import arange
-from Toolbox.CMORresources import CMORAttributes
-
-# ********************************************************************
-# ESGFresources()
-#
-# ********************************************************************
-class ESGFresources:
- '''
- Create dictionary based on a file using key=value standard
- '''
- def __init__(self,rcFile):
- '''
- Read CMIP5 Table and convert into a dictionary
- strip quote double quote, leading and trailing white space
- from key and value.
- '''
- f=open( rcFile, 'r')
- lines=f.readlines()
- f.close()
-
- self.resources=dict([(key.strip(), value.strip(),)
- for line in lines if line != '\n' \
- and line[0] != "#"
- for (key, value) in
- [line.strip().\
- replace("'","").\
- replace("\"","").\
- rstrip().\
- split("=")]])
-
- for key in self.resources.keys():
- self.resources[key]=self.resources[key].replace("\\","\"")
-
- self.xcl=None
- try:
- self.xcl=self.resources['excel_file']
- rc=self.ReadXCL()
- except:
- pass
-
- def __getitem__(self,key):
- '''
- Retreive item from resource dictionary
- '''
- return self.resources[key]
-
- def __setitem__(self,key,value):
- '''
- '''
- self.resources[key]=value
-
- def __delete__(self,key):
- '''
- '''
- del self.resources[key]
-
- def ReadXCL(self):
- '''
- Read Excel Table and fill rc variable related field.
- '''
- try:
- import xlrd
- except:
- print "****** Could not find xlrd Python Package ****"
- print "****** Please install xlrd package to read excel files ****"
-
- if( os.path.isfile(self.xcl) ):
- wb=xlrd.open_workbook(self.xcl)
- else:
- print "****** Could not find "+self.xcl+" file ****"
- print "****** Please check excel file name ****"
- raise NameError(self.xcl)
-
- sheet=wb.sheet_by_name('Variables')
-
- self.resources['cmor_var'] = [ sheet.row( i )[ 0 ].value.\
- encode('ascii','ignore') \
- for i in arange(sheet.nrows-2) + 2 ]
-
- self.resources['original_var'] = [ sheet.row( i )[ 1 ].value.\
- encode('ascii','ignore') \
- for i in arange(sheet.nrows-2) + 2 ]
-
- self.resources['original_units']=[ sheet.row( i )[ 2 ].value.\
- encode('ascii','ignore') \
- for i in arange(sheet.nrows-2) + 2 ]
-
- self.resources['level'] =[ sheet.row( i )[ 3 ].value.\
- encode('ascii','ignore') \
- for i in arange(sheet.nrows-2) + 2]
-
- self.resources['equation'] =[ sheet.row( i )[ 6 ].value.\
- encode('ascii','ignore') \
- for i in arange(sheet.nrows-2) + 2 ]
-
- # -----------------------------------------------------------------
- # Make sure it is a string. The main program will call eval on it.
- # -----------------------------------------------------------------
- self.resources['cmor_var'] = str(self.resources['cmor_var'] )
- self.resources['original_var'] = str(self.resources['original_var'])
- self.resources['original_units'] =str(self.resources['original_units'])
- self.resources['equation'] =str(self.resources['equation'])
- self.resources['level'] =str(self.resources['level'])
- return 1
-
-
-# ********************************************************************
-# movefiles()
-#
-# rename files in new obs4MIPS path
-# ********************************************************************
-def movefiles(rc):
- '''
- Change CMIP5 directory struture to obs4MIPS directory structure.
- Create new path and rename files following with new convention
- Remove old directory tree.
- '''
- # ----------------------
- # New path for obs4MIPS
- # ----------------------
- path = rc['institute_id' ] + '/' + \
- rc['instrument' ] + '/' + \
- rc['project_id' ] + '/' + \
- rc['product' ] + '/' + \
- rc['modeling_realm' ] + '/' + \
- rc['cvrt_cmor_var' ] + '/' + \
- rc['frequency' ] + '/' + \
- rc['data_structure' ] + '/' + \
- rc['institute_id' ] + '/' + \
- rc['instrument' ]
-
- # -------------------------------------------------------------
- # For TRMM data set version_directory is set to 'false' so that
- # no version will be created in the directory
- # -------------------------------------------------------------
- try:
- rc['version_directory'] == 'false'
- pass
- except:
- path += '/V' + rc['processing_version'] + '/'
-
- # -----------------------------
- # Make sure the directory exist
- # ------------------------------
- try:
- os.makedirs(path)
- except:
- None
-
- # -----------------
- # Rename all files
- # -----------------
- cmorpath = rc['project_id' ] + '/' + \
- rc['product' ] + '/' + \
- rc['institute_id'] + '/'
-
- # -----------------
- # Manage attributes
- # -----------------
- for r,d,f in os.walk(cmorpath):
- for files in f:
- if files.endswith(".nc"):
- filetimestamp = files.split('_')[-1].strip(".nc")
- file = os.path.join(r,files)
- print file
- # -----------------
- # Delete attributes
- # ------------------
- Attr=CMORAttributes( file )
- DelGlbAttributes=eval(rc['DelGlbAttributes'].\
- replace('\\','\''))
- for attribute in DelGlbAttributes:
- print "Deleting attribute: %s" % attribute
- Attr.GlbDel(attribute)
- # -----------------
- # set attributes
- # ------------------
- SetGlbAttributes=eval(rc['SetGlbAttributes'].\
- replace('\\','\''))
- for (attribute,Value) in SetGlbAttributes:
- print "Assigning attribute (%s,%s)" % (attribute,Value)
- Attr.GlbSet(attribute,Value)
- Attr.close()
-
- source = ''
- if( rc['source_fn'] == 'SYNOPTIC' ):
- source = rc['SYNOPTIC']+"z_"
- elif( rc['source_fn'] != '' ):
- source = rc['source_fn'].split('_')[0] + '_'
-
- newfilename = rc['cvrt_cmor_var' ] + '_' + \
- rc['instrument' ] + '-' + \
- source + \
- rc['processing_level' ] + '_v'+ \
- rc['processing_version'] + '_' + \
- filetimestamp + '.nc'
-
- newfilename = os.path.join(path,newfilename)
- # -----------
- # Move files
- # -----------
- print file
- print newfilename
- os.rename(file,newfilename)
-
-
- # ----------------
- # Remove cmor path
- # ----------------
- shutil.rmtree( rc['project_id'] )
- return 0
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/Toolbox/__init__.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/Toolbox/__init__.py b/obs4MIPs/Toolbox/__init__.py
deleted file mode 100644
index f1d6f14..0000000
--- a/obs4MIPs/Toolbox/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-__all__ = ['ESGFresources','CMORresources']
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/ECMWF/ECMWFobs4MIPs.rc
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/ECMWF/ECMWFobs4MIPs.rc b/obs4MIPs/examples/ECMWF/ECMWFobs4MIPs.rc
deleted file mode 100644
index a7aece6..0000000
--- a/obs4MIPs/examples/ECMWF/ECMWFobs4MIPs.rc
+++ /dev/null
@@ -1,45 +0,0 @@
-#
-# ECMWF resource file for CMIP5 conversion
-#
-# source_fn = 'SYNOPTIC' will be replaced by 00z or 12z for ECMWF ERA-iterim files
-#
-# Program will loop on years and substiute in template.
-#
-years='2013'
-file_template = "Y{0}/ecint_prs_{0}.nc"
-
-experiment_id = 'obs'
-institution = 'European Centre for Medium-Range Weather Forecasts'
-calendar = 'gregorian'
-institute_id = 'ECMWF'
-model_id = 'Obs-IFS-Cy31r2'
-source = 'ERA Interim, Synoptic Monthly Means, Full Resolution'
-contact = "ECMWF: Dick Dee (dick.dee@ecmwf.int)"
-references = 'http://www.ecmwf.int'
-mip_specs = 'CMIP5'
-data_structure = 'grid'
-source_type = 'reanalysis'
-# used for filename (up to _)
-source_fn = 'ECMWF'
-source_id = 'ERA-interim'
-realm = 'atmos'
-inpath = "Tables"
-table = 'CMIP5_Amon_obs'
-excel_file = 'ecmwf_table_obs4MIPs.xls'
-
-# ------------------------------------
-# Used to create direcgtory structure
-# ------------------------------------
-obs_project = 'obs4MIPs'
-instrument = 'assimilation'
-processing_version = '1.0'
-processing_level = 'level-4'
-
-# --------------------------------------
-# Post-processing
-# --------------------------------------
-DelGlbAttributes = "[\realization\,\experiment\,\physics_version\,\initialization_method\,\experiment_id\,\modeling_realm\,\instrument\,\processing_version\,\processing_level\,\obs_project\]"
-SetGlbAttributes = "[(\title\,\ECMWF Reanalysis output prepared for obs4MIPs \)]"
-InputTimeUnits = "internal"
-OutputTimeUnits = "months since 1900-01-01"
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/ECMWF/Tables
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/ECMWF/Tables b/obs4MIPs/examples/ECMWF/Tables
deleted file mode 120000
index 561a38f..0000000
--- a/obs4MIPs/examples/ECMWF/Tables
+++ /dev/null
@@ -1 +0,0 @@
-../../Tables
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/ECMWF/Y2013/README
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/ECMWF/Y2013/README b/obs4MIPs/examples/ECMWF/Y2013/README
deleted file mode 100644
index 4cf903b..0000000
--- a/obs4MIPs/examples/ECMWF/Y2013/README
+++ /dev/null
@@ -1,49 +0,0 @@
-Prerequisite
-============
-
-Read ECMWF python script documentation from the following link.
-https://software.ecmwf.int/wiki/display/WEBAPI/Accessing+ECMWF+data+servers+in+batch
-
--
-Register to the ECMWF website
-http://apps.ecmwf.int/registration/
--
-
--
-Retrieve your key
-https://api.ecmwf.int/v1/key/
--
-
--
-Create a file in $HOME/.ecmwfapirc containning your key.
- Content of $HOME/.ecmwfapirc
- {
- "url" : "http://api.ecmwf.int/v1",
- "key" : "MYKEY"
- "email" : "MYEMAIL"
- }
--
-=== Install Python package ====
-
-setenv PYTHONPATH /wherever/you/want/to/install/python/packages
--
-Dowload and install python ECMWF API on your system.
-https://software.ecmwf.int/wiki/download/attachments/23694554/ecmwf-api-client-python.tgz?api=v2
-
- wget "https://software.ecmwf.int/wiki/download/attachments/23694554/ecmwf-api-client-python.tgz?api=v2" --output-document=ecmwf-api-client-python.tgz
--
-
-
--
-setenv PYTHONPATH /wherever/you/want/to/install/python/packages
-python setup.py build
-python setup.py install --prefix=/wherever/you/want/to/install/python/packages
-
--
-
-Please run the following script to pull the example file from ECMWF repository
--
-python verification.py
--
-
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/ECMWF/Y2013/verification.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/ECMWF/Y2013/verification.py b/obs4MIPs/examples/ECMWF/Y2013/verification.py
deleted file mode 100755
index a573c55..0000000
--- a/obs4MIPs/examples/ECMWF/Y2013/verification.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-
-from ecmwfapi import ECMWFDataServer
-
-# To run this example, you need an API key
-# available from https://api.ecmwf.int/v1/key/
-
-server = ECMWFDataServer()
-
-server.retrieve({
-"levelist" : "1/2/3/5/7/10/20/30/50/70/100/125/150/175/200/225/250/300/350/400/450/500/550/600/650/700/750/775/800/825/850/875/900/925/950/975/1000",
-"stream" : "moda",
-"levtype" : "pl",
-"param" : "129.128/130.128/131.128/132.128/133.128/135.128/138.128/155.128/157.128/203.128/246.128/247.128/248.128/60.128",
-"dataset" : "interim",
-"grid" : "0.75/0.75",
-"date" : "20130101/20130201/20130301/20130401/20130501/20130601/20130701/20130801/20130901/20131001/20131101/20131201",
-"target" : "ecint_prs_2013.nc",
-"class" : "ei",
-"format" : "netcdf",
-"type" : "an"
-})
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls
deleted file mode 100644
index 271c2a7..0000000
Binary files a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls and /dev/null differ
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/TRMM/TRMM.rc
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/TRMM/TRMM.rc b/obs4MIPs/examples/TRMM/TRMM.rc
deleted file mode 100644
index 4caa27d..0000000
--- a/obs4MIPs/examples/TRMM/TRMM.rc
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# TRMM resource file for CMIP5 conversion
-#
-
-# Program will loop on years and substiute in template.
-# Here we are creating decadal files using a grads ctl file
-#
-years=1998
-file_template = "v7/TRMM_%s.lst"
-
-experiment_id = 'obs'
-institution = 'NASA Goddard Space Flight Center, Greenbelt MD, USA'
-calendar = 'gregorian'
-institute_id = 'NASA-GSFC'
-model_id = 'Obs-TRMM'
-source = 'Global Precipitation Climatology Project (TRMM)'
-contact = "George Huffman george.j.huffman@nasa.gov"
-references = 'http://science.nasa.gov/missions/trmm/'
-instrument = 'TRMM'
-processing_version = '7'
-processing_level = 'L3'
-mip_specs = 'CMIP5'
-data_structure = 'grid'
-source_type = 'satellite_retrieval_and_gauge_analysis'
-# used for filename (up to _)
-source_fn = ''
-source_id = 'TRMM'
-realm = 'atmos'
-obs_project = 'TRMM'
-inpath = "Tables"
-table = 'CMIP5_Amon_obs'
-cmor_var = '[\pr\]'
-level = '[\\]'
-equation = '[\(data*2.78e-4)\]'
-inpath = 'Tables'
-original_var = '[\pcp\]'
-original_units = '[\kg m-2 s-1\]'
-OutputTimeUnits = "months since 1900-1-1"
-InputTimeUnits = "internal"
-SetGlbAttributes = "[(\global\,rc[\product\]),(\processing_version\,rc[\processing_version\]),(\title\,\obs-TRMM output prepared for obs4MIPs NASA-GSFC observations\)]"
-DelGlbAttributes = "[\realization\,\experiment\,\physics_version\,\initialization_method\]"
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/TRMM/Tables
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/TRMM/Tables b/obs4MIPs/examples/TRMM/Tables
deleted file mode 120000
index 4c2c9a9..0000000
--- a/obs4MIPs/examples/TRMM/Tables
+++ /dev/null
@@ -1 +0,0 @@
-../../Tables/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/TRMM/v7/README
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/TRMM/v7/README b/obs4MIPs/examples/TRMM/v7/README
deleted file mode 100644
index a54e428..0000000
--- a/obs4MIPs/examples/TRMM/v7/README
+++ /dev/null
@@ -1,12 +0,0 @@
-
-Please make sure that the wget version is at least 1.11.1; run wget -V to find the version.
-(You can download version 1.11.1 of wget here.)
-
-
-List of files to download at the GES DISC are found in TRMM.data
-
-Here is the proper wget command.
-wget --content-disposition -i TRMM.data
-
-
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/TRMM/v7/TRMM.data
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/TRMM/v7/TRMM.data b/obs4MIPs/examples/TRMM/v7/TRMM.data
deleted file mode 100644
index 0fc1619..0000000
--- a/obs4MIPs/examples/TRMM/v7/TRMM.data
+++ /dev/null
@@ -1,12 +0,0 @@
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F335%2F3B43.19981201.7.HDF&LABEL=3B43.19981201.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F305%2F3B43.19981101.7.HDF&LABEL=3B43.19981101.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F274%2F3B43.19981001.7.HDF&LABEL=3B43.19981001.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F244%2F3B43.19980901.7.HDF&LABEL=3B43.19980901.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F213%2F3B43.19980801.7.HDF&LABEL=3B43.19980801.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F182%2F3B43.19980701.7.HDF&LABEL=3B43.19980701.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F152%2F3B43.19980601.7.HDF&LABEL=3B43.19980601.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F121%2F3B43.19980501.7.HDF&LABEL=3B43.19980501.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F091%2F3B43.19980401.7.HDF&LABEL=3B43.19980401.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F060%2F3B43.19980301.7.HDF&LABEL=3B43.19980301.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F032%2F3B43.19980201.7.HDF&LABEL=3B43.19980201.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
-http://disc2.nascom.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fftp%2Fdata%2Fs4pa%2FTRMM_L3%2FTRMM_3B43%2F1998%2F001%2F3B43.19980101.7.HDF&LABEL=3B43.19980101.7.nc&SHORTNAME=TRMM_3B43&SERVICE=HDF_TO_NetCDF&VERSION=1.02&DATASET_VERSION=007
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/examples/TRMM/v7/TRMM_1998.lst
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/TRMM/v7/TRMM_1998.lst b/obs4MIPs/examples/TRMM/v7/TRMM_1998.lst
deleted file mode 100644
index d41c953..0000000
--- a/obs4MIPs/examples/TRMM/v7/TRMM_1998.lst
+++ /dev/null
@@ -1,12 +0,0 @@
-v7/3B43.19980101.7.nc
-v7/3B43.19980201.7.nc
-v7/3B43.19980301.7.nc
-v7/3B43.19980401.7.nc
-v7/3B43.19980501.7.nc
-v7/3B43.19980601.7.nc
-v7/3B43.19980701.7.nc
-v7/3B43.19980801.7.nc
-v7/3B43.19980901.7.nc
-v7/3B43.19981001.7.nc
-v7/3B43.19981101.7.nc
-v7/3B43.19981201.7.nc
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/factory/__init__.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/factory/__init__.py b/obs4MIPs/factory/__init__.py
deleted file mode 100644
index 15755be..0000000
--- a/obs4MIPs/factory/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from formats import *
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/factory/equations.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/factory/equations.py b/obs4MIPs/factory/equations.py
deleted file mode 100644
index 6f31b3c..0000000
--- a/obs4MIPs/factory/equations.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# from: http://www.eol.ucar.edu/projects/ceop/dm/documents/refdata_report/eqns.html
-from numpy import exp
-import pdb
-
-def CtoF(tf):
- return ( tc * 9. / 5. ) + 32.0
-
-def CtoK(tc):
- return tc + 273.15
-
-def FtoK(tf):
- return ( (tf - 32) * 5.0 / 9.0) + 273.15
-
-def FtoC(tf):
- return (tf - 32.0) * 5.0 / 9.0
-
-def KtoC(tk):
- return (tk - 273.15 )
-
-def KtoF(tk):
- return ( ( tk - 273.15 ) * 9.0 / 5.0 ) + 32.0
-
-def ComputeES(tc):
- '''
- Tc: Temperature in deg Celsius
- es: Saturation vapor pressure in mbar
- '''
- es = 6.112 * exp( ( 17.67 * tc ) / ( tc + 243.5 ) );
-
- return es
-
-def ComputeE(td):
- '''
- Td: dew point in deg Celcius
- e: vapor pressure in mbar
- '''
- e = 6.112 * exp( ( 17.67 * td) / ( td + 243.5 ) );
- return e
-
-def ComputeQ(h):
- '''
- td: in celsius
- p: in mbar
- '''
-
- sp = h.getData( variable='sp' )
- sp = sp/100 # from Pa to Mb
-
- d2m = h.getData( variable='d2m' )
- d2m = KtoC(d2m)
-
- e = ComputeE(d2m)
- q = ( 0.622 * e ) / ( sp - ( 0.378 * e ) );
- return q
-
-
-def ComputeRH( h ):
- '''
- f is the netCDF file handle containing t2m and d2m in the same file
- '''
-
- t2m = h.getData( variable='t2m' )
- t2m = KtoC(t2m)
-
- d2m = h.getData( variable='d2m' )
- d2m = KtoC(d2m)
-
- es = ComputeES(t2m)
- e = ComputeE(d2m)
- RH = 100 * (e /es )
- return RH
-
-def ComputeJRA25RH( h ):
- '''
- f is the netCDF file handle containing and TMPprs in DEPRprs the same file
- '''
- #pdb.set_trace()
- ta = h.getData( variable='tmpprs' )
- ta = KtoC(ta)
-
- da = h.getData( variable='deprprs' )
- da = KtoC(da)
-
- es = ComputeES(ta)
- e = ComputeE(da)
- RH = 100 * (e /es )
- return RH
-
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/factory/formats.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/factory/formats.py b/obs4MIPs/factory/formats.py
deleted file mode 100644
index ced7d74..0000000
--- a/obs4MIPs/factory/formats.py
+++ /dev/null
@@ -1,577 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Class factory
-#
-# Select the right handler depending on the file format
-
-import os
-import pdb
-import sys
-import magic
-import scipy.io
-import cdms2
-import cdtime
-import numpy.ma as ma
-import numpy
-import netCDF4
-import re
-
-from cdms2.grid import createUniformGrid
-
-# ********************************************************************
-# import_equation()
-#
-# provide equation dynamically
-# ********************************************************************
-def import_equation(eq):
- d = eq.rfind(".")
- EqName = eq[ d+1:len(eq) ]
- m = __import__( eq[0:d], globals( ), locals( ), [ EqName ] )
- return getattr( m, EqName )
-
-# ********************************************************************
-# HandlerGrads()
-#
-# Manage Grads file io
-# ********************************************************************
-class HandlerGrads(object):
- '''
- '''
- def __new__(klass, *args, **kwargs):
- # ------------------------------------------------------------------
- # This return make sure __init__ is begin called after the class is
- # created
- # ------------------------------------------------------------------
- return super(HandlerGrads, klass).__new__(klass, *args, **kwargs)
-
- def __init__(self):
- pass
-
- def open(self, file, variable=None):
- '''
- Open ctl file and read data
- '''
- self.f=cdms2.open(file)
- self.variable=variable
- self.data=self.f(variable) # read variable
-
-
- def getData(self, variable=None):
- '''
- Return data array
- '''
- if( variable != None ):
- data=self.f(variable)
- else:
- data=self.data
-
- return data
-
- def getLongitude(self):
- '''
- Return Longitude.
- '''
- return self.data.getLongitude()
-
- def getLatitude(self):
- '''
- Return Latitude.
- '''
- return self.data.getLatitude()
-
- def getTime(self,timeunits=None):
- '''
- Return time in cdtime format.
- '''
- if(timeunits == 'internal'):
- timeunits=self.getTimeUnits()
-
- time_values = numpy.arange(len(self.data.getTime()))
- time_bounds = numpy.arange(len(self.data.getTime())+1)
- cur_time=[cdtime.reltime(i,timeunits)
- for i in time_values]
-
- return cur_time
-
- def getTimeUnits(self,timeunits):
- '''
- return Time units (month since yyyy)
- '''
- if( timeunits == 'internal' ):
- return self.data.getTime().units
- return timeunits
-
-
-# ********************************************************************
-# HandlerNetCDF()
-#
-# Manage netCDF file io. Note, this API reads HDF4 and Netcdf4/HDF5
-# ********************************************************************
-class HandlerNetCDF(object):
- '''
- '''
- def __new__(klass, *args, **kwargs):
- # ------------------------------------------------------------------
- # This return make sure __init__ is begin called after the class is
- # created
- # ------------------------------------------------------------------
- return super(HandlerNetCDF, klass).__new__(klass, *args, **kwargs)
-
- def __init__(self):
- pass
-
- def open(self, file, variable=None):
- '''
- Open ctl file and read data
- '''
- self.f=cdms2.open(file)
- self.data=self.f(variable).astype('float32') # read variable
-
-
- def getData(self):
- '''
- Return data array
- '''
- return self.data
-
- def getLongitude(self):
- '''
- Return Longitude.
- '''
- return self.data.getLongitude()
-
- def getLatitude(self):
- '''
- Return Latitude.
- '''
- return self.data.getLatitude()
-
- def getTime(self,timeunits=None):
- '''
- Return time.
- '''
- if( timeunits == 'internal' ):
- timeunits=self.getTimeUnits()
-
- time_values = self.f.getAxis("time")[:]
-
- deltime = time_values[ 1 ] - time_values[ 0 ]
- time_bounds = time_values
- numpy.append(time_bounds, time_values[-1] + deltime )
-
- cur_time=[ cdtime.reltime( i, timeunits )
- for i in time_values ]
-
- return cur_time
- def getTimeUnits(self,timeunits):
- '''
- return Time units (month since yyyy)
- '''
- if( timeunits == 'internal' ):
- return self.data.getTime().units
- return timeunits
-
-
-# ********************************************************************
-# HandlerNCAggregate()
-#
-# Aggreate a list of NetCDF files from a text file
-# Each file must have the same lat/lon dimension
-# ********************************************************************
-class HandlerNCAggregate(object):
- '''
- '''
- def __new__(klass, *args, **kwargs):
- # ------------------------------------------------------------------
- # This return make sure __init__ is begin called after the class is
- # created
- # ------------------------------------------------------------------
- return super(HandlerNCAggregate, klass).__new__(klass, *args, **kwargs)
-
- def __init__(self):
- '''
- Nothing to do
- '''
- pass
-
- def open(self, file, variable=None):
- '''
- Make sure that all files in the list exist
- '''
- f = open(file,'r')
- self.flist = f.readlines()
- f.close()
-
- for file in self.flist:
- filename=file.strip()
- if( not os.path.exists(filename) ):
- print "File %s does not exist in filelist" % filename
-
- # --------------------------------------------------------
- # Extract General information from first file in the list
- # ---------------------------------------------------------
- firstfile = cdms2.open(self.flist[0].strip(),'r')
- self.variable = firstfile(variable)
- self.missing_value = self.variable.missing_value
-
- # ----------------------------
- # Read first file in the list
- # -----------------------------
- self.timeunits = self.variable.getTime().units
-
- def getData(self, variable=None):
- '''
- Aggreate all data from the file list in the order given.
- '''
- filename = self.flist[0].strip()
- f = cdms2.open( filename, 'r' )
-
- if( variable != None ):
- self.vartoread = variable
- else:
- self.vartoread = self.variable.id
-
- data = f(self.vartoread)[:]
-
- # ---------------------------
- # Concatenate following files
- # ---------------------------
- for filename in self.flist[ 1: ]:
- print "reading %s" % filename.strip()
- f = cdms2.open( filename.strip(), 'r' )
- data2 = f(self.vartoread)[:]
- data = numpy.concatenate((data,data2), axis=0)
- f.close()
-
- data=ma.array(data=data, \
- fill_value=self.missing_value, \
- copy=0, \
- dtype='float32' )
- data=ma.masked_equal(data, self.missing_value, copy=0)
- return data
-
-
- def getLongitude(self):
- '''
- Assume same grid for all files.
- '''
- return self.variable.getLongitude()
-
-
- def getLatitude(self):
- '''
- Assume same grid for all files.
- '''
- return self.variable.getLatitude()
-
-
- def getTime(self,timeunits=None):
- '''
- Return time.
- Assume that file time variable name will be the same in all files.
- '''
- flist=self.flist
- nbFiles = len(self.flist)
- # ----------------------------
- # Read first file in the list
- # -----------------------------
- time = self.variable.getTime()[:]
-
- # ---------------------------
- # Concatenate following files
- # ---------------------------
- for filename in self.flist[ 1: ]:
- f = cdms2.open( filename.strip(), 'r' )
- time2 = f(self.variable.id).getTime()
- timeunits2 = f(self.variable.id).getTime().units
- match=re.match('.*since.*', timeunits2)
- # ---------------------------------------------------
- # if timeunits is not UDUNITS format, pass back value
- # and let InputTimeUnits attribute to take care of
- # the conversion.
- # ----------------------------------------------------
- if( not match ): timeunits2=self.timeunits
- # --------------------------------
- # Make sure we have the same units
- # ---------------------------------
- if( timeunits2 != self.timeunits ):
- file_time = [ cdtime.reltime(time2[ i ], timeunits2 ) \
- for i in range( len( time2 ) )]
- time2 = [ file_time[i].torel( timeunits ).value \
- for i in range( len( time2 ) ) ]
-
- time2=time2[:]
- time = numpy.concatenate((time,time2), axis=0)
- f.close()
-
- self.time = [cdtime.reltime(i,self.timeunits) for i in time]
- return self.time
-
-
- def getTimeUnits(self,timeunits=None):
- '''
- return Timeunits.
- '''
- # ---------------------------------------------------
- # If file timeunits are UDUNITS compliante, overwrite
- # timeunits argument
- # ---------------------------------------------------
- match = re.match('.*since.*',self.variable.getTime().units)
- if( match ):
- return self.variable.getTime().units
- self.timeunits=timeunits
- return timeunits
-
-
-# ********************************************************************
-# MatlabData()
-#
-# Manage Matlab data object.
-#
-# Note: Assume that data is in the key that is not internal to Matlab.
-# Matlab internal keys are surrounded by underscores (__key__).
-# ********************************************************************
-class MatlabData(object):
- '''
- Convert Matlab data Array to cmor2 data Array.
- Create lat/lon grid
- '''
-
-
- def __init__(self,oMatlab):
- '''
- Read data and flip y axis.
- Assume data is -180 to 180 and will be shifted to get an array from
- 0 to 360 degrees.
-
- '''
- DataKey = [key for key in oMatlab.keys()
- if key not in ['__version__', '__header__', '__globals__']]
-
- # ---------------------------------------
- # Read in Data as a Numpy masked array.
- # ---------------------------------------
- self.data=ma.array(data=oMatlab[DataKey[0]], \
- fill_value=0.0, \
- copy=0, \
- dtype='float32' )
-
- # ----------------------------------------------
- # Assume fill value is 0 (Need to pass a value)
- # -----------------------------------------------
- self.data=ma.masked_values(self.data,0)
-
- # -----------------------------------------------
- # Transpose 3D matrix to get the time axis first
- # -----------------------------------------------
- self.data=self.data.transpose(2,0,1)
-
- # -------------
- # Flip y axis
- # -------------
- self.data=self.data[:,::-1,:]
-
- # --------------------------------------------------
- # Extract X lenght and Y length to compute Lat/Lon
- # --------------------------------------------------
- nlat = self.data.shape[1]
- nlon = self.data.shape[2]
-
- # --------------------------------------------------------------------
- # shift array from -180 to 180 degrees to 0 to 360 degrees
- # The last half of the array becomes the first half and the first half
- # becomes the last half
- # --------------------------------------------------------------------
- flipped=ma.array( numpy.zeros( self.data.shape ), \
- fill_value=0.0, \
- dtype='float32' )
-
- # -----------------------------------------
- # Masked fill_value (default to 0 for now)
- # -----------------------------------------
- flipped=ma.masked_values(flipped,0)
-
- flipped[:,:,-nlon/2:]=self.data[:,:,:nlon/2]
- flipped[:,:,:nlon/2] =self.data[:,:,-nlon/2:]
-
- # ---------------------------------
- # New flipped array becomes the data.
- # ----------------------------------
- self.data=flipped
- flipped=0 # In order to save memory
-
- # -------------------------------------------------------
- # Compute latitudes longitude from X length and Y Length
- # Assume Global data on cylindrical rectangular grid.
- # -------------------------------------------------------
- deltaLat = 180.0/nlat
- deltaLon = 360.0/nlon
- startLat = -90 + (deltaLat/2.0)
- startLon = 0 + (deltaLon/2.0)
-
- # -------------------------------
- # Used cdms2 API to create grid
- # --------------------------------
- self.grid= createUniformGrid( startLat, nlat, deltaLat, \
- startLon, nlon, deltaLon)
-
- def getData(self):
- '''
- return data array
- '''
- return self.data
-
- def getLatitude(self):
- '''
- return lattitude array
- '''
- return self.grid.getLatitude()
-
- def getLongitude(self):
- '''
- return longitude array
- '''
- return self.grid.getLongitude()
-
- def getTime( self, cur_timeunits ):
- '''
- '''
- nTime = self.data.shape[0]
- time_values = numpy.arange(nTime)
- fileTime =[ cdtime.reltime(i, cur_timeunits)
- for i in time_values ]
-
- return fileTime
-
- def getTimeUnits(self,timeunits):
- '''
- return Time units (month since yyyy)
- '''
- return timeunits
-
-
- def __getitem__(self, key):
- '''
- '''
- return self.getData()
-
-# ********************************************************************
-# HandlerMatlab()
-#
-# Manage Matlab data file.
-#
-# Note: Assume that data is in the key that is not internal to Matlab.
-# ********************************************************************
-class HandlerMatlab(object):
- '''
- '''
- def __new__(klass, *args, **kwargs):
- # ------------------------------------------------------------------
- # This return make sure __init__ is begin called after the class is
- # created
- # ------------------------------------------------------------------
- return super(HandlerMatlab, klass).__new__(klass, *args, **kwargs)
-
- def __init__(self):
- pass
-
- def open(self, file, variable=None):
- '''
- Open Matlab and read data
- '''
- self.f=scipy.io.loadmat(file)
- self.oMatlab=MatlabData(self.f)
-
-
- def getData(self):
- '''
- Return data array
- '''
- return self.oMatlab.getData()
-
- def getLongitude(self):
- '''
- Return Longitude.
- '''
- return self.oMatlab.getLongitude()
-
- def getLatitude(self):
- '''
- Return Latitude.
- '''
- return self.oMatlab.getLatitude()
-
- def getTime(self,timeunits):
- '''
- Return time.
- '''
- return self.oMatlab.getTime(timeunits)
-
- def getTimeUnits(self,timeunits):
- '''
- return Time units (month since yyyy)
- '''
- return self.oMatlab.getTimeUnits(timeunits)
-
-
-
-# ********************************************************************
-# HandlerFormats()
-#
-# Factory detecting file format and return a pointer to the related
-# format class
-#
-# Reference:
-# http://code.activestate.com/recipes/576687/
-#
-# ********************************************************************
-class HandlerFormats(object):
- '''
- Rerturn Format Hanlder depending on file magic number.
- '''
- Formats= {'NetCDF': HandlerNetCDF,
- 'Hierar': HandlerNetCDF,
- 'NCList': HandlerNCAggregate,
- 'DSET' : HandlerGrads,
- 'Matlab': HandlerMatlab}
-
- def __new__( klass, filename ):
- MagicNumber = magic.from_file( filename )
- try:
- return HandlerFormats.Formats[MagicNumber[0:6]]()
- except:
- # --------------------------------------------------------------------------------
- # Magic is too relax and come backup with 'Bio-Rad .PIC Image File'
- # Since we will never read these files in the program, I just assume this will be
- # a list of files.
- # --------------------------------------------------------------------------------
- if((MagicNumber[0:5] == 'ASCII') or (MagicNumber[0:5] == "Bio-R")):
- f=open(filename,'r')
- lines=f.readlines()
- for line in lines:
- if(line[0:4].upper() == 'DSET'):
- return HandlerFormats.Formats['DSET']()
- elif( line.strip().endswith("nc") ):
- return HandlerFormats.Formats['NCList']()
-
- def __init__(self, filename ):
- pass
-
-
-
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/obs4MIPs_process.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/obs4MIPs_process.py b/obs4MIPs/obs4MIPs_process.py
deleted file mode 100755
index bb6d238..0000000
--- a/obs4MIPs/obs4MIPs_process.py
+++ /dev/null
@@ -1,480 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import cdms2
-import cdtime
-import cmor
-import sys
-import getopt
-import factory
-import numpy
-from factory.formats import import_equation
-from Toolbox.ESGFresources import *
-from Toolbox.CMORresources import CMORTable
-
-
-# ************************************************************************
-# process()
-#
-# Convert to obs4MIPS file format.
-# ************************************************************************
-def process( rc ):
- '''
- Convert netcdf/matlab/grads files into CMIP5 format.
- '''
- #pdb.set_trace()
- # ----------------------------
- # Loop yearly on file list.
- # ----------------------------
- for year in rc['years'].split(","):
- if(year == ''):
- files= os.popen( "ls " + rc['file_template'] ).readlines()
- else:
- # ------------------------------------------------
- # Use string formating for path with same argument
- # ------------------------------------------------
- try:
- tmplFile = rc['file_template'] % (year)
- except:
- tmplFile = rc['file_template'].format(year)
- if( not os.path.isfile( tmplFile) ) :
- print "**** Warning %s not found\n" % ( tmplFile )
- continue
- files= os.popen( "ls " + tmplFile).readlines()
-
- if( files == [] ):
- print "No file found: Check your resource file"
- return -1
- # ------------------------------------------------
- # Get the right handler to manage this file format
- # ------------------------------------------------
- Handler = factory.HandlerFormats(files[0].strip())
-
- # -----------------------------------
- # Take care of cmor initialization.
- # -----------------------------------
- cmor.setup(inpath=rc['inpath'],
- netcdf_file_action = cmor.CMOR_REPLACE)
-
- cmor.dataset(experiment_id = rc['experiment_id'],
- institution = rc['institution' ],
- calendar = rc['calendar' ],
- institute_id = rc['institute_id' ],
- model_id = rc['model_id' ],
- source = rc['source' ],
- contact = rc['contact' ],
- references = rc['references' ])
-
-
- # -----------------------------------------
- # add extra Global Attributes for obs4MIPs.
- # -----------------------------------------
- cmor.set_cur_dataset_attribute( 'instrument', rc['instrument' ])
- cmor.set_cur_dataset_attribute( 'mip_specs', rc['mip_specs' ])
- cmor.set_cur_dataset_attribute( 'data_structure', rc['data_structure'])
- cmor.set_cur_dataset_attribute( 'source_type', rc['source_type' ])
- cmor.set_cur_dataset_attribute( 'source_id', rc['source_id' ])
- cmor.set_cur_dataset_attribute( 'realm', rc['realm' ])
- cmor.set_cur_dataset_attribute( 'obs_project', rc['obs_project' ])
- cmor.set_cur_dataset_attribute( 'processing_version',
- rc['processing_version'] )
- cmor.set_cur_dataset_attribute( 'processing_level',
- rc['processing_level'] )
-
- cmor.load_table(rc['table'])
-
- # ---------------------------------------------------------------------
- # We loop on each file found, a new cmor file will be create on each
- # iteration. If you want to aggregate, you need to use Grads ctl file
- # or NeCDF list of file.
- # ---------------------------------------------------------------------
- for file in files:
-
- fnm=file.strip() # Get rid of \n
- aVariable = eval(rc['original_var'])
- nbVariable = len(aVariable)
-
- # -----------------------------------------------------
- # ECMWF needs synoptic time 00z and 12z in he filename.
- # We extract it from the first file in the list.
- # -----------------------------------------------------
- if( rc['source_fn'] == 'SYNOPTIC' ):
- index = fnm.find('z.')
- rc['SYNOPTIC'] = fnm[index-2:index]
-
- # -----------------------
- # Work on all variables
- # -------------------------
- for j in arange(nbVariable):
- # ----------------------------------------------------
- # Fetch the variable directly or excute equation.
- # ----------------------------------------------------
- try:
- variable=aVariable[j]
- Handler.open(fnm, variable=variable)
- rc['cvrt_original_var'] = aVariable[j]
- print "Working on variable %s " % variable
- except:
- if( aVariable[j] != 'equation' ) :
- print "Variable %s can't open" % variable
- continue
- else:
- print "Executing %s " % eval(rc['equation'])[j]
-
-# pdb.set_trace()
- rc['cvrt_original_units'] = eval(rc['original_units'])[j]
- rc['cvrt_cmor_var'] = eval(rc['cmor_var'])[j]
- rc['cvrt_equation'] = eval(rc['equation'])[j]
- rc['cvrt_level'] = eval(rc['level'])[j]
-
- data=Handler.getData()
-
- # ----------------------------------------------------------
- # Evaluate equation if needed. Usually used to change units
- # ----------------------------------------------------------
- if( rc['cvrt_equation'][0] == '@' ):
- fncName = rc['cvrt_equation'][1:]
- fnc = import_equation( "equations.%s" % fncName )
- data[:]= fnc(Handler)
- else:
- data[:]=eval(rc['cvrt_equation'])
-
- # -------------------------------------------------------------
- # Save filled value in case data type is changed in createAxes
- # -------------------------------------------------------------
- fill_value = data.fill_value
-
- # ---------------------------------------------
- # Extract latitude/longitude
- # ---------------------------------------------
- lonvals=Handler.getLongitude()
- latvals=Handler.getLatitude()
- # ---------------------
- # Create cmor time axis
- # ----------------------
- (rel_time, rel_time_bnds) = createTime(Handler, rc)
-
- # ---------------------------------------------------
- # Create cmor axes and add an axis to data if needed
- # ---------------------------------------------------
- (axes, data) = createAxes( rc, latvals, lonvals, data )
-
- axis_ids = list()
- for axis in axes:
- axis_id = cmor.axis(**axis)
- axis_ids.append(axis_id)
-
- # ----------------------------------------------------------
- # Create cmor variable
- # Note: since this is in the loop, a new cmor file will be
- # create for each cmor write command.
- # ----------------------------------------------------------
- varid = cmor.variable(table_entry = rc['cvrt_cmor_var'],
- axis_ids = axis_ids,
- history = '',
- missing_value = fill_value,
- original_name = rc['cvrt_original_var'],
- units = rc['cvrt_original_units']
- )
- # -------------------------------
- # Write data for this time frame.
- # -------------------------------
- cmor.write(varid,data,\
- time_vals=rel_time,time_bnds=rel_time_bnds)
- cmor.close(varid,file_name=True)
-
- # ---------------------------------------
- # Rename cmor files according to project.
- # ---------------------------------------
- if( movefiles(rc) ):
- return -2
-
- cmor.close()
- return 0
-
-
-
-# ********************************************************************
-#
-# createTime()
-#
-# Define Time and Time bound axes for cmor
-#
-# ********************************************************************
-def createTime(Handler, rc):
- '''
- InputtimeUnits: specified from resource file or from first file
- in a list of file.
-
- return relative time and time bounds using OutputTimeUnits from
- resource file.
- '''
- # ----------------------------------------------------
- # Retrieve time units from file if not provided in the
- # resource file.
- # ----------------------------------------------------
- InputTimeUnits = Handler.getTimeUnits(rc['InputTimeUnits'])
-
- # --------------------------------------------------------
- # Create time relative to January 1st 1900 to facilitate
- # Threds software file handling.
- # -------------------------------------------------------
-
- cur_time = Handler.getTime(InputTimeUnits)
-
- rel_time =[cur_time[i].torel(rc['OutputTimeUnits']).value
- for i in range(len(cur_time))]
-
- if( len(rel_time) == 1 ) :
- deltarel = 1
- else:
- deltarel = rel_time[2] - rel_time[1]
-
- rel_time_bnds = rel_time[:]
- rel_time_bnds.append(rel_time[-1]+deltarel)
- return rel_time, rel_time_bnds
-
-# ********************************************************************
-#
-# getCMIP5lev()
-#
-# Extract CMIP5 mandatory level and recreate a new data array.
-# They are 16 mandatory levels.
-#
-# ********************************************************************
-def getCMIP5lev(data,rc):
- '''
- '''
- oTable = CMORTable(rc['inpath'], rc['table'], "plevs")
- # ----------------------
- # Extract spefied levels
- # ----------------------
- if( 'levels' in oTable.dico.keys() ):
- #pdb.set_trace()
- dataLevels = data.getLevel()[:]
- if( data.getLevel().units == "millibars" or
- data.getLevel().units == "hPa" or
- data.getLevel().units == "mbar" ):
- # --------------------------
- # Change units for to Pascal
- # ---------------------------
- LevelScaleFactor = 100
- dataLevels = data.getLevel()[:] * LevelScaleFactor
-
- # ----------------------------------------
- # No level selected, return all data array
- # ----------------------------------------
- if( len(rc['cvrt_level'].split(":")) == 1 ):
- levels = [ float(item) for item in dataLevels ]
- lev=cdms2.createAxis( levels )
- lev.designateLevel()
- lev.units="pa"
- lev.long_name=data.getLevel().long_name
- #lev.id="lev"
- #lev=data.getAxis(1)
- #lev.__setattr__('_data_',dataLevels.astype(float))
- #lev.__setattr__('units',"Pa")
- #lev.units="hPa"
- data2=data.pressureRegrid(lev)
- return data2
-
- if( rc['cvrt_level'].split(':')[1] == "CMIP5" ):
- lev=cdms2.createAxis( [ float(item/LevelScaleFactor)
- for item in dataLevels
- if item in oTable.dico['levels' ] ] )
-
- lev.designateLevel()
- lev.units="pa"
- lev.long_name = data.getLevel().long_name
- data2=data.pressureRegrid(lev)
- lev[:]=lev[:]*LevelScaleFactor
- return data2
- else:
- # -----------------------
- # Assume a list of values
- # -----------------------
- levels = rc['cvrt_level'].split(':')[1].split(",")
- # --------------------------
- # Change units to Pascal
- # ---------------------------
- dataLevels = [ float(rc['cvrt_level'].split(":")[1].split(",")[i]) * \
- LevelScaleFactor for i in range(len(levels)) ]
- # -----------------------------------
- # Match dataLevels with CMIP5 levels
- # Use file units
- # -----------------------------------
- lev=cdms2.createAxis( [ float(item/LevelScaleFactor)
- for item in dataLevels
- if item in oTable.dico['levels' ] ] )
- # -----------------------------------
- # Set axis metadata
- # -----------------------------------
- lev.units="pa"
- lev.long_name = data.getLevel().long_name
- lev.designateLevel()
- # -----------------------------------
- # Extract specified levels
- # -----------------------------------
- data2=data.pressureRegrid(lev)
- # -----------------------------------
- # Scale data back
- # -----------------------------------
- lev[:]=lev[:]*LevelScaleFactor
- return data2
-
-
-
-
- return data
-# ********************************************************************
-#
-# createAxes()
-#
-# Define axes required by cmor and add z axis to data if needed
-#
-# ********************************************************************
-def createAxes(rc, latvals, lonvals, data):
- # ---------------------------------------------
- # Create time/lat/lon axes using a dictionary
- # ---------------------------------------------
- axes = [
- {'table_entry' : 'time',
- 'units' : rc['OutputTimeUnits']},
-
- {'table_entry' : 'latitude',
- 'units' : 'degrees_north',
- 'coord_vals' : latvals,
- 'cell_bounds' : latvals.getBounds()},
-
- {'table_entry' : 'longitude',
- 'units' : 'degrees_east',
- 'coord_vals' : lonvals,
- 'cell_bounds' : lonvals.getBounds()},
- ]
-
- fill_value = data.fill_value
-
- if( rc['cvrt_level'] == 'height2m' ):
- axes.append({'table_entry' : 'height2m',
- 'units' : 'm',
- 'coord_vals' : [2.0] })
- data = numpy.array(data[:])
- data = data[:,:,:,numpy.newaxis]
-
- elif( rc['cvrt_level'] != '' ):
- data = getCMIP5lev( data, rc )
- levels=data.getLevel()[:]
- axes = numpy.insert(axes, 1,
- {'table_entry' : 'plevs',
- 'units' : 'Pa',
- 'coord_vals' : levels })
-
-
- return axes, data
-
-# ********************************************************************
-#
-# usage()
-#
-# ********************************************************************
-def usage(message):
- '''
- Describe program synopsis.
- '''
- print
- print "*************************"
- print message
- print "*************************"
- print
- print
- print "obs4MIPS_process.py [-h] -r resource"
- print " resource: File containing Global attributes"
- print ""
- print "obs4MIPS will convert an input data file into CMIP5 format using "
- print "CMOR. A directory path will be creating using CMOR by default or "
- print "using a template provided in the resource file."
- print
-
-# ********************************************************************
-#
-# main()
-#
-# ********************************************************************
-def main():
- '''
- '''
- try:
- opts, args = getopt.getopt(sys.argv[1:], "hy:r:",
- ["help" ,"year=","resource="])
- except getopt.GetoptError, err:
- usage(str(err))# will print something like "option -a not recognized"
- return(2)
-
- # --------------------------
- # Verify passed arguments
- # --------------------------
- year = -1
- resource = None
- for o, a in opts:
- if o in ("-r", "--resource"):
- resource = a
- elif o in ("-h", "--help"):
- usage()
- return(0)
- elif o in ("-y", "--year"):
- yr = a
- else:
- assert False, "unhandled option"
-
- # ------------------------------
- # Does the resource file exist?
- # ------------------------------
- if( resource == None ) or ( not os.path.isfile( resource ) ) :
- usage("bad Input Resource File")
- return 1
-
- # -----------------------
- # Read in "rc" file
- # -----------------------
- rc=ESGFresources(resource)
-
- # --------------------------------
- # Extract CMIP5 Table information
- # --------------------------------
- oTable = CMORTable(rc['inpath'], rc['table'])
- if( not 'original_var' in rc.resources.keys() ):
- sys.exit(-1)
- rc['project_id'] = oTable[ 'project_id' ]
- rc['product'] = oTable[ 'product' ]
- rc['modeling_realm'] = oTable[ 'modeling_realm' ]
- rc['frequency'] = oTable[ 'frequency' ]
- if( process(rc) ):
- return -1
-
- return 0
-
-# ********************************************************************
-#
-# Call main program and return exit code
-#
-# ********************************************************************
-if __name__ == '__main__':
- sys.exit(main())
-
-
http://git-wip-us.apache.org/repos/asf/climate/blob/a53e3af5/obs4MIPs/setup.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/setup.py b/obs4MIPs/setup.py
deleted file mode 100644
index eb9a3cc..0000000
--- a/obs4MIPs/setup.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from distutils.core import setup
-
-setup(
- name='Apache Open CLimate Workbench obs4MIPS',
- version='1.0',
- description='Convert observation data to CMIP5s',
- author='Apache Open Climate Workbench',
- author_email='dev@climate.incubator.apache.org',
- url='http://climate.incubator.apache.org/index.html',
- py_modules=['obs4MIPs_process'],
- packages=['','factory', 'Toolbox'],
- package_data={'': ['Tables/*','examples/ECMWF/Y2013/*','examples/TRMM/v7/*']},
-)