You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by le...@apache.org on 2014/10/16 19:16:57 UTC
[01/50] [abbrv] git commit: Branching to roll Release Candidate v5
for 0.1-incubating
Repository: climate
Updated Branches:
refs/heads/master 7a0a7e2d4 -> 26b6c910c
Branching to roll Release Candidate v5 for 0.1-incubating
git-svn-id: https://svn.apache.org/repos/asf/incubator/climate/branches/0.1-incubating@1507020 13f79535-47bb-0310-9956-ffa450edef68
Former-commit-id: 418ced716599893d4da7a6ccc3a3fe9691141743
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/d3cb9508
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/d3cb9508
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/d3cb9508
Branch: refs/heads/master
Commit: d3cb950856c0d5311ceccbfe62559c8e4d0f3109
Parents: 1201406 0591a90
Author: Cameron Eugene Goodale <go...@apache.org>
Authored: Thu Jul 25 15:39:21 2013 +0000
Committer: Cameron Eugene Goodale <go...@apache.org>
Committed: Thu Jul 25 15:39:21 2013 +0000
----------------------------------------------------------------------
CHANGES.txt.REMOVED.git-id | 2 +-
LICENSE.txt.REMOVED.git-id | 2 +-
easy-rcmet/install.sh | 4 +-
rcmet/LICENSE.txt.REMOVED.git-id | 2 +-
rcmet/src/main/python/rcmes/cli/rcmet_ui.py | 5 +
.../python/rcmes/services/directory_helpers.py | 25 +++++
.../python/rcmes/services/list_vars_in_file.py | 4 +-
.../rcmes/storage/files.py.REMOVED.git-id | 2 +-
.../toolkit/do_data_prep.py.REMOVED.git-id | 2 +-
.../rcmes/toolkit/metrics.py.REMOVED.git-id | 2 +-
.../rcmes/toolkit/process.py.REMOVED.git-id | 2 +-
.../python/rcmes/utils/misc.py.REMOVED.git-id | 2 +-
rcmet/src/main/python/tests/test_metrics.py | 52 +++++++++++
rcmet/src/main/ui/app/css/app.css | 39 ++++++--
.../main/ui/app/img/globe.png.REMOVED.git-id | 1 +
rcmet/src/main/ui/app/index.html.REMOVED.git-id | 2 +-
rcmet/src/main/ui/app/js/app.js | 2 +-
.../app/js/controllers/ParameterSelectCtrl.js | 2 +-
.../ui/app/js/controllers/RcmedSelectionCtrl.js | 6 +-
.../main/ui/app/js/controllers/ResultCtrl.js | 30 ++++++
.../main/ui/app/js/controllers/SettingsCtrl.js | 5 +-
.../main/ui/app/js/controllers/TimelineCtrl.js | 96 +++++++++-----------
.../main/ui/app/js/controllers/WorldMapCtrl.js | 82 +++++++++--------
.../src/main/ui/app/js/directives/LeafletMap.js | 7 +-
.../src/main/ui/app/js/directives/PreviewMap.js | 69 ++++++++++++++
rcmet/src/main/ui/app/js/directives/Timeline.js | 16 +++-
.../angular-ui-0.4-tpls.js.REMOVED.git-id | 1 +
.../ui/app/js/services/EvaluationSettings.js | 1 +
.../js/services/SelectedDatasetInformation.js | 2 +-
rcmet/src/main/ui/app/results.html | 68 ++++++++++++++
rcmet/src/main/ui/config/karma.conf.js | 1 +
.../test/unit/controllers/SettingsCtrlTest.js | 12 ++-
.../test/unit/controllers/WorldMapCtrlTest.js | 3 +-
.../ui/test/unit/directives/PreviewMapTest.js | 46 ++++++++++
.../services/SelectedDatasetInfomationTest.js | 2 +-
35 files changed, 472 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
[29/50] [abbrv] git commit: Some linting and refactoring
Posted by le...@apache.org.
Some linting and refactoring
1 Updated doc strings, removed print statements and updated
documentation
2 Refactored checkCriteriaB to be generic function checkCriteria
3 Also fixed findPrecipRate
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/93fbf756
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/93fbf756
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/93fbf756
Branch: refs/heads/master
Commit: 93fbf7565a858526ea526bcc0579ed6cb9a72589
Parents: 96aad4b
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 08:55:11 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:04 2014 -0700
----------------------------------------------------------------------
mccsearch/README.docx | Bin 0 -> 195234 bytes
mccsearch/code/mainProg.py | 19 +-
mccsearch/code/mccSearch.py | 1833 +++++++++++---------------------------
3 files changed, 531 insertions(+), 1321 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/93fbf756/mccsearch/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/README.docx b/mccsearch/README.docx
new file mode 100644
index 0000000..a0a7d1e
Binary files /dev/null and b/mccsearch/README.docx differ
http://git-wip-us.apache.org/repos/asf/climate/blob/93fbf756/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 37e1cc8..90fb76d 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -26,7 +26,7 @@ def main():
allCETRMMList =[]
subprocess.call('export DISPLAY=:0.0', shell=True)
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy"
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy1"
#mainDirStr= "/home/whitehal/mccsearch/caseStudy2"
#directories for the original data
#TRMMdirName ="/home/whitehal/summer2006TRMM"
@@ -55,15 +55,12 @@ def main():
mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
print ("-"*80)
- print 'in main', len(mergImgs)
- print 'timeList', timeList
- print 'TRMMdirName ', TRMMdirName
print "\n -------------- TESTING findCloudElements ----------"
CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
#if the TRMMdirName isnt entered for what ever reason
# CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
- # sys.exit()
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
+ #sys.exit()
print ("-"*80)
print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
print ("-"*80)
@@ -95,11 +92,11 @@ def main():
print ("-"*80)
#print "MCCTimes is: ", MCCTimes
#mccSearch.plotAccTRMM(MCCList)
- mccSearch.displayPrecip(MCCList)
- mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- mccSearch.displaySize(MCCList)
- mccSearch.displayPrecip(MCCList)
- mccSearch.plotHistograms(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistogram(MCCList)
print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
[40/50] [abbrv] git commit: Remove unused imports from mainProg.py
Posted by le...@apache.org.
Remove unused imports from mainProg.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/e19fe299
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/e19fe299
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/e19fe299
Branch: refs/heads/master
Commit: e19fe299702577896424e377ce5670adfa19ee9b
Parents: e4fedfa
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 17:12:11 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/code/mainProg.py | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/e19fe299/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index bb032fc..71d6278 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -19,13 +19,8 @@
# running the program
'''
-import sys
import networkx as nx
-from mccSearch
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
+import mccSearch
import subprocess
@@ -82,7 +77,6 @@ def main():
print ("-"*80)
print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
print ("-"*80)
- #sys.exit()
print "\n -------------- TESTING findMCCs ----------"
MCCList,MCSList = mccSearch.findMCC(prunedGraph)
print ("-"*80)
[47/50] [abbrv] git commit: added a UI
Posted by le...@apache.org.
added a UI
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fce6fece
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fce6fece
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fce6fece
Branch: refs/heads/master
Commit: fce6fececfeb34fb399739c52880653481d30edf
Parents: c1e22a9
Author: kwhitehall <k_...@yahoo.com>
Authored: Wed Apr 30 19:34:05 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/README.docx | Bin 195234 -> 192118 bytes
mccsearch/code/mainProg.py | 6 +-
mccsearch/code/mainProgTemplate.py | 99 +++++++
mccsearch/code/mccSearch.py | 447 ++++++++++++++++++++++----------
mccsearch/code/mccSearchUI.py | 293 +++++++++++++++++++++
5 files changed, 712 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/fce6fece/mccsearch/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/README.docx b/mccsearch/README.docx
index a0a7d1e..648637e 100644
Binary files a/mccsearch/README.docx and b/mccsearch/README.docx differ
http://git-wip-us.apache.org/repos/asf/climate/blob/fce6fece/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 9be65b5..43ca921 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -27,9 +27,9 @@ def main():
mainDirStr= "/directory/to/where/to/store/outputs"
TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
CEoriDirName = "/directory/to/the/MERG/netCDF/files"
-
+
#for first time working with the raw MERG zipped files
- # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # mccSearch.preprocessingMERG("/directory/to/where/the/raw/MERG/files/are")
# ---------------------------------------------------------------------------------
@@ -88,7 +88,7 @@ def main():
#some plots that work
# mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
+ mccSearch.displayPrecip(MCCList)
# mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
# mccSearch.displaySize(MCCList)
# mccSearch.displayPrecip(MCCList)
http://git-wip-us.apache.org/repos/asf/climate/blob/fce6fece/mccsearch/code/mainProgTemplate.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemplate.py b/mccsearch/code/mainProgTemplate.py
new file mode 100644
index 0000000..9be65b5
--- /dev/null
+++ b/mccsearch/code/mainProgTemplate.py
@@ -0,0 +1,99 @@
+'''
+# running the program
+'''
+
+import sys
+import networkx as nx
+import mccSearch
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+import subprocess
+
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+
+ #for GrADs
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ mainDirStr= "/directory/to/where/to/store/outputs"
+ TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+
+ #for first time working with the raw MERG zipped files
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # ---------------------------------------------------------------------------------
+
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ # mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # -------------------------------------------------------------------------------------------------
+
+ #let's go!
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ #print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
+ # ----------------------------------------------------------------------------------------------
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ print "MCS List has been acquired ", len(MCSList)
+ print ("-"*80)
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+
+ #some calculations/metrics that work that work
+ # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistogram(MCCList)
+ #
+ print ("-"*80)
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/fce6fece/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index 7c647d5..8c2c801 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -42,8 +42,8 @@ import process
#the first point closest the the value (for the min) from the MERG data is used, etc.
LATMIN = '5.0' #min latitude; -ve values in the SH e.g. 5S = -5
LATMAX = '19.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
-LONMIN = '-6.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
-LONMAX = '3.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+LONMIN = '-5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '9.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
XRES = 4.0 #x direction spatial resolution in km
YRES = 4.0 #y direction spatial resolution in km
TRES = 1 #temporal resolution in hrs
@@ -545,7 +545,6 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
nodeExist = False
cloudElementCenter=[]
cloudElement = []
- cloudElementCenter = []
cloudElementLat=[]
cloudElementLon =[]
cloudElementLatLons =[]
@@ -986,7 +985,7 @@ def findMCC (prunedGraph):
return definiteMCC, definiteMCS
#******************************************************************
-def traverseTree(subGraph,node, queue, checkedNodes=None):
+def traverseTree(subGraph,node, stack, checkedNodes=None):
'''
Purpose::
To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
@@ -995,8 +994,8 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
subGraph: a Networkx DiGraph representing a CC
lengthOfsubGraph: an integer representing the length of the subgraph
node: a string representing the node currently being checked
- queue: a list of strings representing a list of nodes in a stack functionality
- i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
+ stack: a list of strings representing a list of nodes in a stack functionality
+ i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node
checkedNodes: a list of strings representing the list of the nodes in the traversal
Output::
@@ -1010,31 +1009,31 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
return checkedNodes
if not checkedNodes:
- queue =[]
+ stack =[]
checkedNodes.append(node)
#check one level infront first...if something does exisit, stick it at the front of the stack
upOneLevel = subGraph.predecessors(node)
downOneLevel = subGraph.successors(node)
for parent in upOneLevel:
- if parent not in checkedNodes and parent not in queue:
+ if parent not in checkedNodes and parent not in stack:
for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
- queue.insert(0,child)
+ if child not in checkedNodes and child not in stack:
+ stack.insert(0,child)
- queue.insert(0,parent)
+ stack.insert(0,parent)
for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
+ if child not in checkedNodes and child not in stack:
if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
- queue.insert(0,child)
+ stack.insert(0,child)
else:
- queue.append(child)
+ stack.append(child)
- for eachNode in queue:
+ for eachNode in stack:
if eachNode not in checkedNodes:
checkedNodes.append(eachNode)
- return traverseTree(subGraph, eachNode, queue, checkedNodes)
+ return traverseTree(subGraph, eachNode, stack, checkedNodes)
return checkedNodes
#******************************************************************
@@ -1788,6 +1787,7 @@ def findCESpeed(node, MCSList):
theList = CLOUD_ELEMENT_GRAPH.successors(node)
nodeLatLon=thisDict(node)['cloudElementCenter']
+
for aNode in theList:
if aNode in MCSList:
@@ -1795,15 +1795,25 @@ def findCESpeed(node, MCSList):
aNodeLatLon = thisDict(aNode)['cloudElementCenter']
#calculate CE speed
#checking the lats
- nodeLatLon[0] += 90.0
- aNodeLatLon[0] += 90.0
- delta_lat = (nodeLatLon[0] - aNodeLatLon[0])
- nodeLatLon[1] += 360.0
- aNodeLatLon[1] += 360.0
- delta_lon = (nodeLatLon[1] - aNodeLatLon[1])
- theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
+ # nodeLatLon[0] += 90.0
+ # aNodeLatLon[0] += 90.0
+ # delta_lat = (nodeLatLon[0] - aNodeLatLon[0])
+ delta_lat = ((thisDict(node)['cloudElementCenter'][0] +90.0) - (thisDict(aNode)['cloudElementCenter'][0]+90.0))
+ # nodeLatLon[1] += 360.0
+ # aNodeLatLon[1] += 360.0
+ # delta_lon = (nodeLatLon[1] - aNodeLatLon[1])
+ delta_lon = ((thisDict(node)['cloudElementCenter'][1]+360.0) - (thisDict(aNode)['cloudElementCenter'][1]+360.0))
+
+ try:
+ theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
+ except:
+ theSpeed = 0.0
CEspeed.append(theSpeed)
+
+ # print "~~~ ", thisDict(aNode)['uniqueID']
+ # print "*** ", nodeLatLon, thisDict(node)['cloudElementCenter']
+ # print "*** ", aNodeLatLon, thisDict(aNode)['cloudElementCenter']
if not CEspeed:
return 0.0
@@ -1866,6 +1876,155 @@ def createMainDirectory(mainDirStr):
return
#******************************************************************
+def checkForFiles(startTime, endTime, thisDir, fileType):
+ '''
+ Purpose:: To ensure all the files between the starttime and endTime
+ exist in the directory supplied
+
+ Input::
+ startTime: a string yyyymmmddhh representing the starttime
+ endTime: a string yyyymmmddhh representing the endTime
+ thisDir: a string representing the directory path where to
+ look for the file
+ fileType: an integer representing the type of file in the directory
+ 1 - MERG original files, 2 - TRMM original files
+
+ Output::
+ status: a boolean representing whether all files exists
+
+ '''
+ startFilename = ''
+ endFilename =''
+ currFilename = ''
+ status = False
+ startyr = int(startTime[:4])
+ startmm = int(startTime[4:6])
+ startdd = int(startTime[6:8])
+ starthr = int(startTime[-2:])
+ endyr = int(endTime[:4])
+ endmm = int(endTime[4:6])
+ enddd = int(endTime[6:8])
+ endhh = int(endTime[-2:])
+ curryr = startyr
+ currmm = startmm
+ currdd = startdd
+ currhr = starthr
+ currmmStr = ''
+ currddStr = ''
+ currhrStr = ''
+ endmmStr = ''
+ endddStr =''
+ endhhStr = ''
+
+ #check that the startTime is before the endTime
+ if fileType == 1:
+ #print "fileType is 1"
+ startFilename = "merg_"+startTime+"_4km-pixel.nc"
+ endFilename = thisDir+"/merg_"+endTime+"_4km-pixel.nc"
+
+ if fileType == 2:
+ #TODO:: determine closest time for TRMM files for end
+ #http://disc.sci.gsfc.nasa.gov/additional/faq/precipitation_faq.shtml#convert
+ #How do I extract time information from the TRMM 3B42 file name? section
+ # startFilename = "3B42."+startTime[:8]+"."+currhr+".7A.nc"
+ # endFilename = "3B42."+endTime[:8]+"."+endTime[-2:]+".7A.nc"
+ if starthr%3 == 2:
+ currhr += 1
+ elif starthr%3 ==1:
+ currhr -= 1
+ else:
+ currhr = starthr
+
+ curryr, currmmStr, currddStr, currhrStr,_,_,_ = findTime(curryr, currmm, currdd, currhr)
+
+ startFilename = "3B42."+str(curryr)+currmmStr+currddStr+"."+currhrStr+".7A.nc"
+ if endhh%3 == 2:
+ endhh += 1
+ elif endhh%3 ==1:
+ endhh -= 1
+
+ endyr, endmmStr, endddStr, endhhStr, _, _, _ = findTime(endyr, endmm, enddd, endhh)
+
+ endFilename = thisDir+"/3B42."+str(endyr)+endmmStr+endddStr+"."+endhhStr+".7A.nc"
+
+ #check for files between startTime and endTime
+ currFilename = thisDir+"/"+startFilename
+
+ while currFilename is not endFilename:
+ if not os.path.isfile(currFilename):
+ print "file is missing! Filename: ", currFilename
+ status = False
+ return status
+
+ status = True
+ if currFilename == endFilename:
+ break
+
+ #generate new currFilename
+ if fileType == 1:
+ currhr +=1
+ elif fileType ==2:
+ currhr += 3
+
+ curryr, currmmStr, currddStr, currhrStr, currmm, currdd, currhr = findTime(curryr, currmm, currdd, currhr)
+
+ if fileType == 1:
+ currFilename = thisDir+"/"+"merg_"+str(curryr)+currmmStr+currddStr+currhrStr+"_4km-pixel.nc"
+ if fileType == 2:
+ currFilename = thisDir+"/"+"3B42."+str(curryr)+currmmStr+currddStr+"."+currhrStr+".7A.nc"
+
+ return status
+#******************************************************************
+def findTime(curryr, currmm, currdd, currhr):
+ '''
+ Purpose:: To determine the new yr, mm, dd, hr
+
+ Input:: curryr, an integer representing the year
+ currmm, an integer representing the month
+ currdd, an integer representing the day
+ currhr, an integer representing the hour
+
+ Output::curryr, an integer representing the year
+ currmm, an integer representing the month
+ currdd, an integer representing the day
+ currhr, an integer representing the hour
+ '''
+ if currhr > 23:
+ currhr = 0
+ currdd += 1
+ if currdd > 30 and (currmm == 4 or currmm == 6 or currmm == 9 or currmm == 11):
+ currmm +=1
+ elif currdd > 31 and (currmm == 1 or currmm ==3 or currmm == 5 or currmm == 7 or currmm == 8 or currmm == 10):
+ currmm +=1
+ currdd = 1
+ elif currdd > 31 and currmm == 12:
+ currmm = 1
+ currdd = 1
+ curryr += 1
+ elif currdd > 28 and currmm == 2 and (curryr%4)!=0:
+ currmm = 3
+ currdd = 1
+ elif (curryr%4)==0 and currmm == 2 and currdd>29:
+ currmm = 3
+ currdd = 1
+
+ if currmm < 10:
+ currmmStr="0"+str(currmm)
+ else:
+ currmmStr = str(currmm)
+
+ if currdd < 10:
+ currddStr = "0"+str(currdd)
+ else:
+ currddStr = str(currdd)
+
+ if currhr < 10:
+ currhrStr = "0"+str(currhr)
+ else:
+ currhrStr = str(currhr)
+
+ return curryr, currmmStr, currddStr, currhrStr, currmm, currdd, currhr
+#******************************************************************
def find_nearest(thisArray,value):
'''
Purpose :: to determine the value within an array closes to
@@ -2007,8 +2166,7 @@ def postProcessingNetCDF(dataset, dirName = None):
2 User can write files in location where script is being called
'''
- coreDir = os.path.dirname(MAINDIRECTORY)
- #coreDir = os.path.dirname(os.path.abspath(__file__))
+ coreDir = os.path.dirname(os.path.abspath(__file__))
ImgFilename = ''
frameList=[]
fileList =[]
@@ -2024,35 +2182,29 @@ def postProcessingNetCDF(dataset, dirName = None):
if dataset == 1:
var = 'ch4'
- dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- gsFile = coreDir+"/GrADSscripts/cs2.gs"
- sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
+ origsFile = coreDir+"/../GrADSscripts/cs1.gs"
+ gsFile = coreDir+"/../GrADSscripts/cs2.gs"
+ sologsFile = coreDir+"/../GrADSscripts/mergeCE.gs"
lineNum = 50
elif dataset ==2:
var = 'precipAcc'
- dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
- origsFile = coreDir+"/GrADSscripts/cs3.gs"
- gsFile = coreDir+"/GrADSscripts/cs4.gs"
- sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
+ origsFile = coreDir+"/../GrADSscripts/cs3.gs"
+ gsFile = coreDir+"/../GrADSscripts/cs4.gs"
+ sologsFile = coreDir+"/../GrADSscripts/TRMMCE.gs"
lineNum = 10
elif dataset ==3:
var = 'ch4'
ctlTitle = 'TITLE MERG DATA'
ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
- if dirName is None:
- print "Enter directory for original files"
- return
- else:
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- sologsFile = coreDir+"/GrADSscripts/infrared.gs"
- lineNum = 54
+ origsFile = coreDir+"/../GrADSscripts/cs1.gs"
+ sologsFile = coreDir+"/../GrADSscripts/infrared.gs"
+ lineNum = 54
#sort files
os.chdir((dirName+'/'))
@@ -2066,8 +2218,13 @@ def postProcessingNetCDF(dataset, dirName = None):
for eachfile in files:
fullFname = os.path.splitext(eachfile)[0]
fnameNoExtension = fullFname.split('.nc')[0]
+
+ if dataset == 2 and fnameNoExtension[:4] != "TRMM":
+ continue
+
if dataset == 1 or dataset == 2:
frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
+
#create the ctlFile
ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
#the ctl file
@@ -2083,9 +2240,20 @@ def postProcessingNetCDF(dataset, dirName = None):
subprocess.call(lineToWrite, shell=True)
lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo XDEF 413 LINEAR -9.984375 0.036378335 >> '+ctlFile1
+ fname = dirName+'/'+fnameNoExtension+'.nc'
+ if os.path.isfile(fname):
+ #open NetCDF file add info to the accu
+ print "opening file ", fname
+ fileData = Dataset(fname,'r',format='NETCDF4')
+ lats = fileData.variables['latitude'][:]
+ lons = fileData.variables['longitude'][:]
+ LONDATA, LATDATA = np.meshgrid(lons,lats)
+ nygrd = len(LATDATA[:,0])
+ nxgrd = len(LONDATA[0,:])
+ fileData.close()
+ lineToWrite = 'echo XDEF '+ str(nxgrd) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrd) +' >> ' +ctlFile1
subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo YDEF 412 LINEAR 5.03515625 0.036378335 >> '+ctlFile1
+ lineToWrite = 'echo YDEF '+ str(nygrd)+' LINEAR ' + str(min(lats)) + ' ' + str((max(lats)-min(lats))/nygrd) +' >> '+ctlFile1
subprocess.call(lineToWrite, shell=True)
lineToWrite = 'echo ZDEF 01 LEVELS 1 >> '+ctlFile1
subprocess.call(lineToWrite, shell=True)
@@ -2171,10 +2339,6 @@ def postProcessingNetCDF(dataset, dirName = None):
GrADSscript.writelines(lines1)
GrADSscript.close()
- # if frameNum == 44:
- # sys.exit()
-
-
#run the script
runGrads = 'run '+ gsFile
gradscmd = 'grads -blc ' + '\'' +runGrads + '\''+'\n'
@@ -2492,85 +2656,6 @@ def commonFeatureSize(finalMCCList):
hist, bin_edges = np.histogram(thisMCCAvg)
return hist,bin_edges
#******************************************************************
-def displaySize(finalMCCList):
- '''
- Purpose::
- To create a figure showing the area verse time for each MCS
-
- Input::
- finalMCCList: a list of list of strings representing the list of nodes representing a MCC
-
- Output::
- None
-
- '''
- timeList =[]
- count=1
- imgFilename=''
- minArea=10000.0
- maxArea=0.0
- eachNode={}
-
- #for each node in the list, get the area information from the dictionary
- #in the graph and calculate the area
-
- if finalMCCList:
- for eachMCC in finalMCCList:
- #get the info from the node
- for node in eachMCC:
- eachNode=thisDict(node)
- timeList.append(eachNode['cloudElementTime'])
-
- if eachNode['cloudElementArea'] < minArea:
- minArea = eachNode['cloudElementArea']
- if eachNode['cloudElementArea'] > maxArea:
- maxArea = eachNode['cloudElementArea']
-
-
- #sort and remove duplicates
- timeList=list(set(timeList))
- timeList.sort()
- tdelta = timeList[1] - timeList[0]
- starttime = timeList[0]-tdelta
- endtime = timeList[-1]+tdelta
- timeList.insert(0, starttime)
- timeList.append(endtime)
-
- #plot info
- plt.close('all')
- title = 'Area distribution of the MCC over somewhere'
- fig=plt.figure(facecolor='white', figsize=(18,10)) #figsize=(10,8))#figsize=(16,12))
- fig,ax = plt.subplots(1, facecolor='white', figsize=(10,10))
-
- #the data
- for node in eachMCC: #for eachNode in eachMCC:
- eachNode=thisDict(node)
- if eachNode['cloudElementArea'] < 80000 : #2400.00:
- ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'bo', markersize=10)
- elif eachNode['cloudElementArea'] >= 80000.00 and eachNode['cloudElementArea'] < 160000.00:
- ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'yo',markersize=20)
- else:
- ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'ro',markersize=30)
-
- #axes and labels
- maxArea += 1000.00
- ax.set_xlim(starttime,endtime)
- ax.set_ylim(minArea,maxArea)
- ax.set_ylabel('Area in km^2', fontsize=12)
- ax.set_title(title)
- ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d%H:%M:%S')
- fig.autofmt_xdate()
-
- plt.subplots_adjust(bottom=0.2)
-
- imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif'
- plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
-
- #if time in not already in the time list, append it
- timeList=[]
- count += 1
- return
-#******************************************************************
def precipTotals(finalMCCList):
'''
Purpose::
@@ -2671,6 +2756,89 @@ def precipMaxMin(finalMCCList):
return MCSPrecip
#******************************************************************
+#
+# PLOTS
+#
+#******************************************************************
+def displaySize(finalMCCList):
+ '''
+ Purpose::
+ To create a figure showing the area verse time for each MCS
+
+ Input::
+ finalMCCList: a list of list of strings representing the list of nodes representing a MCC
+
+ Output::
+ None
+
+ '''
+ timeList =[]
+ count=1
+ imgFilename=''
+ minArea=10000.0
+ maxArea=0.0
+ eachNode={}
+
+ #for each node in the list, get the area information from the dictionary
+ #in the graph and calculate the area
+
+ if finalMCCList:
+ for eachMCC in finalMCCList:
+ #get the info from the node
+ for node in eachMCC:
+ eachNode=thisDict(node)
+ timeList.append(eachNode['cloudElementTime'])
+
+ if eachNode['cloudElementArea'] < minArea:
+ minArea = eachNode['cloudElementArea']
+ if eachNode['cloudElementArea'] > maxArea:
+ maxArea = eachNode['cloudElementArea']
+
+
+ #sort and remove duplicates
+ timeList=list(set(timeList))
+ timeList.sort()
+ tdelta = timeList[1] - timeList[0]
+ starttime = timeList[0]-tdelta
+ endtime = timeList[-1]+tdelta
+ timeList.insert(0, starttime)
+ timeList.append(endtime)
+
+ #plot info
+ plt.close('all')
+ title = 'Area distribution of the MCC over somewhere'
+ fig=plt.figure(facecolor='white', figsize=(18,10)) #figsize=(10,8))#figsize=(16,12))
+ fig,ax = plt.subplots(1, facecolor='white', figsize=(10,10))
+
+ #the data
+ for node in eachMCC: #for eachNode in eachMCC:
+ eachNode=thisDict(node)
+ if eachNode['cloudElementArea'] < 80000 : #2400.00:
+ ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'bo', markersize=10)
+ elif eachNode['cloudElementArea'] >= 80000.00 and eachNode['cloudElementArea'] < 160000.00:
+ ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'yo',markersize=20)
+ else:
+ ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'ro',markersize=30)
+
+ #axes and labels
+ maxArea += 1000.00
+ ax.set_xlim(starttime,endtime)
+ ax.set_ylim(minArea,maxArea)
+ ax.set_ylabel('Area in km^2', fontsize=12)
+ ax.set_title(title)
+ ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d%H:%M:%S')
+ fig.autofmt_xdate()
+
+ plt.subplots_adjust(bottom=0.2)
+
+ imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif'
+ plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
+
+ #if time in not already in the time list, append it
+ timeList=[]
+ count += 1
+ return
+#******************************************************************
def displayPrecip(finalMCCList):
'''
Purpose::
@@ -2720,6 +2888,7 @@ def displayPrecip(finalMCCList):
percentagePrecipitating.append((eachNode['TRMMArea']/eachNode['cloudElementArea'])*100.0)
CEArea.append(eachNode['cloudElementArea'])
nodes.append(eachNode['uniqueID'])
+ # print eachNode['uniqueID'], eachNode['cloudElementCenter'][1], eachNode['cloudElementCenter'][0]
x.append(eachNode['cloudElementCenter'][1])#-xStart)
y.append(eachNode['cloudElementCenter'][0])#-yStart)
@@ -2731,6 +2900,9 @@ def displayPrecip(finalMCCList):
totalSize = sum(CEArea)
partialArea = [(a/totalSize)*30000 for a in CEArea]
+
+ # print "x ", x
+ # print "y ", y
#plot info
plt.close('all')
@@ -2931,6 +3103,7 @@ def plotAccTRMM (finalMCCList):
imgFilename = ''
firstPartName = ''
firstTime = True
+ replaceExpXDef = ''
#Just incase the X11 server is giving problems
subprocess.call('export DISPLAY=:0.0', shell=True)
@@ -2944,7 +3117,7 @@ def plotAccTRMM (finalMCCList):
if os.path.isfile(fname):
#open NetCDF file add info to the accu
- print "opening TRMM file ", fname
+ #print "opening TRMM file ", fname
TRMMCEData = Dataset(fname,'r',format='NETCDF4')
precipRate = TRMMCEData.variables['precipitation_Accumulation'][:]
lats = TRMMCEData.variables['latitude'][:]
@@ -2997,6 +3170,7 @@ def plotAccTRMM (finalMCCList):
accuTRMMData.close()
#generate the image with GrADS
+ #print "ny,nx ", nygrdTRMM, nxgrdTRMM, min(lats), max(lats)
#the ctl file
subprocess.call('rm acc.ctl', shell=True)
subprocess.call('touch acc.ctl', shell=True)
@@ -3006,8 +3180,12 @@ def plotAccTRMM (finalMCCList):
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
subprocess.call('echo "UNDEF 0" >> acc.ctl', shell=True)
subprocess.call('echo "TITLE TRMM MCS accumulated precipitation" >> acc.ctl', shell=True)
- subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
- subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpXDef = 'echo XDEF '+ str(nxgrdTRMM) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrdTRMM) +' >> acc.ctl'
+ subprocess.call(replaceExpXDef, shell=True)
+ #subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
+ #subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
+ subprocess.call(replaceExpYDef, shell=True)
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
#subprocess.call(replaceExpTdef, shell=True)
@@ -3031,6 +3209,7 @@ def plotAccTRMM (finalMCCList):
subprocess.call('echo "''\'quit''\'" >> accuTRMM1.gs', shell=True)
gradscmd = 'grads -blc ' + '\'run accuTRMM1.gs''\''
subprocess.call(gradscmd, shell=True)
+ sys.exit()
#clean up
subprocess.call('rm accuTRMM1.gs', shell=True)
@@ -3133,8 +3312,12 @@ def plotAccuInTimeRange(starttime, endtime):
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
subprocess.call('echo "UNDEF 0" >> acc.ctl', shell=True)
subprocess.call('echo "TITLE TRMM MCS accumulated precipitation" >> acc.ctl', shell=True)
- subprocess.call('echo "XDEF 384 LINEAR -8.96875 0.036378335 " >> acc.ctl', shell=True)
- subprocess.call('echo "YDEF 384 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpXDef = 'echo XDEF '+ str(nxgrdTRMM) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrdTRMM) +' >> acc.ctl'
+ subprocess.call(replaceExpXDef, shell=True)
+ replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
+ subprocess.call(replaceExpYDef, shell=True)
+ #subprocess.call('echo "XDEF 384 LINEAR -8.96875 0.036378335 " >> acc.ctl', shell=True)
+ #subprocess.call('echo "YDEF 384 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
@@ -3236,13 +3419,16 @@ def createTextFile(finalMCCList, identifier):
if identifier == 1:
MCSUserFile = open((MAINDIRECTORY+'/textFiles/MCCsUserFile.txt'),'wb')
MCSSummaryFile = open((MAINDIRECTORY+'/textFiles/MCCSummary.txt'),'wb')
+ MCSPostFile = open((MAINDIRECTORY+'/textFiles/MCCPostPrecessing.txt'),'wb')
if identifier == 2:
MCSUserFile = open((MAINDIRECTORY+'/textFiles/MCSsUserFile.txt'),'wb')
MCSSummaryFile = open((MAINDIRECTORY+'/textFiles/MCSSummary.txt'),'wb')
+ MCSPostFile = open((MAINDIRECTORY+'/textFiles/MCSPostPrecessing.txt'),'wb')
for eachPath in finalMCCList:
eachPath.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0], nodeID.split('CE')[1]))
+ MCSPostFile.write("\n %s" %eachPath)
startTime = thisDict(eachPath[0])['cloudElementTime']
endTime = thisDict(eachPath[-1])['cloudElementTime']
@@ -3494,6 +3680,7 @@ def createTextFile(finalMCCList, identifier):
MCSUserFile.close
MCSSummaryFile.close
+ MCSPostFile.close
return
#******************************************************************
# PLOTTING UTIL SCRIPTS
http://git-wip-us.apache.org/repos/asf/climate/blob/fce6fece/mccsearch/code/mccSearchUI.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchUI.py b/mccsearch/code/mccSearchUI.py
new file mode 100644
index 0000000..9dbb727
--- /dev/null
+++ b/mccsearch/code/mccSearchUI.py
@@ -0,0 +1,293 @@
+'''
+# Wizard for running the mccSearch program
+'''
+
+import sys
+import networkx as nx
+import numpy as np
+import numpy.ma as ma
+import os
+import matplotlib.pyplot as plt
+import subprocess
+
+#mccSearch modules
+import mccSearch
+import files
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ DIRS={}
+ # DIRS={
+ # mainDirStr= "/directory/to/where/to/store/outputs"
+ # TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ # CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+ # }
+ preprocessing = ''
+ rawMERG = ''
+
+ #for GrADs
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+
+ print "Running MCCSearch ..... \n"
+ DIRS['mainDirStr'] = raw_input("> Please enter working directory: \n" ) # This is where data created will be stored
+
+ preprocessing = raw_input ("> Do you need to preprocess the MERG files? [y/n]: \n")
+ while preprocessing.lower() != 'n':
+ if preprocessing.lower() == 'y':
+ #get location for raw files
+ rawMERG = raw_input("> Please enter the directory to the RAW MERG (.Z) files: \n")
+ #run preprocessing
+ mccSearch.preprocessingMERG(rawMERG)
+ continue
+ elif preprocessing.lower() == 'n' :
+ pass
+ else:
+ print "Error! Invalid choice "
+ preprocessing = raw_input ("> Do you need to preprocess the MERG files? [y/n]: \n")
+
+
+ #get the location of the MERG and TRMM data
+ DIRS['CEoriDirName'] = raw_input("> Please enter the directory to the MERG netCDF files: \n")
+
+ try:
+ if not os.path.exists(DIRS['CEoriDirName']):
+ print "Error! MERG invalid path!"
+ DIRS['CEoriDirName'] = raw_input("> Please enter the directory to the MERG netCDF files: \n")
+ except:
+ print "..."
+
+
+ DIRS['TRMMdirName'] = raw_input("> Please enter the location to the raw TRMM netCDF files: \n")
+ try:
+ if not os.path.exists(DIRS['TRMMdirName']):
+ print "Error: TRMM invalid path!"
+ DIRS['TRMMdirName'] = raw_input("> Please enter the location to the raw TRMM netCDF files: \n")
+ except:
+ pass
+
+ #get the dates for analysis
+ startDateTime = raw_input("> Please enter the start date and time yyyymmddhr: \n")
+ #check validity of time
+ while validDate(startDateTime) == 0:
+ print "Invalid time entered for startDateTime!"
+ startDateTime = raw_input("> Please enter the start date and time yyyymmddhr: \n")
+
+ endDateTime = raw_input("> Please enter the end date and time yyyymmddhr: \n")
+ while validDate(endDateTime) == 0:
+ print "Invalid time entered for endDateTime!"
+ endDateTime = raw_input("> Please enter the end date and time yyyymmddhr: \n")
+
+ #check if all the files exisits in the MERG and TRMM directories entered
+ test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['TRMMdirName'], 2)
+ if test == False:
+ print "Error with files in the original MERG directory entered. Please check your files before restarting. "
+ return
+ test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['CEoriDirName'],1)
+ if test == False:
+ print "Error with files in the original TRMM directory entered. Please check your files before restarting. "
+ return
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(DIRS['mainDirStr'])
+ TRMMCEdirName = DIRS['mainDirStr']+'/TRMMnetcdfCEs'
+ CEdirName = DIRS['mainDirStr']+'/MERGnetcdfCEs'
+
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ postprocessing = raw_input("> Do you wish to postprocess data? [y/n] \n")
+ while postprocessing.lower() != 'n':
+ if postprocessing.lower() == 'y':
+ option = postProcessingplotMenu(DIRS)
+ return
+ elif postprocessing.lower() == 'n':
+ pass
+ else:
+ print "\n Invalid option."
+ postprocessing = raw_input("> Do you wish to postprocess data? [y/n] \n")
+ # -------------------------------------------------------------------------------------------------
+ # Getting started. Make it so number one!
+ print ("-"*80)
+ print "\t\t Starting the MCCSearch Analysis "
+ print ("-"*80)
+ print "\n -------------- Reading MERG and TRMM Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(DIRS['CEoriDirName'])
+ print "\n -------------- findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,DIRS['TRMMdirName'])
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(DIRS['TRMMdirName'],timeList)
+ # ----------------------------------------------------------------------------------------------
+ print "\n -------------- findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print "\n -------------- findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ #now ready to perform various calculations/metrics
+ print ("-"*80)
+ print "\n -------------- METRICS ----------"
+ print ("-"*80)
+ #some calculations/metrics that work that work
+ print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ plotMenu(MCCList, MCSList)
+
+ #Let's get outta here! Engage!
+ print ("-"*80)
+#*********************************************************************************************************************
+def plotMenu(MCCList, MCSList):
+ '''
+ Purpose:: The flow of plots for the user to choose
+
+ Input:: MCCList: a list of directories representing a list of nodes in the MCC
+ MCSList: a list of directories representing a list of nodes in the MCS
+
+ Output:: None
+ '''
+ option = displayPlotMenu()
+ while option != 0:
+ try:
+ if option == 1:
+ print "Generating Accumulated Rainfall from TRMM for the entire period ...\n"
+ mccSearch.plotAccTRMM(MCCList)
+ if option == 2:
+ startDateTime = raw_input("> Please enter the start date and time yyyy-mm-dd_hr:mm:ss format: \n")
+ endDateTime = raw_input("> Please enter the end date and time yyyy-mm-dd_hr:mm:ss format: \n")
+ print "Generating acccumulated rainfall between ", startDateTime," and ", endDateTime, " ... \n"
+ mccSearch.plotAccuInTimeRange(startDateTime, endDateTime)
+ if option == 3:
+ print "Generating area distribution plot ... \n"
+ mccSearch.displaySize(MCCList)
+ if option == 4:
+ print "Generating precipitation and area distribution plot ... \n"
+ mccSearch.displayPrecip(MCCList)
+ if option == 5:
+ try:
+ print "Generating histogram of precipitation for each time ... \n"
+ mccSearch.plotPrecipHistograms(MCCList)
+ except:
+ pass
+ except:
+ print "Invalid option. Please try again, enter 0 to exit \n"
+
+ option = displayPlotMenu()
+ return
+#*********************************************************************************************************************
+def displayPlotMenu():
+ '''
+ Purpose:: Display the plot Menu Options
+
+ Input:: None
+
+ Output:: option: an integer representing the choice of the user
+ '''
+ print "**************** PLOTS ************** \n"
+ print "0. Exit \n"
+ print "1. Accumulated TRMM precipitation \n"
+ print "2. Accumulated TRMM precipitation between dates \n"
+ print "3. Area distribution of the system over time \n"
+ print "4. Precipitation and area distribution of the system \n"
+ print "5. Histogram distribution of the rainfall in the area \n"
+ option = int(raw_input("> Please enter your option for plots: \n"))
+ return option
+#*********************************************************************************************************************
+def displayPostprocessingPlotMenu():
+ '''
+ Purpose:: Display the plot Menu Options
+
+ Input:: None
+
+ Output:: option: an integer representing the choice of the user
+ '''
+ print "**************** POST PROCESSING PLOTS ************** \n"
+ print "0. Exit \n"
+ print "1. Map plots of the original MERG data \n"
+ print "2. Map plots of the cloud elements using IR data \n"
+ print "3. Map plots of the cloud elements rainfall accumulations using TRMM data \n"
+ #print "4. Accumulated TRMM precipitation \n"
+ #print "5. Accumulated TRMM precipitation between dates \n"
+
+ option = int(raw_input("> Please enter your option for plots: \n"))
+ return option
+#*********************************************************************************************************************
+def postProcessingplotMenu(DIRS):
+ '''
+ Purpose:: The flow of plots for the user to choose
+
+ Input:: DIRS a dictionary of directories
+ # DIRS={
+ # mainDirStr= "/directory/to/where/to/store/outputs"
+ # TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ # CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+ # }
+
+ Output:: None
+ '''
+ TRMMCEdirName = DIRS['mainDirStr']+'/TRMMnetcdfCEs'
+ CEdirName = DIRS['mainDirStr']+'/MERGnetcdfCEs'
+
+ option = displayPostprocessingPlotMenu()
+ while option != 0:
+ try:
+ if option == 1:
+ print "Generating images from the original MERG dataset ... \n"
+ mccSearch.postProcessingNetCDF(3, DIRS['CEoriDirName'])
+ if option == 2:
+ print "Generating images from the cloud elements using MERG IR data ... \n"
+ mccSearch.postProcessingNetCDF(1, CEdirName)
+ if option == 3:
+ print "Generating precipitation accumulation images from the cloud elements using TRMM data ... \n"
+ mccSearch.postProcessingNetCDF(2, TRMMCEdirName)
+ # if option == 4:
+ # print "Generating Accumulated TRMM rainfall from cloud elements for each MCS ... \n"
+ # featureType = int(raw_input("> Please enter type of MCS MCC-1 or MCS-2: \n"))
+ # if featureType == 1:
+ # filename = DIRS['mainDirStr']+'/textFiles/MCCPostProcessing.txt'
+ # try:
+ # if os.path.isfile(filename):
+ # #read each line as a list
+ # mccSearch.plotAccTRMM()
+ # if option == 5:
+ # mccSearch.plotAccuInTimeRange()
+ except:
+ print "Invalid option, please try again"
+ option = displayPostprocessingPlotMenu()
+ return
+#*********************************************************************************************************************
+def validDate(dataString):
+ '''
+ '''
+
+ if len(dataString) > 10:
+ print "invalid time entered"
+ return 0
+
+ yr = int(dataString[:4])
+ mm = int(dataString[4:6])
+ dd = int(dataString[6:8])
+ hh = int(dataString[-2:])
+
+ if mm < 1 or mm > 12:
+ return 0
+ elif hh < 0 or hh > 23:
+ return 0
+ elif (dd< 0 or dd > 30) and (mm == 4 or mm == 6 or mm == 9 or mm == 11):
+ return 0
+ elif (dd< 0 or dd > 31) and (mm == 1 or mm ==3 or mm == 5 or mm == 7 or mm == 8 or mm == 10):
+ return 0
+ elif dd > 28 and mm == 2 and (yr%4)!=0:
+ return 0
+ elif (yr%4)==0 and mm == 2 and dd>29:
+ return 0
+ elif dd > 31 and mm == 12:
+ return 0
+ else:
+ return 1
+#*********************************************************************************************************************
+
+main()
+
[18/50] [abbrv] git commit: fixed bug regarding ability to access a
subset of files in a folder
Posted by le...@apache.org.
fixed bug regarding ability to access a subset of files in a folder
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/0d1f3d0f
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/0d1f3d0f
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/0d1f3d0f
Branch: refs/heads/master
Commit: 0d1f3d0f891ece9492e3c355c19464c4ff24faaf
Parents: a5b7677
Author: georgette <k_...@yahoo.com>
Authored: Tue Jun 3 09:37:52 2014 -0400
Committer: georgette <k_...@yahoo.com>
Committed: Tue Jun 3 09:37:52 2014 -0400
----------------------------------------------------------------------
mccsearch/.DS_Store | Bin 0 -> 6148 bytes
mccsearch/GrADsScripts/.DS_Store | Bin 0 -> 6148 bytes
mccsearch/code/.DS_Store | Bin 0 -> 6148 bytes
mccsearch/code/mccSearch.py | 17 ++++++++++++-----
mccsearch/code/mccSearchUI.py | 10 +++++-----
5 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/0d1f3d0f/mccsearch/.DS_Store
----------------------------------------------------------------------
diff --git a/mccsearch/.DS_Store b/mccsearch/.DS_Store
new file mode 100644
index 0000000..9cc3ce1
Binary files /dev/null and b/mccsearch/.DS_Store differ
http://git-wip-us.apache.org/repos/asf/climate/blob/0d1f3d0f/mccsearch/GrADsScripts/.DS_Store
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/.DS_Store b/mccsearch/GrADsScripts/.DS_Store
new file mode 100644
index 0000000..5008ddf
Binary files /dev/null and b/mccsearch/GrADsScripts/.DS_Store differ
http://git-wip-us.apache.org/repos/asf/climate/blob/0d1f3d0f/mccsearch/code/.DS_Store
----------------------------------------------------------------------
diff --git a/mccsearch/code/.DS_Store b/mccsearch/code/.DS_Store
new file mode 100644
index 0000000..5008ddf
Binary files /dev/null and b/mccsearch/code/.DS_Store differ
http://git-wip-us.apache.org/repos/asf/climate/blob/0d1f3d0f/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index 8c2c801..b7dc4eb 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -77,13 +77,14 @@ PRUNED_GRAPH = nx.DiGraph()
#------------------------ End GLOBAL VARS -------------------------
#************************ Begin Functions *************************
#******************************************************************
-def readMergData(dirname):
+def readMergData(dirname, filelist = None):
'''
Purpose::
Read MERG data into RCMES format
Input::
- Directory to the MERG files in NETCDF format
+ dirname: a string representing the directory to the MERG files in NETCDF format
+ filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
Output::
A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
@@ -107,7 +108,8 @@ def readMergData(dirname):
mergLonVarName = 'longitude'
filelistInstructions = dirname + '/*'
- filelist = glob.glob(filelistInstructions)
+ if filelist == None:
+ filelist = glob.glob(filelistInstructions)
#sat_img is the array that will contain all the masked frames
@@ -1893,6 +1895,7 @@ def checkForFiles(startTime, endTime, thisDir, fileType):
status: a boolean representing whether all files exists
'''
+ filelist =[]
startFilename = ''
endFilename =''
currFilename = ''
@@ -1951,10 +1954,14 @@ def checkForFiles(startTime, endTime, thisDir, fileType):
currFilename = thisDir+"/"+startFilename
while currFilename is not endFilename:
+
if not os.path.isfile(currFilename):
print "file is missing! Filename: ", currFilename
status = False
- return status
+ return status, filelist
+ else:
+ #create filelist
+ filelist.append(currFilename)
status = True
if currFilename == endFilename:
@@ -1973,7 +1980,7 @@ def checkForFiles(startTime, endTime, thisDir, fileType):
if fileType == 2:
currFilename = thisDir+"/"+"3B42."+str(curryr)+currmmStr+currddStr+"."+currhrStr+".7A.nc"
- return status
+ return status,filelist
#******************************************************************
def findTime(curryr, currmm, currdd, currhr):
'''
http://git-wip-us.apache.org/repos/asf/climate/blob/0d1f3d0f/mccsearch/code/mccSearchUI.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchUI.py b/mccsearch/code/mccSearchUI.py
index 9dbb727..45dc73a 100644
--- a/mccsearch/code/mccSearchUI.py
+++ b/mccsearch/code/mccSearchUI.py
@@ -85,13 +85,13 @@ def main():
endDateTime = raw_input("> Please enter the end date and time yyyymmddhr: \n")
#check if all the files exisits in the MERG and TRMM directories entered
- test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['TRMMdirName'], 2)
+ test, _ = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['TRMMdirName'], 2)
if test == False:
- print "Error with files in the original MERG directory entered. Please check your files before restarting. "
+ print "Error with files in the original TRMM directory entered. Please check your files before restarting. "
return
- test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['CEoriDirName'],1)
+ test, filelist = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['CEoriDirName'],1)
if test == False:
- print "Error with files in the original TRMM directory entered. Please check your files before restarting. "
+ print "Error with files in the original MERG directory entered. Please check your files before restarting. "
return
#create main directory and file structure for storing intel
@@ -116,7 +116,7 @@ def main():
print "\t\t Starting the MCCSearch Analysis "
print ("-"*80)
print "\n -------------- Reading MERG and TRMM Data ----------"
- mergImgs, timeList = mccSearch.readMergData(DIRS['CEoriDirName'])
+ mergImgs, timeList = mccSearch.readMergData(DIRS['CEoriDirName'], filelist)
print "\n -------------- findCloudElements ----------"
CEGraph = mccSearch.findCloudElements(mergImgs,timeList,DIRS['TRMMdirName'])
#if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
[28/50] [abbrv] Some linting and refactoring
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/93fbf756/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index 177aef8..7c647d5 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -1,8 +1,6 @@
'''
# All the functions for the MCC search algorithm
# Following RCMES dataformat in format (t,lat,lon), value
-# Kim Whitehall
-# Mimumum MCS is 3 hours
'''
import datetime
@@ -42,38 +40,34 @@ import process
# --------------------- User defined variables ---------------------
#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
#the first point closest the the value (for the min) from the MERG data is used, etc.
-LATMIN = '5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
-LATMAX = '19.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
-LONMIN = '-9.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
-LONMAX = '5.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+LATMIN = '5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '19.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
+LONMIN = '-6.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '3.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
XRES = 4.0 #x direction spatial resolution in km
YRES = 4.0 #y direction spatial resolution in km
TRES = 1 #temporal resolution in hrs
-T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
-T_BB_MIN = 218 #221 #cooler temp for the center of the system
+LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
#have same rank of the matrix it is being compared against
+#criteria for determining cloud elements and edges
+T_BB_MAX = 243 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
+T_BB_MIN = 218 #cooler temp for the center of the system
CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
-MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
-MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
-LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
-LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
-DIRECTION = 45.0 #general direction of the wind flow in degrees
AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
-MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
-#---Assuming using the MCC function, these will have to be changed
+#---the MCC criteria
ECCENTRICITY_THRESHOLD_MAX = 1.0 #tending to 1 is a circle e.g. hurricane,
-ECCENTRICITY_THRESHOLD_MIN = 0.70 #0.65 #tending to 0 is a linear e.g. squall line
-OUTER_CLOUD_SHIELD_AREA = 80000.0 #100000.0 #km^2
-INNER_CLOUD_SHIELD_AREA = 30000.0 #50000.0 #km^2
+ECCENTRICITY_THRESHOLD_MIN = 0.50 #tending to 0 is a linear e.g. squall line
+OUTER_CLOUD_SHIELD_AREA = 80000.0 #km^2
+INNER_CLOUD_SHIELD_AREA = 30000.0 #km^2
OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
-MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
-MAXIMUM_DURATION = 100 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
-
-#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, MCCs is 6hrs)
+MAXIMUM_DURATION = 24#max number of framce the MCC can last for
#------------------- End user defined Variables -------------------
edgeWeight = [1,2,3] #weights for the graph edges
#graph object fo the CEs meeting the criteria
@@ -94,15 +88,13 @@ def readMergData(dirname):
Output::
A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
criteria for each frame
- **remove below**
- A dictionary of all the MERG data from the files in the directory given.
- The dictionary contains, a string representing the time (a datetime string), a
- tuple (lat,lon,value) representing only the data that meets the temperature requirements
- i.e. T_BB_MAX
Assumptions::
The MERG data has been converted to NETCDF using LATS4D
The data has the same lat/lon format
+
+ TODO:: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
+
'''
global LAT
@@ -136,8 +128,7 @@ def readMergData(dirname):
else:
# Open the first file in the list to read in lats, lons and generate the grid for comparison
tmp = Nio.open_file(filelist[0], format='nc')
- #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
-
+
#clip the lat/lon grid according to user input
#http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
@@ -149,14 +140,11 @@ def readMergData(dirname):
latsraw =[]
lonsraw = []
nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
- #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
tmp.close
for files in filelist:
try:
thisFile = Nio.open_file(files, format='nc')
- #TODO: thisFile = netCDF4.Dataset(files)
-
#clip the dataset according to user lat,lon coordinates
#mask the data and fill with zeros for later
tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
@@ -176,8 +164,6 @@ def readMergData(dirname):
#extend instead of append because getModelTimes returns a list already and we don't
#want a list of list
timelist.extend(time2store)
- #to get the actual masked data only
- #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
mergImgs.extend(tempMaskedValue)
thisFile.close
thisFile = None
@@ -195,21 +181,20 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
Determines the contiguous boxes for a given time of the satellite images i.e. each frame
using scipy ndimage package
- Input::
- 3 variables
- sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
+ Input::
+ mergImgs: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
maximum acceptable temperature, T_BB_MAX
timelist: a list of python datatimes
TRMMdirName (optional): string representing the path where to find the TRMM datafiles
Output::
- 1 variable
- cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
- The dictionary is
+ CLOUD_ELEMENT_GRAPH: a Networkx directed graph where each node contains the information in cloudElementDict
+ The nodes are determined according to the area of contiguous squares. The nodes are linked through weighted edges.
+
cloudElementDict = {'uniqueID': unique tag for this CE,
'cloudElementTime': time of the CE,
'cloudElementLatLon': (lat,lon,value) of MERG data of CE,
- 'cloudElementCenter':tuple of floating-point (lat,lon) representing the CE's center
+ 'cloudElementCenter':list of floating-point [lat,lon] representing the CE's center
'cloudElementArea':floating-point representing the area of the CE,
'cloudElementEccentricity': floating-point representing the shape of the CE,
'cloudElementTmax':integer representing the maximum Tb in CE,
@@ -248,7 +233,6 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
precip =[]
TRMMCloudElementLatLons =[]
- #edgeWeight = [1,2]
minCELatLimit = 0.0
minCELonLimit = 0.0
maxCELatLimit = 0.0
@@ -258,12 +242,9 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#openfile for storing ALL cloudElement information
cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
-
-
#openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
- #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
-
+
#NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
for t in xrange(mergImgs.shape[0]):
#-------------------------------------------------
@@ -299,9 +280,8 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
numOfBoxes = np.count_nonzero(cloudElement)
cloudElementArea = numOfBoxes*XRES*YRES
- #If the area is greater than the area required, then start considering as CE
- #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
- #if it does, still consider as CE
+ #If the area is greater than the area required, or if the area is smaller than the suggested area, check if it meets a convective fraction requirement
+ #consider as CE
if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
@@ -514,7 +494,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
thisCElen = len(cloudElementLatLons)
percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
- #change weights to integers because the built in shortest path chokes on floating pts
+ #change weights to integers because the built in shortest path chokes on floating pts according to Networkx doc
#according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
if percentageOverlap >= 0.95:
CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
@@ -598,18 +578,19 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
print ("*"*80)
#hierachial graph output
- #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
+ graphTitle = "Cloud Elements observed over somewhere from 0000Z to 0000Z"
drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
return CLOUD_ELEMENT_GRAPH
#******************************************************************
-def findPrecipRate(TRMMdirName):
- '''
- TODO: needs fixing
- Purpose:: Determines the precipitation rates for MCSs found if TRMMdirName was not entered in findCloudElements this can be used
+def findPrecipRate(TRMMdirName, timelist):
+ '''
+ Purpose::
+ Determines the precipitation rates for MCSs found if TRMMdirName was not entered in findCloudElements this can be used
- Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
+ Input::
+ TRMMdirName: a string representing the directory for the original TRMM netCDF files
+ timelist: a list of python datatimes
Output:: a list of dictionary of the TRMM data
NB: also creates netCDF with TRMM data for each CE (for post processing) index
@@ -631,137 +612,141 @@ def findPrecipRate(TRMMdirName):
files.sort(key=lambda x: os.path.getmtime(x))
for afile in files:
- #try:
- print "in for ", afile
fullFname = os.path.splitext(afile)[0]
noFrameExtension = (fullFname.replace("_","")).split('F')[0]
CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
fileDateTime = fileDateTimeChar.replace("-","")
fileDate = fileDateTime[:-6]
- fileHr=fileDateTime[-6:-4]
+ fileHr1=fileDateTime[-6:-4]
cloudElementData = Dataset(afile,'r', format='NETCDF4')
- brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
+ brightnesstemp1 = cloudElementData.variables['brightnesstemp'][:,:,:]
latsrawCloudElements = cloudElementData.variables['latitude'][:]
lonsrawCloudElements = cloudElementData.variables['longitude'][:]
+
+ brightnesstemp = np.squeeze(brightnesstemp1, axis=0)
+
+ if int(fileHr1) % temporalRes == 0:
+ fileHr = fileHr1
+ else:
+ fileHr = (int(fileHr1)/temporalRes) * temporalRes
+
+ if fileHr < 10:
+ fileHr = '0'+str(fileHr)
+ else:
+ str(fileHr)
+
+ TRMMfileName = TRMMdirName+"/3B42."+ str(fileDate) + "."+str(fileHr)+".7A.nc"
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
- if int(fileHr) % 3 == 0:
- TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
- #print "TRMMfileName is ", TRMMfileName
- TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
- precipRate = TRMMData.variables['pcp'][:,:,:]
- latsrawTRMMData = TRMMData.variables['latitude'][:]
- lonsrawTRMMData = TRMMData.variables['longitude'][:]
- lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
- LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
-
- #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
-
- precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
- #---------regrid the TRMM data to the MERG dataset ----------------------------------
- #regrid using the do_regrid stuff from the Apache OCW
- regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
- regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
- #----------------------------------------------------------------------------------
-
- # #get the lat/lon info from cloudElement
- latCEStart = LAT[0][0]
- latCEEnd = LAT[-1][0]
- lonCEStart = LON[0][0]
- lonCEEnd = LON[0][-1]
-
- #get the lat/lon info for TRMM data (different resolution)
- latStartT = find_nearest(latsrawTRMMData, latCEStart)
- latEndT = find_nearest(latsrawTRMMData, latCEEnd)
- lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
- lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
- latStartIndex = np.where(latsrawTRMMData == latStartT)
- latEndIndex = np.where(latsrawTRMMData == latEndT)
- lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
- lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
-
- #get the relevant TRMM info
- CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
- TRMMData.close()
+ # ------ NETCDF File stuff ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
+ currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
+ currNetCDFTRMMData.calendar = 'standard'
+ currNetCDFTRMMData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFTRMMData.createDimension('time', None)
+ currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ fileDateTime[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp1.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+ for index,value in np.ndenumerate(brightnesstemp):
+ lat_index, lon_index = index
+ currTimeValue = 0
+ if value > 0:
+
+ finalCETRMMvalues[0,lat_index,lon_index] = regriddedTRMM[int(np.where(LAT[:,0]==LAT[lat_index,0])[0]), int(np.where(LON[0,:]==LON[0,lon_index])[0])]
-
- # ------ NETCDF File stuff ------------------------------------
- thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
- # print "thisFileName ", thisFileName
- # sys.exit()
- currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
- currNetCDFData.calendar = 'standard'
- currNetCDFData.conventions = 'COARDS'
- # dimensions
- currNetCDFData.createDimension('time', None)
- currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
- currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
- # variables
- TRMMprecip = ('time','lat', 'lon',)
- times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
- longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
- rainFallacc.units = 'mm'
-
- longitude[:] = LON[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
-
- latitude[:] = LAT[:,0]
- latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
-
- finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
- #-----------End most of NETCDF file stuff ------------------------------------
- #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
- for index,value in np.ndenumerate(brightnesstemp):
- #print "index, value ", index, value
- time_index, lat_index, lon_index = index
- currTimeValue = 0
- if value > 0:
- finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
-
+ rainFallacc[:] = finalCETRMMvalues
+ currNetCDFTRMMData.close()
+
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ precipTotal += value
+
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+
+ try:
+ minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ minCEprecipRate = 0.0
+
+ try:
+ maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ maxCEprecipRate = 0.0
+
+ #add info to CLOUDELEMENTSGRAPH
+ #TODO try block
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
+ if eachdict[1]['uniqueID'] == CEuniqueID:
+ if not 'cloudElementPrecipTotal' in eachdict[1].keys():
+ eachdict[1]['cloudElementPrecipTotal'] = precipTotal
+ if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
+ eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
+ if not 'TRMMArea' in eachdict[1].keys():
+ eachdict[1]['TRMMArea'] = TRMMArea
+ if not 'CETRMMmin' in eachdict[1].keys():
+ eachdict[1]['CETRMMmin'] = minCEprecipRate
+ if not 'CETRMMmax' in eachdict[1].keys():
+ eachdict[1]['CETRMMmax'] = maxCEprecipRate
- # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
- # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
- # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
- # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
- # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
- # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
- # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
- # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
- # # precipTotal = value*temporalRes*1.0
- # # prevLatValue = currLatvalue
- # # prevLonValue = currLonValue
-
- TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
- TRMMArea = TRMMnumOfBoxes*XRES*YRES
- rainFallacc[:] = finalCETRMMvalues
- currNetCDFData.close()
- for index, value in np.ndenumerate(finalCETRMMvalues):
- #print "index, value ", index, value
- precipTotal += value
-
- # #add info to the dictionary and to the list
- # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
- # allCEnodesTRMMdata.append(TRMMdataDict)
- # print "precipTotal is ", precipTotal
- #add info to CLOUDELEMENTSGRAPH
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
- if eachdict[1]['uniqueID'] == CEuniqueID:
- if not 'cloudElementPrecipTotal' in eachdict[1].keys():
- eachdict[1]['cloudElementPrecipTotal'] = precipTotal
- if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
- eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
- if not 'TRMMArea' in eachdict[1].keys():
- eachdict[1]['TRMMArea'] = TRMMArea
#clean up
precipTotal = 0.0
latsrawTRMMData =[]
@@ -777,13 +762,16 @@ def findPrecipRate(TRMMdirName):
#******************************************************************
def findCloudClusters(CEGraph):
'''
- Purpose:: Determines the cloud clusters properties from the subgraphs in
+ Purpose::
+ Determines the cloud clusters properties from the subgraphs in
the graph i.e. prunes the graph according to the minimum depth
- Input:: CEGraph: 1 graph - a directed graph of the CEs with weighted edges
- according the area overlap between nodes (CEs) of consectuive frames
+ Input::
+ CEGraph: a Networkx directed graph of the CEs with weighted edges
+ according the area overlap between nodes (CEs) of consectuive frames
- Output:: PRUNED_GRAPH: 1 graph - a directed graph of with CCs/ MCSs
+ Output::
+ PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
'''
@@ -825,8 +813,7 @@ def findCloudClusters(CEGraph):
print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
print ("*"*80)
- #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
+ graphTitle = "Cloud Clusters observed over somewhere during sometime"
drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
cloudClustersFile.close
@@ -834,13 +821,17 @@ def findCloudClusters(CEGraph):
#******************************************************************
def findMCC (prunedGraph):
'''
- Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
+ Purpose::
+ Determines if subtree is a MCC according to Laurent et al 1998 criteria
- Input:: prunedGraph: a Networkx Graph representing the CCs
+ Input::
+ prunedGraph: a Networkx Graph representing the CCs
- Output:: finalMCCList: a list of list of tuples representing a MCC
+ Output::
+ finalMCCList: a list of list of tuples representing a MCC
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
MCCList = []
@@ -997,18 +988,22 @@ def findMCC (prunedGraph):
#******************************************************************
def traverseTree(subGraph,node, queue, checkedNodes=None):
'''
- Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+ Purpose::
+ To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
- Input:: subGraph: a Networkx DiGraph representing a CC
+ Input::
+ subGraph: a Networkx DiGraph representing a CC
lengthOfsubGraph: an integer representing the length of the subgraph
node: a string representing the node currently being checked
queue: a list of strings representing a list of nodes in a stack functionality
i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
checkedNodes: a list of strings representing the list of the nodes in the traversal
- Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
+ Output::
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
if len(checkedNodes) == len(subGraph):
@@ -1026,7 +1021,7 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
for child in downOneLevel:
if child not in checkedNodes and child not in queue:
queue.insert(0,child)
- #downCheckFlag = True
+
queue.insert(0,parent)
for child in downOneLevel:
@@ -1036,10 +1031,8 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
else:
queue.append(child)
- #print "^^^^^stack ", stack
for eachNode in queue:
if eachNode not in checkedNodes:
- #print "if eachNode ", checkedNodes, eachNode, stack
checkedNodes.append(eachNode)
return traverseTree(subGraph, eachNode, queue, checkedNodes)
@@ -1047,14 +1040,17 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
#******************************************************************
def checkedNodesMCC (prunedGraph, nodeList):
'''
- Purpose :: Determine if this path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
+ Purpose ::
+ Determine if this path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- nodeList: list of strings (CE ID) from the traversal
+ Input::
+ prunedGraph: a Networkx Graph representing all the cloud clusters
+ nodeList: list of strings (CE ID) from the traversal
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
- dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+ Output::
+ potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
'''
CounterCriteriaAFlag = False
@@ -1063,14 +1059,13 @@ def checkedNodesMCC (prunedGraph, nodeList):
MATURITYFLAG = False
DECAYFLAG = False
thisdict = {} #will have the same items as the cloudElementDict
- cloudElementArea = 0.0
+ cloudElementAreaB = 0.0
+ cloudElementAreaA = 0.0
epsilon = 0.0
frameNum =0
oldNode =''
potentialMCCList =[]
durationAandB = 0
- CounterCriteriaA = 0
- CountercriteriaB = 0
#check for if the list contains only one string/node
if type(nodeList) is str:
@@ -1085,41 +1080,62 @@ def checkedNodesMCC (prunedGraph, nodeList):
existingFrameFlag = False
if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
- #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
CounterCriteriaAFlag = True
INITIATIONFLAG = True
MATURITYFLAG = False
- #check if criteriaB is meet
- cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
-
- #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
- if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
- #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- CounterCriteriaBFlag = True
- #append this information on to the dictionary
- addInfothisDict(node, cloudElementArea, criteriaB)
- INITIATIONFLAG = False
- MATURITYFLAG = True
- stage = 'M'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- else:
- #criteria B failed
- CounterCriteriaBFlag = False
- if INITIATIONFLAG == True:
- stage = 'I'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
-
- elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
- DECAYFLAG = True
- MATURITYFLAG = False
- stage = 'D'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ #check if criteriaA is met
+ cloudElementAreaA, criteriaA = checkCriteria(thisdict['cloudElementLatLon'], OUTER_CLOUD_SHIELD_TEMPERATURE)
+ #TODO: calcuate the eccentricity at this point and read over????or create a new field in the dict
+
+ if cloudElementAreaA >= OUTER_CLOUD_SHIELD_AREA:
+ #check if criteriaB is met
+ cloudElementAreaB,criteriaB = checkCriteria(thisdict['cloudElementLatLon'], INNER_CLOUD_SHIELD_TEMPERATURE)
+
+ #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
+ if cloudElementAreaB >= INNER_CLOUD_SHIELD_AREA:
+ #TODO: add another field to the dictionary for the OUTER_AREA_SHIELD area
+ CounterCriteriaBFlag = True
+ #append this information on to the dictionary
+ addInfothisDict(node, cloudElementAreaB, criteriaB)
+ INITIATIONFLAG = False
+ MATURITYFLAG = True
+ stage = 'M'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria B failed
+ CounterCriteriaBFlag = False
+ if INITIATIONFLAG == True:
+ stage = 'I'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
+ DECAYFLAG = True
+ MATURITYFLAG = False
+ stage = 'D'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria A failed
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ #add as a CE before or after the main feature
+ if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
+ stage = "D"
+ DECAYFLAG = True
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+
else:
#criteria A failed
CounterCriteriaAFlag = False
CounterCriteriaBFlag = False
- #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- #add as a CE before or after the main feature
+ #add as a CE before or after the main feature
if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
stage ="I"
potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
@@ -1135,16 +1151,19 @@ def checkedNodesMCC (prunedGraph, nodeList):
#******************************************************************
def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
'''
- Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
+ Purpose::
+ Utility function to determine if a path is (or is part of) a MCC and provides
preliminary information regarding the stages of the feature
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- node: a string representing the cloud element currently being assessed
- CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
- CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
+ Input::
+ prunedGraph: a Networkx Graph representing all the cloud clusters
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ node: a string representing the cloud element currently being assessed
+ CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
+ CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ Output::
+ potentialMCCList: list of dictionaries representing all possible MCC within the path
dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
'''
@@ -1173,7 +1192,7 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
if predecessorsFlag == True:
- for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
+ for eachNode in potentialMCCList[index]["possMCCList"]:
if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
existingFrameFlag = True
@@ -1218,7 +1237,7 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
if successorsFlag == True:
- for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
+ for eachNode in potentialMCCList[index]["possMCCList"]:
if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
existingFrameFlag = True
@@ -1310,16 +1329,19 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
#******************************************************************
def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
'''
- Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
+ Purpose::
+ Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
- Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
- upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
- node: a string representing the cloud element currently being assessed
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
+ Input::
+ prunedGraph:a Networkx Graph representing all the cloud clusters
+ upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
+ node: a string representing the cloud element currently being assessed
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
- Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
- index: an integer representing the location in the potentialMCCList where thisFlag occurs
+ Output::
+ thisFlag: a boolean representing whether the list passed has in the parent or child of the node
+ index: an integer representing the location in the potentialMCCList where thisFlag occurs
'''
thisFlag = False
@@ -1337,7 +1359,7 @@ def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
index = -1
for MCCDict in potentialMCCList:
index += 1
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ if aNode in list(x[0] for x in MCCDict[checkList]):
thisFlag = True
#get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
return thisFlag, index
@@ -1350,7 +1372,7 @@ def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
for MCCDict in potentialMCCList:
index += 1
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ if aNode in list(x[0] for x in MCCDict[checkList]):
thisFlag = True
return thisFlag, index
@@ -1358,12 +1380,16 @@ def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
#******************************************************************
def maxExtentAndEccentricity(eachList):
'''
- Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
+ Purpose::
+ Perform the final check for MCC based on maximum extent and eccentricity criteria
+
+ Input::
+ eachList: a list of strings representing the node of the possible MCCs within a path
- Input:: eachList: a list of strings representing the node of the possible MCCs within a path
+ Output::
+ maxShieldNode: a string representing the node with the maximum maxShieldNode
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
- Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
- definiteMCCFlag: a boolean indicating that the MCC has met all requirements
'''
maxShieldNode =''
maxShieldArea = 0.0
@@ -1385,13 +1411,16 @@ def maxExtentAndEccentricity(eachList):
#******************************************************************
def findMaxDepthAndMinPath (thisPathDistanceAndLength):
'''
- Purpose:: To determine the maximum depth and min path for the headnode
+ Purpose::
+ To determine the maximum depth and min path for the headnode
- Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
- thisPathDistanceAndLength({distance}, {path})
+ Input::
+ tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
+ thisPathDistanceAndLength({distance}, {path})
{distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
- Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
+ Output::
+ tuple of the max pathLength and min pathDistance as a tuple (like what was input)
minDistanceAndMaxPath = ({distance},{path})
'''
maxPathLength = 0
@@ -1416,30 +1445,37 @@ def findMaxDepthAndMinPath (thisPathDistanceAndLength):
if pathLength == maxPathLength :
if pathDistance <= minPath:
minPath = pathLength
- #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
+ #store details if absolute minPath and deepest
minDistanceAndMaxPath = (pathDistance, path)
return minDistanceAndMaxPath
#******************************************************************
def thisDict (thisNode):
'''
- Purpose:: return dictionary from graph if node exist in tree
+ Purpose::
+ Return dictionary from graph if node exist in tree
- Input:: String - thisNode
+ Input::
+ thisNode: a string representing the CE to get the information for
- Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
+ Output ::
+ eachdict[1]: a dictionary representing the info associated with thisNode from the graph
'''
for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
if eachdict[1]['uniqueID'] == thisNode:
return eachdict[1]
#******************************************************************
-def checkCriteriaB (thisCloudElementLatLon):
+def checkCriteria (thisCloudElementLatLon, aTemperature):
'''
- Purpose:: Determine if criteria B is met for a CEGraph
+ Purpose::
+ Determine if criteria B is met for a CEGraph
- Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
+ Input::
+ thisCloudElementLatLon: 2D array of (lat,lon) variable from the node dictionary being currently considered
+ aTemperature:a integer representing the temperature maximum for masking
- Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
+ Output ::
+ cloudElementArea: a floating-point number representing the area in the array that meet the criteria - criteriaB
'''
cloudElementCriteriaBLatLon=[]
@@ -1453,25 +1489,20 @@ def checkCriteriaB (thisCloudElementLatLon):
minLon = min(x[1]for x in thisCloudElementLatLon)
maxLon = max(x[1]for x in thisCloudElementLatLon)
- #print "minLat, maxLat, minLon, maxLon ", minLat, maxLat, minLon, maxLon
minLatIndex = np.argmax(LAT[:,0] == minLat)
maxLatIndex = np.argmax(LAT[:,0]== maxLat)
minLonIndex = np.argmax(LON[0,:] == minLon)
maxLonIndex = np.argmax(LON[0,:] == maxLon)
- #print "minLatIndex, maxLatIndex, minLonIndex, maxLonIndex ", minLatIndex, maxLatIndex, minLonIndex, maxLonIndex
-
criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
for x in thisCloudElementLatLon:
#to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
#index given in the original array to get the indices for the new array
- #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
- #print criteriaBframe
- #keep only those values < TBB_MAX
- tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
+ #keep only those values < aTemperature
+ tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= aTemperature), fill_value = 0)
#get the actual values that the mask returned
criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
@@ -1493,10 +1524,6 @@ def checkCriteriaB (thisCloudElementLatLon):
cloudElementArea = 0.0
return cloudElementArea, cloudElementCriteriaBLatLon
- print "after loc", loc
- print "************************"
- #print "criteriaB ", criteriaB.shape
- print criteriaB
try:
cloudElementCriteriaB = ma.zeros((criteriaB.shape))
cloudElementCriteriaB =criteriaB[loc]
@@ -1513,7 +1540,7 @@ def checkCriteriaB (thisCloudElementLatLon):
cloudElementCriteriaBLatLon.append(lat_lon_tuple)
cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
- #do some
+ #do some cleaning up
tempMask =[]
criteriaB =[]
cloudElementCriteriaB=[]
@@ -1522,11 +1549,13 @@ def checkCriteriaB (thisCloudElementLatLon):
#******************************************************************
def hasMergesOrSplits (nodeList):
'''
- Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
- Input:: nodeList - list of nodes from a path
- Output:: two list_vars_in_file
- splitList list of all the nodes in the path that split
- mergeList list of all the nodes in the path that merged
+ Purpose::
+ Determine if nodes within a path defined from shortest_path splittingNodeDict
+ Input::
+ nodeList: list of strings representing the nodes from a path
+ Output::
+ splitList: a list of strings representing all the nodes in the path that split
+ mergeList: a list of strings representing all the nodes in the path that merged
'''
mergeList=[]
splitList=[]
@@ -1546,13 +1575,16 @@ def hasMergesOrSplits (nodeList):
#******************************************************************
def allAncestors(path, aNode):
'''
- Purpose:: Utility script to provide the path leading up to a nodeList
+ Purpose::
+ Utility script to provide the path leading up to a nodeList
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for parents
+ Input::
+ path: a list of strings representing the nodes in the path
+ aNode: a string representing a node to be checked for parents
- Output:: list of strings - path: the list of the nodes connected to aNode through its parents
- integer - numOfChildren: the number of parents of the node passed
+ Output::
+ path: a list of strings representing the list of the nodes connected to aNode through its parents
+ numOfChildren: an integer representing the number of parents of the node passed
'''
numOfParents = PRUNED_GRAPH.in_degree(aNode)
@@ -1569,13 +1601,16 @@ def allAncestors(path, aNode):
#******************************************************************
def allDescendants(path, aNode):
'''
- Purpose:: Utility script to provide the path leading up to a nodeList
+ Purpose::
+ Utility script to provide the path leading up to a nodeList
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for children
+ Input::
+ path: a list of strings representing the nodes in the path
+ aNode: a string representing a node to be checked for children
- Output:: list of strings - path: the list of the nodes connected to aNode through its children
- integer - numOfChildren: the number of children of the node passed
+ Output::
+ path: a list of strings representing the list of the nodes connected to aNode through its children
+ numOfChildren: an integer representing the number of children of the node passed
'''
numOfChildren = PRUNED_GRAPH.out_degree(aNode)
@@ -1594,13 +1629,15 @@ def allDescendants(path, aNode):
#******************************************************************
def addInfothisDict (thisNode, cloudElementArea,criteriaB):
'''
- Purpose:: update original dictionary node with information
+ Purpose::
+ Update original dictionary node with information
- Input:: String - thisNode
- float - cloudElementArea,
- masked array of floats meeting the criteria - criteriaB
+ Input::
+ thisNode: a string representing the unique ID of a node
+ cloudElementArea: a floating-point number representing the area of the cloud element
+ criteriaB: a masked array of floating-point numbers representing the lat,lons meeting the criteria
- Output ::
+ Output:: None
'''
for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
@@ -1613,8 +1650,9 @@ def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
'''
Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
- Input:: String - thisNode
- String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
+ Input::
+ thisNode: a string representing the unique ID of a node
+ nodeBehaviorIdentifier: a string representing the behavior S- split, M- merge, B- both split and merge, N- neither split or merge
Output :: None
@@ -1627,11 +1665,12 @@ def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
#******************************************************************
def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
'''
- Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+ Purpose::
+ Add an identifier to the node dictionary to indicate splitting, merging or neither node
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
+ Input::
+ thisNode: a string representing the unique ID of a node
+ nodeMCSIdentifier: a string representing the stage of the MCS lifecyle 'I' for Initiation, 'M' for Maturity, 'D' for Decay
Output :: None
@@ -1644,11 +1683,12 @@ def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
#******************************************************************
def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
'''
- Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
+ Purpose::
+ Update an identifier to the node dictionary to indicate splitting, merging or neither node
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
+ Input::
+ thisNode: thisNode: a string representing the unique ID of a node
+ nodeMCSIdentifier: a string representing the stage of the MCS lifecyle 'I' for Initiation, 'M' for Maturity, 'D' for Decay
Output :: None
@@ -1667,12 +1707,10 @@ def eccentricity (cloudElementLatLon):
values tending to 0 are more linear
Input::
- 1 variable
- cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
+ cloudElementLatLon: 2D array in (lat,lon) representing T_bb contiguous squares
Output::
- 1 variable
- epsilon: a float representing the eccentricity of the matrix passed
+ epsilon: a floating-point representing the eccentricity of the matrix passed
'''
@@ -1698,14 +1736,12 @@ def cloudElementOverlap (currentCELatLons, previousCELatLons):
Determines the percentage overlap between two list of lat-lons passed
Input::
- 2 sorted list of tuples
- currentCELatLons - the list of tuples for the current CE
- previousCELatLons - the list of tuples for the other CE being considered
+ currentCELatLons: a list of tuples for the current CE
+ previousCELatLons: a list of tuples for the other CE being considered
Output::
- 2 variables
- percentageOverlap - a float representing the number of overlapping lat_lon tuples
- areaOverlap - a floating-point number representing the area overlapping
+ percentageOverlap: a floating-point representing the number of overlapping lat_lon tuples
+ areaOverlap: a floating-point number representing the area overlapping
'''
@@ -1732,13 +1768,15 @@ def cloudElementOverlap (currentCELatLons, previousCELatLons):
#******************************************************************
def findCESpeed(node, MCSList):
'''
- Purpose:: to determine the speed of the CEs
- uses vector displacement delta_lat/delta_lon (y/x)
+ Purpose::
+ To determine the speed of the CEs uses vector displacement delta_lat/delta_lon (y/x)
- Input:: node: a string representing the CE
- MCSList: a list of strings representing the feature
+ Input::
+ node: a string representing the CE
+ MCSList: a list of strings representing the feature
Output::
+ CEspeed: a floating-point number representing the speed of the CE
'''
@@ -1750,42 +1788,23 @@ def findCESpeed(node, MCSList):
theList = CLOUD_ELEMENT_GRAPH.successors(node)
nodeLatLon=thisDict(node)['cloudElementCenter']
- #print "nodeLatLon ", nodeLatLon
-
-
+
for aNode in theList:
if aNode in MCSList:
#if aNode is part of the MCSList then determine distance
aNodeLatLon = thisDict(aNode)['cloudElementCenter']
- #print "aNodeLatLon ", aNodeLatLon
#calculate CE speed
#checking the lats
nodeLatLon[0] += 90.0
aNodeLatLon[0] += 90.0
- delta_lat = (nodeLatLon[0] - aNodeLatLon[0]) #convert to m
- #delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
- #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
-
- #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
- #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
- # if nodeLatLon[1] < 0.0:
- # nodeLatLon[1] += 360.0
- # if aNodeLatLon[1] <= 0.0:
- # delta_lon = aNodeLatLon[1] + 360.0
- #0E --> 360 and E lons > west lons
+ delta_lat = (nodeLatLon[0] - aNodeLatLon[0])
nodeLatLon[1] += 360.0
aNodeLatLon[1] += 360.0
- delta_lon = (nodeLatLon[1] - aNodeLatLon[1]) #convert to m
- #delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
-
- #print "delta_lat, delta_lon ", delta_lat, delta_lon
-
- #theSpeed = abs(((delta_lat*LAT_DISTANCE*1000)/(delta_lon*LON_DISTANCE*1000))/(TRES*3600)) #convert to s --> m/s
+ delta_lon = (nodeLatLon[1] - aNodeLatLon[1])
theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
CEspeed.append(theSpeed)
- #print "aNode CE speed is ", aNode, (((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600)), theSpeed
-
+
if not CEspeed:
return 0.0
else:
@@ -1802,13 +1821,11 @@ def maenumerate(mArray):
Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
Input::
- 1 variable
- mArray - the masked array returned from the ma.array() command
+ mArray: the masked array returned from the ma.array() command
Output::
- 1 variable
- maskedValues - 3D (t,lat,lon), value of only masked values
+ maskedValues: 3D (t,lat,lon), value of only masked values
'''
@@ -1820,9 +1837,11 @@ def maenumerate(mArray):
#******************************************************************
def createMainDirectory(mainDirStr):
'''
- Purpose:: to create the main directory for storing information and
- the subdirectories for storing information
- Input:: mainDir: a directory for where all information generated from
+ Purpose::
+ To create the main directory for storing information and
+ the subdirectories for storing information
+ Input::
+ mainDir: a directory for where all information generated from
the program are to be stored
Output:: None
@@ -2001,7 +2020,6 @@ def postProcessingNetCDF(dataset, dirName = None):
#Just incase the X11 server is giving problems
subprocess.call('export DISPLAY=:0.0', shell=True)
- print "coreDir is ", str(coreDir)
prevFrameNum = 0
if dataset == 1:
@@ -2183,16 +2201,20 @@ def postProcessingNetCDF(dataset, dirName = None):
#******************************************************************
def drawGraph (thisGraph, graphTitle, edgeWeight=None):
'''
- Purpose:: Utility function to draw graph in the hierachial format
+ Purpose::
+ Utility function to draw graph in the hierachial format
- Input:: a Networkx graph
- a string representing the graph title
- a list of integers representing the edge weights in the graph
+ Input::
+ thisGraph: a Networkx directed graph
+ graphTitle: a string representing the graph title
+ edgeWeight: (optional) a list of integers representing the edge weights in the graph
+
Output:: None
+
'''
imgFilename = MAINDIRECTORY+'/images/'+ graphTitle+".gif"
- fig=plt.figure(facecolor='white', figsize=(16,12)) #figsize=(10,8))#figsize=(16,12))
+ fig=plt.figure(facecolor='white', figsize=(16,12))
edge95 = [(u,v) for (u,v,d) in thisGraph.edges(data=True) if d['weight'] == edgeWeight[0]]
edge90 = [(u,v) for (u,v,d) in thisGraph.edges(data=True) if d['weight'] == edgeWeight[1]]
@@ -2201,41 +2223,28 @@ def drawGraph (thisGraph, graphTitle, edgeWeight=None):
nx.write_dot(thisGraph, 'test.dot')
plt.title(graphTitle)
pos = nx.graphviz_layout(thisGraph, prog='dot')
- #nx.draw(thisGraph, pos, with_labels=True, arrows=True)
#draw graph in parts
#nodes
nx.draw_networkx_nodes(thisGraph, pos, with_labels=True, arrows=False)
#edges
- nx.draw_networkx_edges(thisGraph, pos, edgelist=edge95, alpha=0.5, arrows=False) #with_labels=True, arrows=True)
+ nx.draw_networkx_edges(thisGraph, pos, edgelist=edge95, alpha=0.5, arrows=False)
nx.draw_networkx_edges(thisGraph, pos, edgelist=edge90, edge_color='b', style='dashed', arrows=False)
nx.draw_networkx_edges(thisGraph, pos, edgelist=edegeOverlap, edge_color='y', style='dashed', arrows=False)
#labels
- nx.draw_networkx_labels(thisGraph,pos, arrows=False) #,font_size=20,font_family='sans-serif')
+ nx.draw_networkx_labels(thisGraph,pos, arrows=False)
plt.axis('off')
plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
#do some clean up...and ensuring that we are in the right dir
os.chdir((MAINDIRECTORY+'/'))
subprocess.call('rm test.dot', shell=True)
- #plt.show()
-#******************************************************************
-def convert_timedelta(duration):
- '''
- Taken from http://stackoverflow.com/questions/14190045/how-to-convert-datetime-timedelta-to-minutes-hours-in-python
-
- '''
-
- days, seconds = duration.days, duration.seconds
- hours = days * 24 + seconds // 3600
- minutes = (seconds % 3600) // 60
- seconds = (seconds % 60)
- return hours, minutes, seconds
#******************************************************************
def tempMaskedImages(imgFilename):
'''
- Purpose:: To generate temperature-masked images for a first pass verification
+ Purpose::
+ To generate temperature-masked images for a first pass verification
Input::
- filename for the gif file
+ imgFilename: filename for the gif file
Output::
None - Gif images for each file of T_bb less than 250K are generated in folder called mergImgs
@@ -2258,61 +2267,47 @@ def tempMaskedImages(imgFilename):
subprocess.call('echo "''\'set clevs 190 200 210 220 230 240 250''\'" >> tempMaskedImages.gs', shell=True)
subprocess.call('echo "''\'d ch4+75''\'" >> tempMaskedImages.gs', shell=True)
subprocess.call('echo "''\'draw title Masked Temp @ '+imgFilename +'\'" >> tempMaskedImages.gs', shell=True)
- #printimCmd = 'printim '+imgFilename +' x1000 y800'
subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800''\'" >> tempMaskedImages.gs', shell=True)
subprocess.call('echo "''\'quit''\'" >> tempMaskedImages.gs', shell=True)
gradscmd = 'grads -blc ' + '\'run tempMaskedImages.gs''\''
subprocess.call(gradscmd, shell=True)
return
#******************************************************************
-#******************************************************************
#
# METRICS FUNCTIONS FOR MERG.PY
+# TODO: rewrite these metrics so that they use the data from the
+# file instead of the graph as this reduce mem resources needed
+#
#
-#------------------------------------------------------------------
-#1)Need to know the starttime and end time
-# - determine most common, least common and avg initiation time
-# - determine duration
-#
-#2)Need criteria B area and temperature
-# - determine max, min and avg area and T
-# - visualization opportunity to show all the criteriaB for time
-# on a map
-#
-#3)Need temperature and area data
-# - calculate the max, min and average temp
-#
-#4)Need lat/lon and T for all MCC
-# - visualization of all MCCs during the time
-# - calculate speed of the system
-# ****** Comparing with other datasets ******
-#5)Need lat/lon and times for all MCCs
-# - use to locate same positions in dataset2 (e.g. TRMM) to
-# determine precip
#******************************************************************
def numberOfFeatures(finalMCCList):
'''
- Purpose:: To count the number of MCCs found for the period
+ Purpose::
+ To count the number of MCCs found for the period
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
+ Input::
+ finalMCCList: a list of list of strings representing a list of list of nodes representing a MCC
- Output::an integer representing the number of MCCs found
-
- Assumptions::
+ Output::
+ an integer representing the number of MCCs found
'''
return len(finalMCCList)
#******************************************************************
def temporalAndAreaInfoMetric(finalMCCList):
'''
- Purpose:: To provide information regarding the temporal properties of the MCCs found
+ Purpose::
+ To provide information regarding the temporal properties of the MCCs found
- Input:: List of dictionaries - finalMCCList: a list of dictionaries of nodes representing a MCC
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Output:: list of dictionaries - allMCCtimes {MCCtimes, starttime, endtime, duration, area): a list of dictionaries
- of MCC temporal details for each MCC in the period considered
+ Output::
+ allMCCtimes: a list of dictionaries {MCCtimes, starttime, endtime, duration, area} representing a list of dictionaries
+ of MCC temporal details for each MCC in the period considered
- Assumptions:: the final time hour --> the event lasted throughout that hr, therefore +1 to endtime
+ Assumptions::
+ the final time hour --> the event lasted throughout that hr, therefore +1 to endtime
'''
#TODO: in real data edit this to use datetime
#starttime =0
@@ -2323,19 +2318,15 @@ def temporalAndAreaInfoMetric(finalMCCList):
MCSArea =[]
if finalMCCList:
- #print "len finalMCCList is: ", len(finalMCCList)
for eachMCC in finalMCCList:
- #print "eachMCC count in temporalInfoMetric is ", len(eachMCC)
#get the info from the node
for eachNode in eachMCC:
- #print "eachNode in temporalInfoMetric is ", eachNode['uniqueID'], eachNode['cloudElementTime']
MCCtimes.append(thisDict(eachNode)['cloudElementTime'])
MCSArea.append(thisDict(eachNode)['cloudElementArea'])
#sort and remove duplicates
MCCtimes=list(set(MCCtimes))
MCCtimes.sort()
- #print "MCCtimes are: ", MCCtimes
tdelta = MCCtimes[1] - MCCtimes[0]
starttime = MCCtimes[0]
endtime = MCCtimes[-1]
@@ -2346,19 +2337,22 @@ def temporalAndAreaInfoMetric(finalMCCList):
MCSArea=[]
else:
allMCCtimes =[]
- tdelta = 0 # 00:00:00
+ tdelta = 0
return allMCCtimes, tdelta
#******************************************************************
def longestDuration(allMCCtimes):
'''
- Purpose:: To determine the longest MCC for the period
+ Purpose::
+ To determine the longest MCC for the period
- Input:: list of dictionaries - allMCCtimes {MCCtimes, starttime, endtime, duration): a list of dictionaries
- of MCC temporal details for each MCC in the period considered
+ Input::
+ allMCCtimes: a list of dictionaries {MCCtimes, starttime, endtime, duration, area} representing a list of dictionaries
+ of MCC temporal details for each MCC in the period considered
- Output::an integer - lenMCC: representing the duration of the longest MCC found
- a list of strings - longestMCC: representing the nodes of longest MCC
+ Output::
+ an integer - lenMCC: representing the duration of the longest MCC found
+ a list of strings - longestMCC: representing the nodes of longest MCC
Assumptions::
@@ -2421,11 +2415,15 @@ def averageDuration(allMCCtimes):
#******************************************************************
def averageTime (allTimes):
'''
- Purpose:: To determine the average time in a list of datetimes
- e.g. of use is finding avg starttime,
- Input:: allTimes: a list of datetimes representing all of a given event e.g. start time
+ Purpose::
+ To determine the average time in a list of datetimes
+ e.g. of use is finding avg starttime,
+ Input::
+ allTimes: a list of datetimes representing all of a given event e.g. start time
+
+ Output::
+ a floating-point number representing the average of the times given
- Output:: a float representing the average of the times given
'''
avgTime = 0
@@ -2466,11 +2464,14 @@ def averageFeatureSize(finalMCCList):
#******************************************************************
def commonFeatureSize(finalMCCList):
'''
- Purpose:: To determine the common (mode) MCC size for the period
+ Purpose::
+ To determine the common (mode) MCC size for the period
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
+ Input::
+ finalMCCList: a list of list of strings representing the list of nodes representing a MCC
- Output::a floating-point representing the average area of a MCC in the period
+ Output::
+ a floating-point representing the average area of a MCC in the period
Assumptions::
@@ -2491,19 +2492,16 @@ def commonFeatureSize(finalMCCList):
hist, bin_edges = np.histogram(thisMCCAvg)
return hist,bin_edges
#******************************************************************
-def commonFeatureDuration (finalMCCList):
- '''
- '''
-#******************************************************************
def displaySize(finalMCCList):
'''
- Purpose:: To create a figure showing the area verse time for each MCS
+ Purpose::
+ To create a figure showing the area verse time for each MCS
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
+ Input::
+ finalMCCList: a list of list of strings representing the list of nodes representing a MCC
- Output:: None
-
- Assumptions::
+ Output::
+ None
'''
timeList =[]
@@ -2517,9 +2515,7 @@ def displaySize(finalMCCList):
#in the graph and calculate the area
if finalMCCList:
- #print "len finalMCCList is: ", len(finalMCCList)
for eachMCC in finalMCCList:
- #print "eachMCC count in temporalInfoMetric is ", len(eachMCC)
#get the info from the node
for node in eachMCC:
eachNode=thisDict(node)
@@ -2542,18 +2538,15 @@ def displaySize(finalMCCList):
#plot info
plt.close('all')
- #title = 'Area verses time for MCS'+ str(count)
- title = 'Area distribution of the MCC over Burkina Faso'
+ title = 'Area distribution of the MCC over somewhere'
fig=plt.figure(facecolor='white', figsize=(18,10)) #figsize=(10,8))#figsize=(16,12))
fig,ax = plt.subplots(1, facecolor='white', figsize=(10,10))
#the data
for node in eachMCC: #for eachNode in eachMCC:
eachNode=thisDict(node)
- #ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'ro')
if eachNode['cloudElementArea'] < 80000 : #2400.00:
ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'bo', markersize=10)
- #eachNode['cloudElementArea'] >= 2400.00 and eachNode['cloudElementArea'] < 10000.00:
elif eachNode['cloudElementArea'] >= 80000.00 and eachNode['cloudElementArea'] < 160000.00:
ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'yo',markersize=20)
else:
@@ -2569,7 +2562,6 @@ def displaySize(finalMCCList):
fig.autofmt_xdate()
plt.subplots_adjust(bottom=0.2)
- #plt.show()
imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif'
plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
@@ -2581,11 +2573,14 @@ def displaySize(finalMCCList):
#******************************************************************
def precipTotals(finalMCCList):
'''
- Purpose:: Precipitation totals associated with a cloud element
+ Purpose::
+ Precipitation totals associated with a cloud element
- Input:: List of dictionaries - finalMCCList: a list of dictionaries of nodes representing a MCC
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Output:: precipTotal - to total amount of precipitation associated
+ Output::
+ precipTotal: a floating-point number representing the total amount of precipitation associated
with the feature
'''
precipTotal = 0.0
@@ -2604,11 +2599,7 @@ def precipTotals(finalMCCList):
if count == 1:
prevHr = int(str(eachNode['cloudElementTime']).replace(" ", "")[-8:-6])
- #print "prevHr ", prevHr
currHr =int(str(eachNode['cloudElementTime']).replace(" ", "")[-8:-6])
- #print "currHr ", currHr
- print "time ",eachNode['cloudElementTime'], prevHr, currHr
- print "eachNode[cloudElementPrecipTotal] ", eachNode['cloudElementPrecipTotal']
if prevHr == currHr:
CEprecip += eachNode['cloudElementPrecipTotal']
else:
@@ -2622,8 +2613,7 @@ def precipTotals(finalMCCList):
prevHr = currHr
MCSPrecip.append(('0',precipTotal))
- print "MCSPrecip ", MCSPrecip
-
+
allMCSPrecip.append(MCSPrecip)
precipTotal =0.0
CEprecip = 0.0
@@ -2637,10 +2627,14 @@ def precipTotals(finalMCCList):
def precipMaxMin(finalMCCList):
'''
TODO: this doesnt work the np.min/max function seems to be not working with the nonzero option..possibly a problem upstream with cloudElementLatLonTRMM
- Purpose:: Precipitation maximum and min rates associated with each CE in MCS
- Input:: List of dictionaries - finalMCCList: a list of dictionaries of nodes representing a MCC
+ Purpose::
+ Precipitation maximum and min rates associated with each CE in MCS
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
+
+ Output::
+ MCSPrecip: a list indicating max and min rate for each CE identified
- Output:: a list tuples indicating max and min rate for each CE identified
'''
maxCEprecip = 0.0
minCEprecip =0.0
@@ -2660,30 +2654,17 @@ def precipMaxMin(finalMCCList):
print "maxCEprecip ", np.max(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
sys.exit()
maxCEprecip = np.max(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.max((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
minCEprecip = np.min(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.min((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
MCSPrecip.append((eachNode['uniqueID'],minCEprecip, maxCEprecip))
- # for node in finalMCCList:
- # eachNode=thisDict(node)
- # #find min and max precip
- # minCEprecip = min((eachNode['cloudElementLatLonTRMM'])[2])
- # maxCEprecip = max((eachNode['cloudElementLatLonTRMM'])[2])
- # MCSPrecip.append((eachNode['uniqueID'],minCEprecip, maxCEprecip))
-
+
else:
for eachMCC in finalMCCList:
#get the info from the node
for node in eachMCC:
eachNode=thisDict(node)
#find min and max precip
- # minCEprecip = min((eachNode['cloudElementLatLonTRMM'])[2])
- # maxCEprecip = max((eachNode['cloudElementLatLonTRMM'])[2])
- # np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
maxCEprecip = np.max(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.max((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
minCEprecip = np.min(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.min((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
MCSPrecip.append((eachNode['uniqueID'],minCEprecip, maxCEprecip))
allMCSPrecip.append(MCSPrecip)
MCSPrecip =[]
@@ -2692,13 +2673,13 @@ def precipMaxMin(finalMCCList):
#******************************************************************
def displayPrecip(finalMCCList):
'''
- Purpose:: To create a figure showing the precip rate verse time for each MCS
+ Purpose::
+ To create a figure showing the precip rate verse time for each MCS
+
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
-
Output:: None
-
- Assumptions::
'''
timeList =[]
@@ -2728,7 +2709,6 @@ def displayPrecip(finalMCCList):
#in the graph and calculate the area
if finalMCCList:
- #print "len finalMCCList is: ", len(finalMCCList)
for eachMCC in finalMCCList:
#get the info from the node
for node in eachMCC:
@@ -2736,10 +2716,8 @@ def displayPrecip(finalMCCList):
if firstTime == True:
xStart = eachNode['cloudElementCenter'][1]#lon
yStart = eachNode['cloudElementCenter'][0]#lat
- #print "eachNode in temporalInfoMetric is ", eachNode['uniqueID'], eachNode['cloudElementTime']
timeList.append(eachNode['cloudElementTime'])
percentagePrecipitating.append((eachNode['TRMMArea']/eachNode['cloudElementArea'])*100.0)
- #areaCenter.append((eachNode['cloudElementCenter'][1]-xStart, eachNode['cloudElementCenter'][0]-yStart))
CEArea.append(eachNode['cloudElementArea'])
nodes.append(eachNode['uniqueID'])
x.append(eachNode['cloudElementCenter'][1])#-xStart)
@@ -2757,13 +2735,11 @@ def displayPrecip(finalMCCList):
#plot info
plt.close('all')
- #title = 'Area verses time for MCS'+ str(count)
title = 'Precipitation distribution of the MCS '
- #fig=plt.figure(facecolor='white', figsize=(10,10)) #figsize=(10,8))#figsize=(16,12))
fig,ax = plt.subplots(1, facecolor='white', figsize=(20,7))
cmap = cm.jet
- ax.scatter(x, y, s=partialArea, c= colorBarTime, edgecolors='none', marker='o', cmap =cmap) #,vmin=colorBarTime[0],vmax=colorBarTime[-1])
+ ax.scatter(x, y, s=partialArea, c= colorBarTime, edgecolors='none', marker='o', cmap =cmap)
colorBarTime=[]
colorBarTime =list(set(timeList))
colorBarTime.sort()
@@ -2799,11 +2775,14 @@ def displayPrecip(finalMCCList):
#******************************************************************
def plotPrecipHistograms(finalMCCList):
'''
- Purpose:: to create plots (histograms) of the each TRMMnetcdfCEs files
+ Purpose::
+ To create plots (histograms) of the each TRMMnetcdfCEs files
- Input:: finalMCCList: the list of strings representing the list of MCCs
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Output:: plots
+ Output::
+ plots
'''
num_bins = 5
precip =[]
@@ -2857,11 +2836,9 @@ def plotPrecipHistograms(finalMCCList):
imgFilename = MAINDIRECTORY+'/images/'+str(thisTime)+eachNode['uniqueID']+'TRMMMCS.gif'
plt.savefig(imgFilename, transparent=True)
- #plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
precip =[]
# ------ NETCDF File get info ------------------------------------
- #thisFileName = '/Users/kimwhitehall/Documents/HU/research/mccsearch/TRMMnetcdf/TRMM' + str(thisTime).replace(" ", "_") + eachNode['uniqueID'] +'.nc'
thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + str(thisTime).replace(" ", "_") + eachNode['uniqueID'] +'.nc'
TRMMData = Dataset(thisFileName,'r', format='NETCDF4')
precipRate = TRMMData.variables['precipitation_Accumulation'][:,:,:]
@@ -2872,12 +2849,10 @@ def plotPrecipHistograms(finalMCCList):
totalPrecip = np.add(totalPrecip, precipRate)
# ------ End NETCDF File ------------------------------------
- #if str(thisTime) == lastTime:
for index, value in np.ndenumerate(CEprecipRate):
if value != 0.0:
precip.append(value)
- #print "*******eachNode ", eachNode['uniqueID']
lastTime = str(thisTime)
firstTime = False
else:
@@ -2887,13 +2862,16 @@ def plotPrecipHistograms(finalMCCList):
#******************************************************************
def plotHistogram(aList, aTitle, aLabel):
'''
- Purpose:: to create plots (histograms) of the data entered in aList
+ Purpose::
+ To create plots (histograms) of the data entered in aList
- Input:: aList: the list of floating points representing values for e.g. averageArea, averageDuration, etc.
- aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
- aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
+ Input::
+ aList: the list of floating points representing values for e.g. averageArea, averageDuration, etc.
+ aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
+ aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
- Output:: plots (gif files)
+ Output::
+ plots (gif files)
'''
num_bins = 10
precip =[]
@@ -2921,32 +2899,32 @@ def plotHistogram(aList, aTitle, aLabel):
plt.xlim(min(binsdg), max(binsdg))
ax.set_xticks(binsdg)#, rotation=45)
ax.set_xlabel(aLabel, fontsize=12)
- #ax.set_ylabel('Frequency', fontsize=12)
ax.set_title(aTitle)
plt.xticks(rotation =45)
# Set the formatter
- #plt.gca().yaxis.set_major_formatter(formatter)
plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%0.0f'))
plt.subplots_adjust(bottom=0.2)
imgFilename = MAINDIRECTORY+'/images/'+aTitle.replace(" ","_")+'.gif'
- #plt.savefig(imgFilename, transparent=True)
plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
return
#******************************************************************
def plotAccTRMM (finalMCCList):
'''
- Purpose:: (1) generate a file with the accumulated precipiation for the MCS
- (2) generate the appropriate image
- TODO: NB: as the domain changes, will need to change XDEF and YDEF by hand to accomodate the new domain
- TODO: look into getting the info from the NETCDF file
-
- Input:: finalMCCList: a list of strings representing the nodes that make a MCC (or MCS)
-
- Output:: a netcdf file containing the accumulated precip
- a gif (generated in Grads)
+ Purpose::
+ (1) generate a file with the accumulated precipiation for the MCS
+ (2) generate the appropriate image
+ TODO: NB: as the domain changes, will need to change XDEF and YDEF by hand to accomodate the new domain
+ TODO: look into getting the info from the NETCDF file
+
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
+
+ Output::
+ a netcdf file containing the accumulated precip
+ a gif (generated in Grads)
'''
os.chdir((MAINDIRECTORY+'/TRMMnetcdfCEs'))
fname =''
@@ -3023,7 +3001,6 @@ def plotAccTRMM (finalMCCList):
subprocess.call('rm acc.ctl', shell=True)
subprocess.call('touch acc.ctl', shell=True)
replaceExpDset = 'echo DSET ' + accuTRMMFile +' >> acc.ctl'
- # replaceExpTdef = 'echo TDEF 99999 LINEAR 00Z 00000000 30mn' +' >> acc.ctl'
subprocess.call(replaceExpDset, shell=True)
subprocess.call('echo "OPTIONS yrev little_endian template" >> acc.ctl', shell=True)
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
@@ -3039,13 +3016,10 @@ def plotAccTRMM (finalMCCList):
subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
#generate GrADS script
- #print "thisNode ", thisNode
- #imgFilename = MAINDIRECTORY+'/images/MCSaccu'+firstPartName+str(thisNode['cloudElementTime']).replace(" ", "_")+'.gif'
subprocess.call('rm accuTRMM1.gs', shell=True)
subprocess.call('touch accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'reinit''\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'open acc.ctl ''\'" >> accuTRMM1.gs', shell=True)
- #subprocess.call('echo "''\'open '+accuTRMMFile+ '\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'set grads off''\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'set mpdset hires''\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'set gxout shaded''\'" >> accuTRMM1.gs', shell=True)
@@ -3066,13 +3040,16 @@ def plotAccTRMM (finalMCCList):
#******************************************************************
def plotAccuInTimeRange(starttime, endtime):
'''
- Purpose:: create accumulated precip plot within a time range given using all CEs
+ Purpose::
+ Create accumulated precip plot within a time range given using all CEs
- Input:: starttime: a string representing the time to start the accumulations format yyyy-mm-dd_hh:mm:ss
- endtime: a string representing the time to end the accumulations format yyyy-mm-dd_hh:mm:ss
+ Input::
+ starttime: a string representing the time to start the accumulations format yyyy-mm-dd_hh:mm:ss
+ endtime: a string representing the time to end the accumulations format yyyy-mm-dd_hh:mm:ss
- Output:: a netcdf file containing the accumulated precip for specified times
- a gif (generated in Grads)
+ Output::
+ a netcdf file containing the accumulated precip for specified times
+ a gif (generated in Grads)
TODO: pass of pick up from the NETCDF file the lat, lon and resolution for generating the ctl file
'''
@@ -3081,7 +3058,6 @@ def plotAccuInTimeRange(starttime, endtime):
#Just incase the X11 server is giving problems
subprocess.call('export DISPLAY=:0.0', shell=True)
- #fname =''
imgFilename = ''
firstPartName = ''
firstTime = True
@@ -3089,7 +3065,6 @@ def plotAccuInTimeRange(starttime, endtime):
fileList = []
sTime = datetime.strptime(starttime.replace("_"," "),'%Y-%m-%d %H:%M:%S')
eTime = datetime.strptime(endtime.replace("_"," "),'%Y-%m-%d %H:%M:%S')
- #tdelta = datetime.timedelta(0, 3600)
thisTime = sTime
while thisTime <= eTime:
@@ -3115,7 +3090,8 @@ def plotAccuInTimeRange(starttime, endtime):
thisTime +=timedelta(hours=TRES)
#create new netCDF file
- accuTRMMFile = MAINDIRECTORY+'TRMMnetcdfCEs/accu'+starttime+'-'+endtime+'.nc'
+ accuTRMMFile = MAINDIRECTORY+'/TRMMnetcdfCEs/accu'+starttime+'-'+endtime+'.nc'
+ print "accuTRMMFile ", accuTRMMFile
#write the file
accuTRMMData = Dataset(accuTRMMFile, 'w', format='NETCDF4')
accuTRMMData.description = 'Accumulated precipitation data'
@@ -3157,8 +3133,8 @@ def plotAccuInTimeRange(starttime, endtime):
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
subprocess.call('echo "UNDEF 0" >> acc.ctl', shell=True)
subprocess.call('echo "TITLE TRMM MCS accumulated precipitation" >> acc.ctl', shell=True)
- subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
- subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ subprocess.call('echo "XDEF 384 LINEAR -8.96875 0.036378335 " >> acc.ctl', shell=True)
+ subprocess.call('echo "YDEF 384 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
@@ -3190,22 +3166,22 @@ def plotAccuInTimeRange(starttime, endtime):
#******************************************************************
def createTextFile(finalMCCList, identifier):
'''
- Purpose:: Create a text file with information about the MCS
- This function is expected to be especially of use regarding
- long term record checks
+ Purpose::
+ Create a text file with information about the MCS
+ This function is expected to be especially of use regarding long term record checks
- Input:: finalMCCList: list of list of strings representing list of MCSs nodes list
- identifier: an integer representing the type of list that has been entered...this is for creating file purposes
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
+ identifier: an integer representing the type of list that has been entered...this is for creating file purposes
1 - MCCList; 2- MCSList
Output::
- a user readable text file with all information about each MCS
- a user readable text file with the summary of the MCS
+ a user readable text file with all information about each MCS
+ a user readable text file with the summary of the MCS
Assumptions::
'''
- #allTimes =[]
durations=0.0
startTimes =[]
endTimes=[]
@@ -3213,7 +3189,6 @@ def createTextFile(finalMCCList, identifier):
speedCounter = 0
maxArea =0.0
amax = 0.0
- #avgMaxArea = 0.0
avgMaxArea =[]
maxAreaCounter =0.0
maxAreaTime=''
@@ -3238,7 +3213,6 @@ def createTextFile(finalMCCList, identifier):
bigPtotal = 0.0
bigPtotalCounter = 0
allPropagationSpeeds =[]
- #averageAreas = 0.0
averageAreas =[]
areaAvg = 0.0
avgPrecipTotal = 0.0
@@ -3249,6 +3223,7 @@ def createTextFile(finalMCCList, identifier):
avgMinMCSPrecipRateCounter = 0
minMax =[]
avgPrecipArea = []
+ location =[]
avgPrecipAreaPercent = 0.0
precipArea = 0.0
precipAreaPercent = 0.0
@@ -3266,9 +3241,6 @@ def createTextFile(finalMCCList, identifier):
MCSUserFile = open((MAINDIRECTORY+'/textFiles/MCSsUserFile.txt'),'wb')
MCSSummaryFile = open((MAINDIRECTORY+'/textFiles/MCSSummary.txt'),'wb')
-
- #MCCsFile = open((MAINDIRECTORY+'/textFiles/MCCsFile.txt'),'wb')
-
for eachPath in finalMCCList:
eachPath.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0], nodeID.split('CE')[1]))
@@ -3284,16 +3256,13 @@ def createTextFile(finalMCCList, identifier):
endTimes.append(endTime)
#get the precip info
- #MCSPrecip = precipMaxMin(eachPath)
-
+
for eachNode in eachPath:
thisNode = thisDict(eachNode)
#set first time min "fake" values
if firstTime == True:
- # minCEprecipRate = MCSPrecip[0][1]
- # avgMinMCSPrecipRate = MCSPrecip[0][1]
minCEprecipRate = thisNode['CETRMMmin']
avgMinMCSPrecipRate += thisNode['CETRMMmin']
firstTime = False
@@ -3306,27 +3275,19 @@ def createTextFile(finalMCCList, identifier):
#Amax: find max area
if thisNode['cloudElementArea'] > maxArea:
maxArea = thisNode['cloudElementArea']
- # avgMaxArea += maxArea
- # maxAreaCounter += 1
maxAreaTime = str(thisNode['cloudElementTime'])
eccentricity = thisNode['cloudElementEccentricity']
-
+ location = thisNode['cloudElementCenter']
+
#determine the time the feature matures
if matureFlag == True:
timeMCSMatures = str(thisNode['cloudElementTime'])
matureFlag = False
- #find min and max precip rate
- # minMax = list(value_tuple for node,value_tuple in enumerate(MCSPrecip) if value_tuple[0] == eachNode)
- # if minMax[0][1] < minCEprecipRate:
- # minCEprecipRate = minMax[0][1]
- # print "minCEprecipRate ", minCEprecipRate
- # if minMa
<TRUNCATED>
[17/50] [abbrv] git commit: Merge git://git.apache.org/climate into
mccsearch
Posted by le...@apache.org.
Merge git://git.apache.org/climate into mccsearch
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/a5b7677b
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/a5b7677b
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/a5b7677b
Branch: refs/heads/master
Commit: a5b7677b528197daf1dee5cb05adeac35bc4846d
Parents: f21e24d b6db8f8
Author: kwhitehall <k_...@yahoo.com>
Authored: Thu May 15 15:23:27 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Thu May 15 15:23:27 2014 -0400
----------------------------------------------------------------------
ocw-ui/backend/rcmed_helpers.py | 1 +
ocw-ui/backend/tests/__init__.py | 16 ++++
ocw-ui/backend/tests/test_directory_helpers.py | 85 +++++++++++++-------
.../test_local_file_metadata_extractors.py | 17 ++++
ocw-ui/backend/tests/test_processing.py | 58 +++++--------
ocw-ui/backend/tests/test_run_webservices.py | 26 ++++--
ocw/evaluation.py | 2 +
setup.py | 6 +-
8 files changed, 135 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
[24/50] [abbrv] git commit: Fix formatting, remove unused imports,
update files.py and process.py to most recent OCW code
Posted by le...@apache.org.
Fix formatting, remove unused imports, update files.py and process.py to most recent OCW code
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/4d64d214
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/4d64d214
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/4d64d214
Branch: refs/heads/master
Commit: 4d64d2142736aa7b1a3d1e83b8f56322c4dba128
Parents: abcc73c
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 16:45:06 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Mon Aug 25 16:45:06 2014 -0700
----------------------------------------------------------------------
mccsearch/code/__init__.py | 0
mccsearch/code/files.py | 375 ++++++----
mccsearch/code/mainProg.py | 19 +-
mccsearch/code/mainProgTemplate.py | 16 +
mccsearch/code/mccSearch.py | 449 ++++++------
mccsearch/code/mccSearchUI.py | 16 +
mccsearch/code/process.py | 1221 ++++++++++++-------------------
mccsearch/docs/mccsearch.md | 18 +-
8 files changed, 992 insertions(+), 1122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/code/__init__.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/__init__.py b/mccsearch/code/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/code/files.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.py b/mccsearch/code/files.py
index 82b508a..b238754 100644
--- a/mccsearch/code/files.py
+++ b/mccsearch/code/files.py
@@ -1,25 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
"""
-Module for handling data input files. Requires PyNIO and Numpy be
+Module for handling data input files. Requires netCDF and Numpy be
installed.
-This module can easily open NetCDF, HDF and Grib files. Search the PyNIO
+This module can easily open NetCDF, HDF and Grib files. Search the netCDF4
documentation for a complete list of supported formats.
"""
from os import path
-
-try:
- import Nio
-except ImportError:
- import nio as Nio
-
+import netCDF4
import numpy as np
import numpy.ma as ma
import sys
-import process #KDW added
-#from toolkit import process
-#from utils import fortranfile
-#from utils import misc
+
+from toolkit import process
+from utils import fortranfile
+from utils import misc
VARIABLE_NAMES = {'time': ['time', 'times', 'date', 'dates', 'julian'],
@@ -48,15 +59,14 @@ def getVariableByType(filename, variableType):
name match cannot be found.
"""
try:
- f = Nio.open_file(filename)
+ f = netCDF4.Dataset(filename, mode='r')
except:
- #print 'PyNio had an issue opening the filename (%s) you provided' % filename
- print "NIOError:", sys.exc_info()[0]
+ print "netCDF4Error:", sys.exc_info()[0]
raise
variableKeys = f.variables.keys()
f.close()
- variableKeys = [variable.lower() for variable in variableKeys]
+ variableKeys = [variable.encode().lower() for variable in variableKeys]
variableMatch = VARIABLE_NAMES[variableType]
commonVariables = list(set(variableKeys).intersection(variableMatch))
@@ -80,10 +90,9 @@ def getVariableRange(filename, variableName):
variableRange - tuple of order (variableMin, variableMax)
"""
try:
- f = Nio.open_file(filename)
+ f = netCDF4.Dataset(filename, mode='r')
except:
- #print 'PyNio had an issue opening the filename (%s) you provided' % filename
- print "NIOError:", sys.exc_info()[0]
+ print "netCDF4Error:", sys.exc_info()[0]
raise
varArray = f.variables[variableName][:]
@@ -121,13 +130,9 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
# ii) find out how many timesteps in the file
# (assume same ntimes in each file in list)
# -allows you to create an empty array to store variable data for all times
- tmp = Nio.open_file(filename)
+ tmp = netCDF4.Dataset(filename, mode='r')
latsraw = tmp.variables[latVarName][:]
lonsraw = tmp.variables[lonVarName][:]
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
-
- """TODO: Guard against case where latsraw and lonsraw are not the same dim?"""
-
if(latsraw.ndim == 1):
lon, lat = np.meshgrid(lonsraw, latsraw)
if(latsraw.ndim == 2):
@@ -137,7 +142,7 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
timesraw = tmp.variables[timeVarName]
ntimes = len(timesraw)
- print 'Lats and lons read in for first file in filelist', ntimes
+ print 'Lats and lons read in for first file in filelist'
# Create a single empty masked array to store model data from all files
t2store = ma.zeros((ntimes * len(filelist), len(lat[:, 0]), len(lon[0, :])))
@@ -151,16 +156,16 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
# as no assumption made that same number of times in each file...
- for ifile in filelist:
+ for i, ifile in enumerate(filelist):
#print 'Loading data from file: ',filelist[i]
- f = Nio.open_file(ifile)
- t2raw = f.variables[myvar][:]
+ f = netCDF4.Dataset(ifile, mode='r')
+ t2raw = ma.array(f.variables[myvar][:])
timesraw = f.variables[timeVarName]
time = timesraw[:]
ntimes = len(time)
print 'file= ', i, 'ntimes= ', ntimes, filelist[i]
- print '\nt2raw shape: ', t2raw.shape
+ print 't2raw shape: ', t2raw.shape
# Flatten dimensions which needn't exist, i.e. level
# e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
@@ -175,32 +180,26 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
if t2tmp.ndim == 2:
t2tmp = np.expand_dims(t2tmp, 0)
- print "\n timesaccu is: ", timesaccu, time,timesaccu + np.arange(ntimes)
-
t2store[timesaccu + np.arange(ntimes), :, :] = t2tmp[:, :, :]
timestore[timesaccu + np.arange(ntimes)] = time
- timesaccu = timesaccu + ntimes
- print "\n ***timesaccu is: ", timesaccu
+ timesaccu += ntimes
f.close()
- i += 1
-
- print '\nData read in successfully with dimensions: ', t2store.shape, timesaccu
+
+ print 'Data read in successfully with dimensions: ', t2store.shape
# TODO: search for duplicated entries (same time) and remove duplicates.
# Check to see if number of unique times == number of times, if so then no problem
if(len(np.unique(timestore)) != len(np.where(timestore != 0)[0].view())):
- print '\nWARNING: Possible duplicated times'
+ print 'WARNING: Possible duplicated times'
# Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
timestore, _ = process.getModelTimes(filename, timeVarName)
- data_dict = {}
- data_dict['lats'] = lat
- data_dict['lons'] = lon
- data_dict['times'] = timestore
- data_dict['data'] = t2store
- #return lat, lon, timestore, t2store
+ # Make sure latlon grid is monotonically increasing and that the domains
+ # are correct
+ lat, lon, t2store = checkLatLon(lat, lon, t2store)
+ data_dict = {'lats': lat, 'lons': lon, 'times': timestore, 'data': t2store}
return data_dict
def select_var_from_file(myfile, fmt='not set'):
@@ -209,7 +208,7 @@ def select_var_from_file(myfile, fmt='not set'):
Input:
myfile - filename
- fmt - (optional) specify fileformat for PyNIO if filename suffix is non-standard
+ fmt - (optional) specify fileformat for netCDF4 if filename suffix is non-standard
Output:
myvar - variable name in file
@@ -220,12 +219,12 @@ def select_var_from_file(myfile, fmt='not set'):
print fmt
if fmt == 'not set':
- f = Nio.open_file(myfile)
+ f = netCDF4.Dataset(myfile, mode='r')
if fmt != 'not set':
- f = Nio.open_file(myfile, format=fmt)
+ f = netCDF4.Dataset(myfile, mode='r', format=fmt)
- keylist = f.variables.keys()
+ keylist = [key.encode().lower() for key in f.variables.keys()]
i = 0
for v in keylist:
@@ -251,9 +250,8 @@ def select_var_from_wrf_file(myfile):
Peter Lean September 2010
'''
- f = Nio.open_file(myfile, format='nc')
-
- keylist = f.variables.keys()
+ f = netCDF4.Dataset(myfile, mode='r', format='NETCDF4')
+ keylist = [key.encode().lower() for key in f.variables.keys()]
i = 0
for v in keylist:
@@ -270,68 +268,50 @@ def select_var_from_wrf_file(myfile):
return mywrfvar
-def read_lolaT_from_file(filename, latVarName, lonVarName, timeVarName, file_type):
+def read_data_from_one_file(ifile, myvar, latVarName, lonVarName, timeVarName, file_type):
"""
- Function that will return lat, lon, and time arrays
+ Purpose::
+ Read in data from one file at a time
- Input::
- filename - the file to inspect
- latVarName - name of the Latitude Variable
+ Input::
+ filelist - list of filenames (including path)
+ myvar - string containing name of variable to load in (as it appears in file)s
lonVarName - name of the Longitude Variable
timeVarName - name of the Time Variable
- fileType = type of file we are trying to parse
-
- Output::
- lat - MESH GRID of Latitude values with shape (nx, ny)
- lon - MESH GRID of Longitude values with shape (nx, ny)
- timestore - Python list of Datetime objects
-
- MESHGRID docs: http://docs.scipy.org/doc/numpy/reference/generated/numpy.meshgrid.html
+ fileType - type of file we are trying to parse
- """
-
- tmp = Nio.open_file(filename, format=file_type)
- lonsraw = tmp.variables[lonVarName][:]
- latsraw = tmp.variables[latVarName][:]
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
- if(latsraw.ndim == 1):
- lon, lat = np.meshgrid(lonsraw, latsraw)
- if(latsraw.ndim == 2):
- lon = lonsraw
- lat = latsraw
- timestore, _ = process.getModelTimes(filename, timeVarName)
- print ' read_lolaT_from_file: Lats, lons and times read in for the model domain'
- return lat, lon, timestore
-
-def read_data_from_one_file(ifile, myvar, timeVarName, lat, file_type):
- ##################################################################################
- # Read in data from one file at a time
- # Input: filelist - list of filenames (including path)
- # myvar - string containing name of variable to load in (as it appears in file)
- # Output: lat, lon - 2D array of latitude and longitude values
- # times - list of times
- # t2store - numpy array containing data from all files
- # Modified from read_data_from_file_list to read data from multiple models into a 4-D array
- # 1. The code now processes model data that completely covers the 20-yr period. Thus,
- # all model data must have the same time levels (ntimes). Unlike in the oroginal, ntimes
- # is fixed here.
- # 2. Because one of the model data exceeds 240 mos (243 mos), the model data must be
- # truncated to the 240 mons using the ntimes determined from the first file.
- ##################################################################################
- f = Nio.open_file(ifile)
+ Output::
+ lat, lon - 2D arrays of latitude and longitude values
+ times - list of times
+ t2store - numpy array containing data from the file for the requested variable
+ varUnit - units for the variable given by t2store
+ """
+ f = netCDF4.Dataset(ifile, mode='r')
try:
- varUnit = f.variables[myvar].units.upper()
+ varUnit = f.variables[myvar].units.encode().upper()
except:
varUnit = raw_input('Enter the model variable unit: \n> ').upper()
- t2raw = f.variables[myvar][:]
+ t2raw = ma.array(f.variables[myvar][:])
t2tmp = t2raw.squeeze()
if t2tmp.ndim == 2:
t2tmp = np.expand_dims(t2tmp, 0)
- t2tmp = t2tmp
+
+ lonsraw = f.variables[lonVarName][:]
+ latsraw = f.variables[latVarName][:]
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
+
f.close()
print ' success read_data_from_one_file: VarName=', myvar, ' Shape(Full)= ', t2tmp.shape, ' Unit= ', varUnit
timestore = process.decode_model_timesK(ifile, timeVarName, file_type)
- return timestore, t2tmp, varUnit
+
+ # Make sure latlon grid is monotonically increasing and that the domains
+ # are correct
+ lat, lon, t2store = checkLatLon(lat, lon, t2tmp)
+ return lat, lon, timestore, t2store, varUnit
def findTimeVariable(filename):
"""
@@ -343,12 +323,12 @@ def findTimeVariable(filename):
variableNameList - list of variable names from the input filename
"""
try:
- f = Nio.open_file(filename, mode='r')
+ f = netCDF4.Dataset(filename, mode='r')
except:
print("Unable to open '%s' to try and read the Time variable" % filename)
raise
- variableNameList = f.variables.keys()
+ variableNameList = [variable.encode() for variable in f.variables.keys()]
# convert all variable names into lower case
varNameListLowerCase = [x.lower() for x in variableNameList]
@@ -383,12 +363,12 @@ def findLatLonVarFromFile(filename):
-lonMax
"""
try:
- f = Nio.open_file(filename, mode='r')
+ f = netCDF4.Dataset(filename, mode='r')
except:
print("Unable to open '%s' to try and read the Latitude and Longitude variables" % filename)
raise
- variableNameList = f.variables.keys()
+ variableNameList = [variable.encode() for variable in f.variables.keys()]
# convert all variable names into lower case
varNameListLowerCase = [x.lower() for x in variableNameList]
@@ -449,16 +429,17 @@ def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarN
# i) read in lats, lons
# ii) find out how many timesteps in the file (assume same ntimes in each file in list)
# -allows you to create an empty array to store variable data for all times
- tmp = Nio.open_file(filelist[0], format=file_type)
+ tmp = netCDF4.Dataset(filelist[0], mode='r', format=file_type)
latsraw = tmp.variables[latVarName][:]
lonsraw = tmp.variables[lonVarName][:]
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ timesraw = tmp.variables[timeVarName]
+
if(latsraw.ndim == 1):
lon, lat = np.meshgrid(lonsraw, latsraw)
- if(latsraw.ndim == 2):
- lon = lonsraw; lat = latsraw
-
- timesraw = tmp.variables[timeVarName]
+
+ elif(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
ntimes = len(timesraw); nygrd = len(lat[:, 0]); nxgrd = len(lon[0, :])
print 'Lats and lons read in for first file in filelist'
@@ -474,16 +455,13 @@ def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarN
# NB. this method allows for missing times in data files
# as no assumption made that same number of times in each file...
- i = 0
- for ifile in filelist:
+ for i, ifile in enumerate(filelist):
#print 'Loading data from file: ',filelist[i]
- f = Nio.open_file(ifile)
- t2raw = f.variables[myvar][:]
+ f = netCDF4.Dataset(ifile, mode='r')
+ t2raw = ma.array(f.variables[myvar][:])
timesraw = f.variables[timeVarName]
- #time = timesraw[0:ntimes]
- ntimes = len(timesraw)
#ntimes=len(time)
- print 'file= ',i,' ntimes= ',ntimes,filelist[i]
+ #print 'file= ',i,'ntimes= ',ntimes,filelist[i]
## Flatten dimensions which needn't exist, i.e. level
## e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
## Code requires data to have dimensions, (time,lat,lon)
@@ -493,18 +471,20 @@ def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarN
if t2tmp.ndim == 2:
t2tmp = np.expand_dims(t2tmp, 0)
#t2store[timesaccu+np.arange(ntimes),:,:]=t2tmp[0:ntimes,:,:]
- t2store[i, 0:(ntimes-1), :, :] = t2tmp[0:(ntimes-1), :, :]
+ t2store[i, 0:ntimes, :, :] = t2tmp[0:ntimes, :, :]
#timestore[timesaccu+np.arange(ntimes)]=time
#timesaccu=timesaccu+ntimes
f.close()
- i += 1
print 'Data read in successfully with dimensions: ', t2store.shape
# Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
ifile = filelist[0]
timestore, _ = process.getModelTimes(ifile, timeVarName)
-
+
+ # Make sure latlon grid is monotonically increasing and that the domains
+ # are correct
+ lat, lon, t2store = checkLatLon(lat, lon, t2store)
return lat, lon, timestore, t2store
def find_latlon_ranges(filelist, lat_var_name, lon_var_name):
@@ -524,7 +504,7 @@ def find_latlon_ranges(filelist, lat_var_name, lon_var_name):
filename = filelist[0]
try:
- f = Nio.open_file(filename)
+ f = netCDF4.Dataset(filename, mode='r')
lats = f.variables[lat_var_name][:]
latMin = lats.min()
@@ -622,30 +602,30 @@ def writeNCfile(fileName, numSubRgn, lons, lats, obsData, mdlData, obsRgnAvg, md
dimY = mdlData.shape[2] # y-dimension
dimX = mdlData.shape[3] # x-dimension
dimR = obsRgnAvg.shape[1] # the number of subregions
- f = Nio.open_file(fileName, mode='w', format='nc')
+ f = netCDF4.Dataset(fileName, mode='w', format='NETCDF4')
print mdlRgnAvg.shape, dimM, dimR, dimT
#create global attributes
- f.globalAttName = ''
+ f.description = ''
# create dimensions
print 'Creating Dimensions within the NetCDF Object...'
- f.create_dimension('unity', 1)
- f.create_dimension('time', dimT)
- f.create_dimension('west_east', dimX)
- f.create_dimension('south_north', dimY)
- f.create_dimension('obs', dimO)
- f.create_dimension('models', dimM)
+ f.createDimension('unity', 1)
+ f.createDimension('time', dimT)
+ f.createDimension('west_east', dimX)
+ f.createDimension('south_north', dimY)
+ f.createDimension('obs', dimO)
+ f.createDimension('models', dimM)
# create the variable (real*4) to be written in the file
print 'Creating Variables...'
- f.create_variable('lon', 'd', ('south_north', 'west_east'))
- f.create_variable('lat', 'd', ('south_north', 'west_east'))
- f.create_variable('oDat', 'd', ('obs', 'time', 'south_north', 'west_east'))
- f.create_variable('mDat', 'd', ('models', 'time', 'south_north', 'west_east'))
+ f.createVariable('lon', 'd', ('south_north', 'west_east'))
+ f.createVariable('lat', 'd', ('south_north', 'west_east'))
+ f.createVariable('oDat', 'd', ('obs', 'time', 'south_north', 'west_east'))
+ f.createVariable('mDat', 'd', ('models', 'time', 'south_north', 'west_east'))
if subRegions:
- f.create_dimension('regions', dimR)
- f.create_variable('oRgn', 'd', ('obs', 'regions', 'time'))
- f.create_variable('mRgn', 'd', ('models', 'regions', 'time'))
+ f.createDimension('regions', dimR)
+ f.createVariable('oRgn', 'd', ('obs', 'regions', 'time'))
+ f.createVariable('mRgn', 'd', ('models', 'regions', 'time'))
f.variables['oRgn'].varAttName = 'Observation time series: Subregions'
f.variables['mRgn'].varAttName = 'Model time series: Subregions'
@@ -664,33 +644,140 @@ def writeNCfile(fileName, numSubRgn, lons, lats, obsData, mdlData, obsRgnAvg, md
f.variables['mDat'].varAttName = 'Model time series: entire domain'
# assign the values to the variable and write it
- f.variables['lon'][:, :] = lons
- f.variables['lat'][:, :] = lats
+ f.variables['lon'][:] = lons[:]
+ f.variables['lat'][:] = lats[:]
if subRegions:
- f.variables['oRgn'][:, :, :] = obsRgnAvg
- f.variables['mRgn'][:, :, :] = mdlRgnAvg
+ f.variables['oRgn'][:] = obsRgnAvg[:]
+ f.variables['mRgn'][:] = mdlRgnAvg[:]
f.close()
def loadDataIntoNetCDF(fileObject, datasets, dataArray, dataType):
"""
Input::
- fileObject - PyNIO file object data will be loaded into
+ fileObject - netCDF4 file object data will be loaded into
datasets - List of dataset names
dataArray - Multi-dimensional array of data to be loaded into the NetCDF file
dataType - String with value of either 'Model' or 'Observation'
Output::
- No return value. PyNIO file object is updated in place
+ No return value. netCDF4 file object is updated in place
"""
datasetCount = 0
- for dataset in datasets:
+ for datasetCount, dataset in enumerate(datasets):
if dataType.lower() == 'observation':
datasetName = dataset.replace(' ','')
elif dataType.lower() == 'model':
datasetName = path.splitext(path.basename(dataset))[0]
print "Creating variable %s" % datasetName
- fileObject.create_variable(datasetName, 'd', ('time', 'south_north', 'west_east'))
+ fileObject.createVariable(datasetName, 'd', ('time', 'south_north', 'west_east'))
fileObject.variables[datasetName].varAttName = 'Obseration time series: entire domain'
print 'Loading values into %s' % datasetName
- fileObject.variables[datasetName].assign_value(dataArray[datasetCount,:,:,:])
- datasetCount += 1
+ fileObject.variables[datasetName][:] = dataArray[datasetCount,:,:,:]
+
+def checkLatLon(latsin, lonsin, datain):
+ """
+ Purpose::
+ Checks whether latitudes and longitudes are monotonically increasing
+ within the domains [-90, 90) and [-180, 180) respectively, and rearranges the input data
+ accordingly if they are not.
+
+ Input::
+ latsin - Array of latitudes read from a raw netcdf file
+ lonsin - Array of longitudes read from a raw netcdf file
+ datain - Array of data values read from a raw netcdf file.
+ The shape is assumed to be (..., nLat, nLon).
+
+ Output::
+ latsout - 2D array of (rearranged) latitudes
+ lonsout - 2D array of (rearranged) longitudes
+ dataout - Array of (rearranged) data
+ """
+ # Avoid unnecessary shifting if all lons are higher than 180
+ if lonsin.min() > 180:
+ lonsin -= 360
+
+ # Make sure lats and lons are monotonically increasing
+ latsDecreasing = np.diff(latsin[:, 0]) < 0
+ lonsDecreasing = np.diff(lonsin[0]) < 0
+
+ # If all values are decreasing then they just need to be reversed
+ latsReversed, lonsReversed = latsDecreasing.all(), lonsDecreasing.all()
+
+ # If the lat values are unsorted then raise an exception
+ if not latsReversed and latsDecreasing.any():
+ raise ValueError('Latitudes must be monotonically increasing.')
+
+ # Perform same checks now for lons
+ if not lonsReversed and lonsDecreasing.any():
+ raise ValueError('Longitudes must be monotonically increasing.')
+
+ # Also check if lons go from [0, 360), and convert to [-180, 180)
+ # if necessary
+ lonsShifted = lonsin.max() > 180
+ latsout, lonsout, dataout = latsin[:], lonsin[:], datain[:]
+ # Now correct data if latlon grid needs to be shifted
+ if latsReversed:
+ latsout = latsout[::-1]
+ dataout = dataout[..., ::-1, :]
+
+ if lonsReversed:
+ lonsout = lonsout[..., ::-1]
+ dataout = dataout[..., ::-1]
+
+ if lonsShifted:
+ lat1d = latsout[:, 0]
+ dataout, lon1d = shiftgrid(180, dataout, lonsout[0], start=False)
+ lonsout, latsout = np.meshgrid(lon1d, lat1d)
+
+ return latsout, lonsout, dataout
+
+def shiftgrid(lon0, datain, lonsin, start= True, cyclic=360.0):
+ """
+ Purpose::
+ Shift global lat/lon grid east or west. This function is taken directly
+ from the (unreleased) basemap 1.0.7 source code as version 1.0.6 does not
+ currently support arrays with more than two dimensions.
+ https://github.com/matplotlib/basemap
+
+ Input::
+ lon0 - starting longitude for shifted grid (ending longitude if start=False).
+ lon0 must be on input grid (within the range of lonsin).
+ datain - original data with longitude the right-most dimension.
+ lonsin - original longitudes.
+ start - if True, lon0 represents the starting longitude of the new grid.
+ if False, lon0 is the ending longitude. Default True.
+ cyclic - width of periodic domain (default 360)
+
+ Output::
+ dataout - data on shifted grid
+ lonsout - lons on shifted grid
+ """
+ if np.fabs(lonsin[-1]-lonsin[0]-cyclic) > 1.e-4:
+ # Use all data instead of raise ValueError, 'cyclic point not included'
+ start_idx = 0
+ else:
+ # If cyclic, remove the duplicate point
+ start_idx = 1
+ if lon0 < lonsin[0] or lon0 > lonsin[-1]:
+ raise ValueError('lon0 outside of range of lonsin')
+ i0 = np.argmin(np.fabs(lonsin-lon0))
+ i0_shift = len(lonsin)-i0
+ if ma.isMA(datain):
+ dataout = ma.zeros(datain.shape,datain.dtype)
+ else:
+ dataout = np.zeros(datain.shape,datain.dtype)
+ if ma.isMA(lonsin):
+ lonsout = ma.zeros(lonsin.shape,lonsin.dtype)
+ else:
+ lonsout = np.zeros(lonsin.shape,lonsin.dtype)
+ if start:
+ lonsout[0:i0_shift] = lonsin[i0:]
+ else:
+ lonsout[0:i0_shift] = lonsin[i0:]-cyclic
+ dataout[...,0:i0_shift] = datain[...,i0:]
+ if start:
+ lonsout[i0_shift:] = lonsin[start_idx:i0+start_idx]+cyclic
+ else:
+ lonsout[i0_shift:] = lonsin[start_idx:i0+start_idx]
+ dataout[...,i0_shift:] = datain[...,start_idx:i0+start_idx]
+ return dataout,lonsout
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 43ca921..bb032fc 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -1,10 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
'''
# running the program
'''
import sys
import networkx as nx
-import mccSearch
+from mccSearch
import numpy as np
import numpy.ma as ma
import files
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/code/mainProgTemplate.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemplate.py b/mccsearch/code/mainProgTemplate.py
index 9be65b5..d43788e 100644
--- a/mccsearch/code/mainProgTemplate.py
+++ b/mccsearch/code/mainProgTemplate.py
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
'''
# running the program
'''
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index b7dc4eb..e691ed5 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -1,18 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
'''
# All the functions for the MCC search algorithm
# Following RCMES dataformat in format (t,lat,lon), value
'''
-import datetime
from datetime import timedelta, datetime
-import calendar
-import fileinput
import glob
import itertools
-import json
-import math
-import Nio
-from netCDF4 import Dataset, num2date, date2num
+from netCDF4 import Dataset, date2num
import numpy as np
import numpy.ma as ma
import os
@@ -25,10 +35,10 @@ import sys
import time
import networkx as nx
-import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from matplotlib.dates import DateFormatter,HourLocator
-from matplotlib import cm
+
+import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib.colors as mcolors
from matplotlib.ticker import FuncFormatter, FormatStrFormatter
@@ -48,14 +58,14 @@ XRES = 4.0 #x direction spatial resolution in km
YRES = 4.0 #y direction spatial resolution in km
TRES = 1 #temporal resolution in hrs
LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
-LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
- #have same rank of the matrix it is being compared against
+ #have same rank of the matrix it is being compared against
#criteria for determining cloud elements and edges
T_BB_MAX = 243 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
T_BB_MIN = 218 #cooler temp for the center of the system
CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
-MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
+MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
@@ -80,19 +90,19 @@ PRUNED_GRAPH = nx.DiGraph()
def readMergData(dirname, filelist = None):
'''
Purpose::
- Read MERG data into RCMES format
+ Read MERG data into RCMES format
Input::
- dirname: a string representing the directory to the MERG files in NETCDF format
- filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
+ dirname: a string representing the directory to the MERG files in NETCDF format
+ filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
Output::
- A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
- criteria for each frame
+ A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
+ criteria for each frame
Assumptions::
- The MERG data has been converted to NETCDF using LATS4D
- The data has the same lat/lon format
+ The MERG data has been converted to NETCDF using LATS4D
+ The data has the same lat/lon format
TODO:: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
@@ -129,8 +139,9 @@ def readMergData(dirname, filelist = None):
sys.exit()
else:
# Open the first file in the list to read in lats, lons and generate the grid for comparison
- tmp = Nio.open_file(filelist[0], format='nc')
-
+ #tmp = Nio.open_file(filelist[0], format='nc')
+ tmp = Dataset(filelist[0], format='NETCDF4')
+
#clip the lat/lon grid according to user input
#http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
@@ -146,11 +157,12 @@ def readMergData(dirname, filelist = None):
for files in filelist:
try:
- thisFile = Nio.open_file(files, format='nc')
+ #thisFile = Nio.open_file(files, format='nc')
+ thisFile = Dataset(files, format='NETCDF4')
#clip the dataset according to user lat,lon coordinates
#mask the data and fill with zeros for later
tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
- +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
+ +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
@@ -180,8 +192,8 @@ def readMergData(dirname, filelist = None):
def findCloudElements(mergImgs,timelist,TRMMdirName=None):
'''
Purpose::
- Determines the contiguous boxes for a given time of the satellite images i.e. each frame
- using scipy ndimage package
+ Determines the contiguous boxes for a given time of the satellite images i.e. each frame
+ using scipy ndimage package
Input::
mergImgs: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
@@ -190,8 +202,8 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
TRMMdirName (optional): string representing the path where to find the TRMM datafiles
Output::
- CLOUD_ELEMENT_GRAPH: a Networkx directed graph where each node contains the information in cloudElementDict
- The nodes are determined according to the area of contiguous squares. The nodes are linked through weighted edges.
+ CLOUD_ELEMENT_GRAPH: a Networkx directed graph where each node contains the information in cloudElementDict
+ The nodes are determined according to the area of contiguous squares. The nodes are linked through weighted edges.
cloudElementDict = {'uniqueID': unique tag for this CE,
'cloudElementTime': time of the CE,
@@ -207,9 +219,9 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
'CETRMMmax':floating-point representing the max rate in the CE if TRMMdirName entered,
'CETRMMmin':floating-point representing the min rate in the CE if TRMMdirName entered}
Assumptions::
- Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
- required according to Vila et al. (2008) is 2400km^2
- therefore, 2400/16 = 150 contiguous squares
+ Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
+ required according to Vila et al. (2008) is 2400km^2
+ therefore, 2400/16 = 150 contiguous squares
'''
frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
@@ -293,7 +305,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
#-------------------------------------------------
- #textfile name for accesing CE data using MATLAB code
+ #textfile name for accesing CE data using MATLAB code
# thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
# cloudElementsTextFile = open(thisFileName,'w')
#-------------------------------------------------
@@ -417,8 +429,8 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#-----------End most of NETCDF file stuff ------------------------------------
#populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
for index,value in np.ndenumerate(cloudElement):
if value != 0 :
lat_index,lon_index = index
@@ -443,7 +455,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#calculate the total precip associated with the feature
for index, value in np.ndenumerate(finalCETRMMvalues):
precipTotal += value
- precip.append(value)
+ precip.append(value)
rainFallacc[:] = finalCETRMMvalues[:]
currNetCDFTRMMData.close()
@@ -460,19 +472,19 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#determine if the cloud element the shape
cloudElementEpsilon = eccentricity (cloudElement)
- cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
- cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
-
- #latCenter and lonCenter are given according to the particular array defining this CE
- #so you need to convert this value to the overall domain truth
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementCenter.append(latCenter)
- cloudElementCenter.append(lonCenter)
- cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementCenter.append(latCenter)
+ cloudElementCenter.append(lonCenter)
+ cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
@@ -507,42 +519,42 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
elif areaOverlap >= MIN_OVERLAP:
CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
- else:
- #TODO: remove this else as we only wish for the CE details
- #ensure only the non-zero elements are considered
- #store intel in allCE file
- labels, _ = ndimage.label(cloudElement)
- cloudElementsFile.write("\n-----------------------------------------------")
- cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
- # cloudElementLat = LAT[loc[0],0]
- # cloudElementLon = LON[0,loc[1]]
-
- #populate cloudElementLatLons by unpacking the original values from loc
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
- for index,value in np.ndenumerate(cloudElement):
- if value != 0 :
- lat_index,lon_index = index
- lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
- cloudElementLatLons.append(lat_lon_tuple)
-
- cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
- #latCenter and lonCenter are given according to the particular array defining this CE
+ else:
+ #TODO: remove this else as we only wish for the CE details
+ #ensure only the non-zero elements are considered
+ #store intel in allCE file
+ labels, _ = ndimage.label(cloudElement)
+ cloudElementsFile.write("\n-----------------------------------------------")
+ cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ # cloudElementLat = LAT[loc[0],0]
+ # cloudElementLon = LON[0,loc[1]]
+
+ #populate cloudElementLatLons by unpacking the original values from loc
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
+ #latCenter and lonCenter are given according to the particular array defining this CE
#so you need to convert this value to the overall domain truth
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
- cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
- cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
- cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
- cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
- cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
- cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
- cloudElementsFile.write("\n-----------------------------------------------")
-
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ cloudElementsFile.write("\n-----------------------------------------------")
+
#reset list for the next CE
nodeExist = False
cloudElementCenter=[]
@@ -563,7 +575,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
prevFrameCEs =[]
prevFrameCEs = currFrameCEs
currFrameCEs =[]
-
+
cloudElementsFile.close
cloudElementsUserFile.close
#if using ARCGIS data store code, uncomment this file close line
@@ -593,11 +605,11 @@ def findPrecipRate(TRMMdirName, timelist):
TRMMdirName: a string representing the directory for the original TRMM netCDF files
timelist: a list of python datatimes
- Output:: a list of dictionary of the TRMM data
- NB: also creates netCDF with TRMM data for each CE (for post processing) index
- in MAINDIRECTORY/TRMMnetcdfCEs
+ Output:: a list of dictionary of the TRMM data
+ NB: also creates netCDF with TRMM data for each CE (for post processing) index
+ in MAINDIRECTORY/TRMMnetcdfCEs
- Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
+ Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
'''
allCEnodesTRMMdata =[]
@@ -765,14 +777,14 @@ def findCloudClusters(CEGraph):
'''
Purpose::
Determines the cloud clusters properties from the subgraphs in
- the graph i.e. prunes the graph according to the minimum depth
+ the graph i.e. prunes the graph according to the minimum depth
Input::
CEGraph: a Networkx directed graph of the CEs with weighted edges
according the area overlap between nodes (CEs) of consectuive frames
-
- Output::
- PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
+
+ Output::
+ PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
'''
@@ -828,11 +840,11 @@ def findMCC (prunedGraph):
Input::
prunedGraph: a Networkx Graph representing the CCs
- Output::
- finalMCCList: a list of list of tuples representing a MCC
-
- Assumptions:
- frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Output::
+ finalMCCList: a list of list of tuples representing a MCC
+
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
MCCList = []
@@ -999,12 +1011,12 @@ def traverseTree(subGraph,node, stack, checkedNodes=None):
stack: a list of strings representing a list of nodes in a stack functionality
i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node
checkedNodes: a list of strings representing the list of the nodes in the traversal
-
- Output::
- checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Output::
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
- Assumptions:
- frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
if len(checkedNodes) == len(subGraph):
@@ -1043,7 +1055,7 @@ def checkedNodesMCC (prunedGraph, nodeList):
'''
Purpose ::
Determine if this path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
+ preliminary information regarding the stages of the feature
Input::
prunedGraph: a Networkx Graph representing all the cloud clusters
@@ -1154,7 +1166,7 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
'''
Purpose::
Utility function to determine if a path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
+ preliminary information regarding the stages of the feature
Input::
prunedGraph: a Networkx Graph representing all the cloud clusters
@@ -1389,7 +1401,7 @@ def maxExtentAndEccentricity(eachList):
Output::
maxShieldNode: a string representing the node with the maximum maxShieldNode
- definiteMCCFlag: a boolean indicating that the MCC has met all requirements
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
'''
maxShieldNode =''
@@ -1581,7 +1593,7 @@ def allAncestors(path, aNode):
Input::
path: a list of strings representing the nodes in the path
- aNode: a string representing a node to be checked for parents
+ aNode: a string representing a node to be checked for parents
Output::
path: a list of strings representing the list of the nodes connected to aNode through its parents
@@ -1607,7 +1619,7 @@ def allDescendants(path, aNode):
Input::
path: a list of strings representing the nodes in the path
- aNode: a string representing a node to be checked for children
+ aNode: a string representing a node to be checked for children
Output::
path: a list of strings representing the list of the nodes connected to aNode through its children
@@ -1653,7 +1665,7 @@ def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
Input::
thisNode: a string representing the unique ID of a node
- nodeBehaviorIdentifier: a string representing the behavior S- split, M- merge, B- both split and merge, N- neither split or merge
+ nodeBehaviorIdentifier: a string representing the behavior S- split, M- merge, B- both split and merge, N- neither split or merge
Output :: None
@@ -1703,9 +1715,9 @@ def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
def eccentricity (cloudElementLatLon):
'''
Purpose::
- Determines the eccentricity (shape) of contiguous boxes
- Values tending to 1 are more circular by definition, whereas
- values tending to 0 are more linear
+ Determines the eccentricity (shape) of contiguous boxes
+ Values tending to 1 are more circular by definition, whereas
+ values tending to 0 are more linear
Input::
cloudElementLatLon: 2D array in (lat,lon) representing T_bb contiguous squares
@@ -1720,29 +1732,29 @@ def eccentricity (cloudElementLatLon):
#loop over all lons and determine longest (non-zero) col
#loop over all lats and determine longest (non-zero) row
for latLon in cloudElementLatLon:
- #assign a matrix to determine the legit values
-
- nonEmptyLons = sum(sum(cloudElementLatLon)>0)
- nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
-
- lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
- latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
- epsilon = min(latEigenvalues,lonEigenvalues)
-
+ #assign a matrix to determine the legit values
+
+ nonEmptyLons = sum(sum(cloudElementLatLon)>0)
+ nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
+
+ lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
+ latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
+ epsilon = min(latEigenvalues,lonEigenvalues)
+
return epsilon
#******************************************************************
def cloudElementOverlap (currentCELatLons, previousCELatLons):
'''
Purpose::
- Determines the percentage overlap between two list of lat-lons passed
+ Determines the percentage overlap between two list of lat-lons passed
Input::
- currentCELatLons: a list of tuples for the current CE
- previousCELatLons: a list of tuples for the other CE being considered
+ currentCELatLons: a list of tuples for the current CE
+ previousCELatLons: a list of tuples for the other CE being considered
Output::
- percentageOverlap: a floating-point representing the number of overlapping lat_lon tuples
- areaOverlap: a floating-point number representing the area overlapping
+ percentageOverlap: a floating-point representing the number of overlapping lat_lon tuples
+ areaOverlap: a floating-point number representing the area overlapping
'''
@@ -1829,8 +1841,8 @@ def findCESpeed(node, MCSList):
def maenumerate(mArray):
'''
Purpose::
- Utility script for returning the actual values from the masked array
- Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
+ Utility script for returning the actual values from the masked array
+ Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
Input::
mArray: the masked array returned from the ma.array() command
@@ -1844,7 +1856,7 @@ def maenumerate(mArray):
mask = ~mArray.mask.ravel()
#beware yield fast, but generates a type called "generate" that does not allow for array methods
for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
- if maskedValue:
+ if maskedValue:
yield index
#******************************************************************
def createMainDirectory(mainDirStr):
@@ -2046,20 +2058,20 @@ def find_nearest(thisArray,value):
def preprocessingMERG(MERGdirname):
'''
Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format. The files end up in a folder called mergNETCDF in the directory
- where the raw MERG data is
- NOTE: VERY RAW AND DIRTY
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
Input::
- Directory to the location of the raw MERG files, preferably zipped
+ Directory to the location of the raw MERG files, preferably zipped
Output::
none
Assumptions::
1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
+ have been installed on the system and the user can access
2 User can write files in location where script is being called
3 the files havent been unzipped
'''
@@ -2156,20 +2168,20 @@ def postProcessingNetCDF(dataset, dirName = None):
TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
Purpose::
- Utility script displaying the data in generated NETCDF4 files
- in GrADS
- NOTE: VERY RAW AND DIRTY
+ Utility script displaying the data in generated NETCDF4 files
+ in GrADS
+ NOTE: VERY RAW AND DIRTY
Input::
- dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
- string: Directory to the location of the raw (MERG) files, preferably zipped
+ dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
+ string: Directory to the location of the raw (MERG) files, preferably zipped
Output::
images in location as specfied in the code
Assumptions::
1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
+ have been installed on the system and the user can access
2 User can write files in location where script is being called
'''
@@ -2190,7 +2202,7 @@ def postProcessingNetCDF(dataset, dirName = None):
if dataset == 1:
var = 'ch4'
ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
- ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
+ ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
origsFile = coreDir+"/../GrADSscripts/cs1.gs"
gsFile = coreDir+"/../GrADSscripts/cs2.gs"
sologsFile = coreDir+"/../GrADSscripts/mergeCE.gs"
@@ -2199,7 +2211,7 @@ def postProcessingNetCDF(dataset, dirName = None):
elif dataset ==2:
var = 'precipAcc'
ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
- ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
+ ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
origsFile = coreDir+"/../GrADSscripts/cs3.gs"
gsFile = coreDir+"/../GrADSscripts/cs4.gs"
sologsFile = coreDir+"/../GrADSscripts/TRMMCE.gs"
@@ -2208,7 +2220,7 @@ def postProcessingNetCDF(dataset, dirName = None):
elif dataset ==3:
var = 'ch4'
ctlTitle = 'TITLE MERG DATA'
- ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
+ ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
origsFile = coreDir+"/../GrADSscripts/cs1.gs"
sologsFile = coreDir+"/../GrADSscripts/infrared.gs"
lineNum = 54
@@ -2415,15 +2427,15 @@ def tempMaskedImages(imgFilename):
To generate temperature-masked images for a first pass verification
Input::
- imgFilename: filename for the gif file
+ imgFilename: filename for the gif file
Output::
- None - Gif images for each file of T_bb less than 250K are generated in folder called mergImgs
+ None - Gif images for each file of T_bb less than 250K are generated in folder called mergImgs
Assumptions::
Same as for preprocessingMERG
1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
+ have been installed on the system and the user can access
2 User can write files in location where script is being called
3 the files havent been unzipped
'''
@@ -2445,7 +2457,7 @@ def tempMaskedImages(imgFilename):
return
#******************************************************************
#
-# METRICS FUNCTIONS FOR MERG.PY
+# METRICS FUNCTIONS FOR MERG.PY
# TODO: rewrite these metrics so that they use the data from the
# file instead of the graph as this reduce mem resources needed
#
@@ -2523,7 +2535,7 @@ def longestDuration(allMCCtimes):
Output::
an integer - lenMCC: representing the duration of the longest MCC found
- a list of strings - longestMCC: representing the nodes of longest MCC
+ a list of strings - longestMCC: representing the nodes of longest MCC
Assumptions::
@@ -2551,7 +2563,7 @@ def shortestDuration(allMCCtimes):
of MCC temporal details for each MCC in the period considered
Output::an integer - lenMCC: representing the duration of the shortest MCC found
- a list of strings - longestMCC: representing the nodes of shortest MCC
+ a list of strings - longestMCC: representing the nodes of shortest MCC
Assumptions::
@@ -2577,7 +2589,7 @@ def averageDuration(allMCCtimes):
of MCC temporal details for each MCC in the period considered
Output::a floating-point representing the average duration of a MCC in the period
-
+
Assumptions::
'''
@@ -2614,7 +2626,7 @@ def averageFeatureSize(finalMCCList):
Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
Output::a floating-point representing the average area of a MCC in the period
-
+
Assumptions::
'''
@@ -2643,7 +2655,7 @@ def commonFeatureSize(finalMCCList):
Output::
a floating-point representing the average area of a MCC in the period
-
+
Assumptions::
'''
@@ -2917,7 +2929,7 @@ def displayPrecip(finalMCCList):
title = 'Precipitation distribution of the MCS '
fig,ax = plt.subplots(1, facecolor='white', figsize=(20,7))
- cmap = cm.jet
+ cmap = plt.jet
ax.scatter(x, y, s=partialArea, c= colorBarTime, edgecolors='none', marker='o', cmap =cmap)
colorBarTime=[]
colorBarTime =list(set(timeList))
@@ -3012,12 +3024,12 @@ def plotPrecipHistograms(finalMCCList):
# Set the formatter
plt.gca().yaxis.set_major_formatter(formatter)
plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%0.0f'))
- imgFilename = MAINDIRECTORY+'/images/'+str(thisTime)+eachNode['uniqueID']+'TRMMMCS.gif'
-
- plt.savefig(imgFilename, transparent=True)
- precip =[]
-
- # ------ NETCDF File get info ------------------------------------
+ imgFilename = MAINDIRECTORY+'/images/'+str(thisTime)+eachNode['uniqueID']+'TRMMMCS.gif'
+
+ plt.savefig(imgFilename, transparent=True)
+ precip =[]
+
+ # ------ NETCDF File get info ------------------------------------
thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + str(thisTime).replace(" ", "_") + eachNode['uniqueID'] +'.nc'
TRMMData = Dataset(thisFileName,'r', format='NETCDF4')
precipRate = TRMMData.variables['precipitation_Accumulation'][:,:,:]
@@ -3046,8 +3058,8 @@ def plotHistogram(aList, aTitle, aLabel):
Input::
aList: the list of floating points representing values for e.g. averageArea, averageDuration, etc.
- aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
- aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
+ aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
+ aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
Output::
plots (gif files)
@@ -3143,7 +3155,7 @@ def plotAccTRMM (finalMCCList):
accuPrecipRate += precipRate
imgFilename = MAINDIRECTORY+'/images/MCSaccu'+firstPartName+str(thisNode['cloudElementTime']).replace(" ", "_")+'.gif'
- #create new netCDF file
+ #create new netCDF file
accuTRMMFile = MAINDIRECTORY+'/TRMMnetcdfCEs/accu'+firstPartName+str(thisNode['cloudElementTime']).replace(" ", "_")+'.nc'
#write the file
accuTRMMData = Dataset(accuTRMMFile, 'w', format='NETCDF4')
@@ -3190,37 +3202,37 @@ def plotAccTRMM (finalMCCList):
replaceExpXDef = 'echo XDEF '+ str(nxgrdTRMM) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrdTRMM) +' >> acc.ctl'
subprocess.call(replaceExpXDef, shell=True)
#subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
- #subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
- replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
- subprocess.call(replaceExpYDef, shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
- subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
- #subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
- subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
-
- #generate GrADS script
- subprocess.call('rm accuTRMM1.gs', shell=True)
- subprocess.call('touch accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'reinit''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'open acc.ctl ''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set grads off''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set mpdset hires''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set gxout shaded''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set datawarn off''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'d precipacc''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'draw title TRMM Accumulated Precipitation [mm]''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'run cbarn''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800 white''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'quit''\'" >> accuTRMM1.gs', shell=True)
- gradscmd = 'grads -blc ' + '\'run accuTRMM1.gs''\''
- subprocess.call(gradscmd, shell=True)
- sys.exit()
-
- #clean up
- subprocess.call('rm accuTRMM1.gs', shell=True)
- subprocess.call('rm acc.ctl', shell=True)
+ #subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
+ subprocess.call(replaceExpYDef, shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
+ subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
+ #subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
+ subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
+
+ #generate GrADS script
+ subprocess.call('rm accuTRMM1.gs', shell=True)
+ subprocess.call('touch accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'reinit''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'open acc.ctl ''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set grads off''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set mpdset hires''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set gxout shaded''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set datawarn off''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'d precipacc''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'draw title TRMM Accumulated Precipitation [mm]''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'run cbarn''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800 white''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'quit''\'" >> accuTRMM1.gs', shell=True)
+ gradscmd = 'grads -blc ' + '\'run accuTRMM1.gs''\''
+ subprocess.call(gradscmd, shell=True)
+ sys.exit()
+
+ #clean up
+ subprocess.call('rm accuTRMM1.gs', shell=True)
+ subprocess.call('rm acc.ctl', shell=True)
return
#******************************************************************
@@ -3328,7 +3340,7 @@ def plotAccuInTimeRange(starttime, endtime):
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
- subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
+ subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
#generate GrADS script
imgFilename = MAINDIRECTORY+'/images/accu'+starttime+'-'+endtime+'.gif'
@@ -3715,33 +3727,32 @@ def colorbar_index(ncolors, nlabels, cmap):
return
#******************************************************************
def cmap_discretize(cmap, N):
- '''
- Taken from: http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks
- http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
- Return a discrete colormap from the continuous colormap cmap.
-
- cmap: colormap instance, eg. cm.jet.
- N: number of colors.
-
- Example
- x = resize(arange(100), (5,100))
- djet = cmap_discretize(cm.jet, 5)
- imshow(x, cmap=djet)
- '''
-
- if type(cmap) == str:
- cmap = plt.get_cmap(cmap)
- colors_i = np.concatenate((np.linspace(0, 1., N), (0.,0.,0.,0.)))
- colors_rgba = cmap(colors_i)
- indices = np.linspace(0, 1., N+1)
- cdict = {}
- for ki,key in enumerate(('red','green','blue')):
- cdict[key] = [ (indices[i], colors_rgba[i-1,ki], colors_rgba[i,ki])
- for i in xrange(N+1) ]
- # Return colormap object.
- return mcolors.LinearSegmentedColormap(cmap.name + "_%d"%N, cdict, 1024)
+ '''
+ Taken from: http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks
+ http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
+ Return a discrete colormap from the continuous colormap cmap.
+
+ cmap: colormap instance, eg. cm.jet.
+ N: number of colors.
+
+ Example
+ x = resize(arange(100), (5,100))
+ djet = cmap_discretize(cm.jet, 5)
+ imshow(x, cmap=djet)
+ '''
+
+ if type(cmap) == str:
+ cmap = plt.get_cmap(cmap)
+ colors_i = np.concatenate((np.linspace(0, 1., N), (0.,0.,0.,0.)))
+ colors_rgba = cmap(colors_i)
+ indices = np.linspace(0, 1., N+1)
+ cdict = {}
+ for ki,key in enumerate(('red','green','blue')):
+ cdict[key] = [ (indices[i], colors_rgba[i-1,ki], colors_rgba[i,ki])
+ for i in xrange(N+1) ]
+ # Return colormap object.
+ return mcolors.LinearSegmentedColormap(cmap.name + "_%d"%N, cdict, 1024)
#******************************************************************
-
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/code/mccSearchUI.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchUI.py b/mccsearch/code/mccSearchUI.py
index 45dc73a..885ff43 100644
--- a/mccsearch/code/mccSearchUI.py
+++ b/mccsearch/code/mccSearchUI.py
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
'''
# Wizard for running the mccSearch program
'''
[43/50] [abbrv] git commit: Fix formatting, remove unused imports,
update files.py and process.py to most recent OCW code
Posted by le...@apache.org.
Fix formatting, remove unused imports, update files.py and process.py to most recent OCW code
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/dc091596
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/dc091596
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/dc091596
Branch: refs/heads/master
Commit: dc091596038698eb15845a254ae0e76d52cf64ed
Parents: f9ae9c2
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 16:45:06 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/code/__init__.py | 0
mccsearch/code/files.py | 375 ++++++----
mccsearch/code/mainProg.py | 19 +-
mccsearch/code/mainProgTemplate.py | 16 +
mccsearch/code/mccSearch.py | 449 ++++++------
mccsearch/code/mccSearchUI.py | 16 +
mccsearch/code/process.py | 1221 ++++++++++++-------------------
mccsearch/docs/mccsearch.md | 18 +-
8 files changed, 992 insertions(+), 1122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/code/__init__.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/__init__.py b/mccsearch/code/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/code/files.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.py b/mccsearch/code/files.py
index 82b508a..b238754 100644
--- a/mccsearch/code/files.py
+++ b/mccsearch/code/files.py
@@ -1,25 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
"""
-Module for handling data input files. Requires PyNIO and Numpy be
+Module for handling data input files. Requires netCDF and Numpy be
installed.
-This module can easily open NetCDF, HDF and Grib files. Search the PyNIO
+This module can easily open NetCDF, HDF and Grib files. Search the netCDF4
documentation for a complete list of supported formats.
"""
from os import path
-
-try:
- import Nio
-except ImportError:
- import nio as Nio
-
+import netCDF4
import numpy as np
import numpy.ma as ma
import sys
-import process #KDW added
-#from toolkit import process
-#from utils import fortranfile
-#from utils import misc
+
+from toolkit import process
+from utils import fortranfile
+from utils import misc
VARIABLE_NAMES = {'time': ['time', 'times', 'date', 'dates', 'julian'],
@@ -48,15 +59,14 @@ def getVariableByType(filename, variableType):
name match cannot be found.
"""
try:
- f = Nio.open_file(filename)
+ f = netCDF4.Dataset(filename, mode='r')
except:
- #print 'PyNio had an issue opening the filename (%s) you provided' % filename
- print "NIOError:", sys.exc_info()[0]
+ print "netCDF4Error:", sys.exc_info()[0]
raise
variableKeys = f.variables.keys()
f.close()
- variableKeys = [variable.lower() for variable in variableKeys]
+ variableKeys = [variable.encode().lower() for variable in variableKeys]
variableMatch = VARIABLE_NAMES[variableType]
commonVariables = list(set(variableKeys).intersection(variableMatch))
@@ -80,10 +90,9 @@ def getVariableRange(filename, variableName):
variableRange - tuple of order (variableMin, variableMax)
"""
try:
- f = Nio.open_file(filename)
+ f = netCDF4.Dataset(filename, mode='r')
except:
- #print 'PyNio had an issue opening the filename (%s) you provided' % filename
- print "NIOError:", sys.exc_info()[0]
+ print "netCDF4Error:", sys.exc_info()[0]
raise
varArray = f.variables[variableName][:]
@@ -121,13 +130,9 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
# ii) find out how many timesteps in the file
# (assume same ntimes in each file in list)
# -allows you to create an empty array to store variable data for all times
- tmp = Nio.open_file(filename)
+ tmp = netCDF4.Dataset(filename, mode='r')
latsraw = tmp.variables[latVarName][:]
lonsraw = tmp.variables[lonVarName][:]
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
-
- """TODO: Guard against case where latsraw and lonsraw are not the same dim?"""
-
if(latsraw.ndim == 1):
lon, lat = np.meshgrid(lonsraw, latsraw)
if(latsraw.ndim == 2):
@@ -137,7 +142,7 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
timesraw = tmp.variables[timeVarName]
ntimes = len(timesraw)
- print 'Lats and lons read in for first file in filelist', ntimes
+ print 'Lats and lons read in for first file in filelist'
# Create a single empty masked array to store model data from all files
t2store = ma.zeros((ntimes * len(filelist), len(lat[:, 0]), len(lon[0, :])))
@@ -151,16 +156,16 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
# as no assumption made that same number of times in each file...
- for ifile in filelist:
+ for i, ifile in enumerate(filelist):
#print 'Loading data from file: ',filelist[i]
- f = Nio.open_file(ifile)
- t2raw = f.variables[myvar][:]
+ f = netCDF4.Dataset(ifile, mode='r')
+ t2raw = ma.array(f.variables[myvar][:])
timesraw = f.variables[timeVarName]
time = timesraw[:]
ntimes = len(time)
print 'file= ', i, 'ntimes= ', ntimes, filelist[i]
- print '\nt2raw shape: ', t2raw.shape
+ print 't2raw shape: ', t2raw.shape
# Flatten dimensions which needn't exist, i.e. level
# e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
@@ -175,32 +180,26 @@ def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarNam
if t2tmp.ndim == 2:
t2tmp = np.expand_dims(t2tmp, 0)
- print "\n timesaccu is: ", timesaccu, time,timesaccu + np.arange(ntimes)
-
t2store[timesaccu + np.arange(ntimes), :, :] = t2tmp[:, :, :]
timestore[timesaccu + np.arange(ntimes)] = time
- timesaccu = timesaccu + ntimes
- print "\n ***timesaccu is: ", timesaccu
+ timesaccu += ntimes
f.close()
- i += 1
-
- print '\nData read in successfully with dimensions: ', t2store.shape, timesaccu
+
+ print 'Data read in successfully with dimensions: ', t2store.shape
# TODO: search for duplicated entries (same time) and remove duplicates.
# Check to see if number of unique times == number of times, if so then no problem
if(len(np.unique(timestore)) != len(np.where(timestore != 0)[0].view())):
- print '\nWARNING: Possible duplicated times'
+ print 'WARNING: Possible duplicated times'
# Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
timestore, _ = process.getModelTimes(filename, timeVarName)
- data_dict = {}
- data_dict['lats'] = lat
- data_dict['lons'] = lon
- data_dict['times'] = timestore
- data_dict['data'] = t2store
- #return lat, lon, timestore, t2store
+ # Make sure latlon grid is monotonically increasing and that the domains
+ # are correct
+ lat, lon, t2store = checkLatLon(lat, lon, t2store)
+ data_dict = {'lats': lat, 'lons': lon, 'times': timestore, 'data': t2store}
return data_dict
def select_var_from_file(myfile, fmt='not set'):
@@ -209,7 +208,7 @@ def select_var_from_file(myfile, fmt='not set'):
Input:
myfile - filename
- fmt - (optional) specify fileformat for PyNIO if filename suffix is non-standard
+ fmt - (optional) specify fileformat for netCDF4 if filename suffix is non-standard
Output:
myvar - variable name in file
@@ -220,12 +219,12 @@ def select_var_from_file(myfile, fmt='not set'):
print fmt
if fmt == 'not set':
- f = Nio.open_file(myfile)
+ f = netCDF4.Dataset(myfile, mode='r')
if fmt != 'not set':
- f = Nio.open_file(myfile, format=fmt)
+ f = netCDF4.Dataset(myfile, mode='r', format=fmt)
- keylist = f.variables.keys()
+ keylist = [key.encode().lower() for key in f.variables.keys()]
i = 0
for v in keylist:
@@ -251,9 +250,8 @@ def select_var_from_wrf_file(myfile):
Peter Lean September 2010
'''
- f = Nio.open_file(myfile, format='nc')
-
- keylist = f.variables.keys()
+ f = netCDF4.Dataset(myfile, mode='r', format='NETCDF4')
+ keylist = [key.encode().lower() for key in f.variables.keys()]
i = 0
for v in keylist:
@@ -270,68 +268,50 @@ def select_var_from_wrf_file(myfile):
return mywrfvar
-def read_lolaT_from_file(filename, latVarName, lonVarName, timeVarName, file_type):
+def read_data_from_one_file(ifile, myvar, latVarName, lonVarName, timeVarName, file_type):
"""
- Function that will return lat, lon, and time arrays
+ Purpose::
+ Read in data from one file at a time
- Input::
- filename - the file to inspect
- latVarName - name of the Latitude Variable
+ Input::
+ filelist - list of filenames (including path)
+ myvar - string containing name of variable to load in (as it appears in file)s
lonVarName - name of the Longitude Variable
timeVarName - name of the Time Variable
- fileType = type of file we are trying to parse
-
- Output::
- lat - MESH GRID of Latitude values with shape (nx, ny)
- lon - MESH GRID of Longitude values with shape (nx, ny)
- timestore - Python list of Datetime objects
-
- MESHGRID docs: http://docs.scipy.org/doc/numpy/reference/generated/numpy.meshgrid.html
+ fileType - type of file we are trying to parse
- """
-
- tmp = Nio.open_file(filename, format=file_type)
- lonsraw = tmp.variables[lonVarName][:]
- latsraw = tmp.variables[latVarName][:]
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
- if(latsraw.ndim == 1):
- lon, lat = np.meshgrid(lonsraw, latsraw)
- if(latsraw.ndim == 2):
- lon = lonsraw
- lat = latsraw
- timestore, _ = process.getModelTimes(filename, timeVarName)
- print ' read_lolaT_from_file: Lats, lons and times read in for the model domain'
- return lat, lon, timestore
-
-def read_data_from_one_file(ifile, myvar, timeVarName, lat, file_type):
- ##################################################################################
- # Read in data from one file at a time
- # Input: filelist - list of filenames (including path)
- # myvar - string containing name of variable to load in (as it appears in file)
- # Output: lat, lon - 2D array of latitude and longitude values
- # times - list of times
- # t2store - numpy array containing data from all files
- # Modified from read_data_from_file_list to read data from multiple models into a 4-D array
- # 1. The code now processes model data that completely covers the 20-yr period. Thus,
- # all model data must have the same time levels (ntimes). Unlike in the oroginal, ntimes
- # is fixed here.
- # 2. Because one of the model data exceeds 240 mos (243 mos), the model data must be
- # truncated to the 240 mons using the ntimes determined from the first file.
- ##################################################################################
- f = Nio.open_file(ifile)
+ Output::
+ lat, lon - 2D arrays of latitude and longitude values
+ times - list of times
+ t2store - numpy array containing data from the file for the requested variable
+ varUnit - units for the variable given by t2store
+ """
+ f = netCDF4.Dataset(ifile, mode='r')
try:
- varUnit = f.variables[myvar].units.upper()
+ varUnit = f.variables[myvar].units.encode().upper()
except:
varUnit = raw_input('Enter the model variable unit: \n> ').upper()
- t2raw = f.variables[myvar][:]
+ t2raw = ma.array(f.variables[myvar][:])
t2tmp = t2raw.squeeze()
if t2tmp.ndim == 2:
t2tmp = np.expand_dims(t2tmp, 0)
- t2tmp = t2tmp
+
+ lonsraw = f.variables[lonVarName][:]
+ latsraw = f.variables[latVarName][:]
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
+
f.close()
print ' success read_data_from_one_file: VarName=', myvar, ' Shape(Full)= ', t2tmp.shape, ' Unit= ', varUnit
timestore = process.decode_model_timesK(ifile, timeVarName, file_type)
- return timestore, t2tmp, varUnit
+
+ # Make sure latlon grid is monotonically increasing and that the domains
+ # are correct
+ lat, lon, t2store = checkLatLon(lat, lon, t2tmp)
+ return lat, lon, timestore, t2store, varUnit
def findTimeVariable(filename):
"""
@@ -343,12 +323,12 @@ def findTimeVariable(filename):
variableNameList - list of variable names from the input filename
"""
try:
- f = Nio.open_file(filename, mode='r')
+ f = netCDF4.Dataset(filename, mode='r')
except:
print("Unable to open '%s' to try and read the Time variable" % filename)
raise
- variableNameList = f.variables.keys()
+ variableNameList = [variable.encode() for variable in f.variables.keys()]
# convert all variable names into lower case
varNameListLowerCase = [x.lower() for x in variableNameList]
@@ -383,12 +363,12 @@ def findLatLonVarFromFile(filename):
-lonMax
"""
try:
- f = Nio.open_file(filename, mode='r')
+ f = netCDF4.Dataset(filename, mode='r')
except:
print("Unable to open '%s' to try and read the Latitude and Longitude variables" % filename)
raise
- variableNameList = f.variables.keys()
+ variableNameList = [variable.encode() for variable in f.variables.keys()]
# convert all variable names into lower case
varNameListLowerCase = [x.lower() for x in variableNameList]
@@ -449,16 +429,17 @@ def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarN
# i) read in lats, lons
# ii) find out how many timesteps in the file (assume same ntimes in each file in list)
# -allows you to create an empty array to store variable data for all times
- tmp = Nio.open_file(filelist[0], format=file_type)
+ tmp = netCDF4.Dataset(filelist[0], mode='r', format=file_type)
latsraw = tmp.variables[latVarName][:]
lonsraw = tmp.variables[lonVarName][:]
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ timesraw = tmp.variables[timeVarName]
+
if(latsraw.ndim == 1):
lon, lat = np.meshgrid(lonsraw, latsraw)
- if(latsraw.ndim == 2):
- lon = lonsraw; lat = latsraw
-
- timesraw = tmp.variables[timeVarName]
+
+ elif(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
ntimes = len(timesraw); nygrd = len(lat[:, 0]); nxgrd = len(lon[0, :])
print 'Lats and lons read in for first file in filelist'
@@ -474,16 +455,13 @@ def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarN
# NB. this method allows for missing times in data files
# as no assumption made that same number of times in each file...
- i = 0
- for ifile in filelist:
+ for i, ifile in enumerate(filelist):
#print 'Loading data from file: ',filelist[i]
- f = Nio.open_file(ifile)
- t2raw = f.variables[myvar][:]
+ f = netCDF4.Dataset(ifile, mode='r')
+ t2raw = ma.array(f.variables[myvar][:])
timesraw = f.variables[timeVarName]
- #time = timesraw[0:ntimes]
- ntimes = len(timesraw)
#ntimes=len(time)
- print 'file= ',i,' ntimes= ',ntimes,filelist[i]
+ #print 'file= ',i,'ntimes= ',ntimes,filelist[i]
## Flatten dimensions which needn't exist, i.e. level
## e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
## Code requires data to have dimensions, (time,lat,lon)
@@ -493,18 +471,20 @@ def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarN
if t2tmp.ndim == 2:
t2tmp = np.expand_dims(t2tmp, 0)
#t2store[timesaccu+np.arange(ntimes),:,:]=t2tmp[0:ntimes,:,:]
- t2store[i, 0:(ntimes-1), :, :] = t2tmp[0:(ntimes-1), :, :]
+ t2store[i, 0:ntimes, :, :] = t2tmp[0:ntimes, :, :]
#timestore[timesaccu+np.arange(ntimes)]=time
#timesaccu=timesaccu+ntimes
f.close()
- i += 1
print 'Data read in successfully with dimensions: ', t2store.shape
# Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
ifile = filelist[0]
timestore, _ = process.getModelTimes(ifile, timeVarName)
-
+
+ # Make sure latlon grid is monotonically increasing and that the domains
+ # are correct
+ lat, lon, t2store = checkLatLon(lat, lon, t2store)
return lat, lon, timestore, t2store
def find_latlon_ranges(filelist, lat_var_name, lon_var_name):
@@ -524,7 +504,7 @@ def find_latlon_ranges(filelist, lat_var_name, lon_var_name):
filename = filelist[0]
try:
- f = Nio.open_file(filename)
+ f = netCDF4.Dataset(filename, mode='r')
lats = f.variables[lat_var_name][:]
latMin = lats.min()
@@ -622,30 +602,30 @@ def writeNCfile(fileName, numSubRgn, lons, lats, obsData, mdlData, obsRgnAvg, md
dimY = mdlData.shape[2] # y-dimension
dimX = mdlData.shape[3] # x-dimension
dimR = obsRgnAvg.shape[1] # the number of subregions
- f = Nio.open_file(fileName, mode='w', format='nc')
+ f = netCDF4.Dataset(fileName, mode='w', format='NETCDF4')
print mdlRgnAvg.shape, dimM, dimR, dimT
#create global attributes
- f.globalAttName = ''
+ f.description = ''
# create dimensions
print 'Creating Dimensions within the NetCDF Object...'
- f.create_dimension('unity', 1)
- f.create_dimension('time', dimT)
- f.create_dimension('west_east', dimX)
- f.create_dimension('south_north', dimY)
- f.create_dimension('obs', dimO)
- f.create_dimension('models', dimM)
+ f.createDimension('unity', 1)
+ f.createDimension('time', dimT)
+ f.createDimension('west_east', dimX)
+ f.createDimension('south_north', dimY)
+ f.createDimension('obs', dimO)
+ f.createDimension('models', dimM)
# create the variable (real*4) to be written in the file
print 'Creating Variables...'
- f.create_variable('lon', 'd', ('south_north', 'west_east'))
- f.create_variable('lat', 'd', ('south_north', 'west_east'))
- f.create_variable('oDat', 'd', ('obs', 'time', 'south_north', 'west_east'))
- f.create_variable('mDat', 'd', ('models', 'time', 'south_north', 'west_east'))
+ f.createVariable('lon', 'd', ('south_north', 'west_east'))
+ f.createVariable('lat', 'd', ('south_north', 'west_east'))
+ f.createVariable('oDat', 'd', ('obs', 'time', 'south_north', 'west_east'))
+ f.createVariable('mDat', 'd', ('models', 'time', 'south_north', 'west_east'))
if subRegions:
- f.create_dimension('regions', dimR)
- f.create_variable('oRgn', 'd', ('obs', 'regions', 'time'))
- f.create_variable('mRgn', 'd', ('models', 'regions', 'time'))
+ f.createDimension('regions', dimR)
+ f.createVariable('oRgn', 'd', ('obs', 'regions', 'time'))
+ f.createVariable('mRgn', 'd', ('models', 'regions', 'time'))
f.variables['oRgn'].varAttName = 'Observation time series: Subregions'
f.variables['mRgn'].varAttName = 'Model time series: Subregions'
@@ -664,33 +644,140 @@ def writeNCfile(fileName, numSubRgn, lons, lats, obsData, mdlData, obsRgnAvg, md
f.variables['mDat'].varAttName = 'Model time series: entire domain'
# assign the values to the variable and write it
- f.variables['lon'][:, :] = lons
- f.variables['lat'][:, :] = lats
+ f.variables['lon'][:] = lons[:]
+ f.variables['lat'][:] = lats[:]
if subRegions:
- f.variables['oRgn'][:, :, :] = obsRgnAvg
- f.variables['mRgn'][:, :, :] = mdlRgnAvg
+ f.variables['oRgn'][:] = obsRgnAvg[:]
+ f.variables['mRgn'][:] = mdlRgnAvg[:]
f.close()
def loadDataIntoNetCDF(fileObject, datasets, dataArray, dataType):
"""
Input::
- fileObject - PyNIO file object data will be loaded into
+ fileObject - netCDF4 file object data will be loaded into
datasets - List of dataset names
dataArray - Multi-dimensional array of data to be loaded into the NetCDF file
dataType - String with value of either 'Model' or 'Observation'
Output::
- No return value. PyNIO file object is updated in place
+ No return value. netCDF4 file object is updated in place
"""
datasetCount = 0
- for dataset in datasets:
+ for datasetCount, dataset in enumerate(datasets):
if dataType.lower() == 'observation':
datasetName = dataset.replace(' ','')
elif dataType.lower() == 'model':
datasetName = path.splitext(path.basename(dataset))[0]
print "Creating variable %s" % datasetName
- fileObject.create_variable(datasetName, 'd', ('time', 'south_north', 'west_east'))
+ fileObject.createVariable(datasetName, 'd', ('time', 'south_north', 'west_east'))
fileObject.variables[datasetName].varAttName = 'Obseration time series: entire domain'
print 'Loading values into %s' % datasetName
- fileObject.variables[datasetName].assign_value(dataArray[datasetCount,:,:,:])
- datasetCount += 1
+ fileObject.variables[datasetName][:] = dataArray[datasetCount,:,:,:]
+
+def checkLatLon(latsin, lonsin, datain):
+ """
+ Purpose::
+ Checks whether latitudes and longitudes are monotonically increasing
+ within the domains [-90, 90) and [-180, 180) respectively, and rearranges the input data
+ accordingly if they are not.
+
+ Input::
+ latsin - Array of latitudes read from a raw netcdf file
+ lonsin - Array of longitudes read from a raw netcdf file
+ datain - Array of data values read from a raw netcdf file.
+ The shape is assumed to be (..., nLat, nLon).
+
+ Output::
+ latsout - 2D array of (rearranged) latitudes
+ lonsout - 2D array of (rearranged) longitudes
+ dataout - Array of (rearranged) data
+ """
+ # Avoid unnecessary shifting if all lons are higher than 180
+ if lonsin.min() > 180:
+ lonsin -= 360
+
+ # Make sure lats and lons are monotonically increasing
+ latsDecreasing = np.diff(latsin[:, 0]) < 0
+ lonsDecreasing = np.diff(lonsin[0]) < 0
+
+ # If all values are decreasing then they just need to be reversed
+ latsReversed, lonsReversed = latsDecreasing.all(), lonsDecreasing.all()
+
+ # If the lat values are unsorted then raise an exception
+ if not latsReversed and latsDecreasing.any():
+ raise ValueError('Latitudes must be monotonically increasing.')
+
+ # Perform same checks now for lons
+ if not lonsReversed and lonsDecreasing.any():
+ raise ValueError('Longitudes must be monotonically increasing.')
+
+ # Also check if lons go from [0, 360), and convert to [-180, 180)
+ # if necessary
+ lonsShifted = lonsin.max() > 180
+ latsout, lonsout, dataout = latsin[:], lonsin[:], datain[:]
+ # Now correct data if latlon grid needs to be shifted
+ if latsReversed:
+ latsout = latsout[::-1]
+ dataout = dataout[..., ::-1, :]
+
+ if lonsReversed:
+ lonsout = lonsout[..., ::-1]
+ dataout = dataout[..., ::-1]
+
+ if lonsShifted:
+ lat1d = latsout[:, 0]
+ dataout, lon1d = shiftgrid(180, dataout, lonsout[0], start=False)
+ lonsout, latsout = np.meshgrid(lon1d, lat1d)
+
+ return latsout, lonsout, dataout
+
+def shiftgrid(lon0, datain, lonsin, start= True, cyclic=360.0):
+ """
+ Purpose::
+ Shift global lat/lon grid east or west. This function is taken directly
+ from the (unreleased) basemap 1.0.7 source code as version 1.0.6 does not
+ currently support arrays with more than two dimensions.
+ https://github.com/matplotlib/basemap
+
+ Input::
+ lon0 - starting longitude for shifted grid (ending longitude if start=False).
+ lon0 must be on input grid (within the range of lonsin).
+ datain - original data with longitude the right-most dimension.
+ lonsin - original longitudes.
+ start - if True, lon0 represents the starting longitude of the new grid.
+ if False, lon0 is the ending longitude. Default True.
+ cyclic - width of periodic domain (default 360)
+
+ Output::
+ dataout - data on shifted grid
+ lonsout - lons on shifted grid
+ """
+ if np.fabs(lonsin[-1]-lonsin[0]-cyclic) > 1.e-4:
+ # Use all data instead of raise ValueError, 'cyclic point not included'
+ start_idx = 0
+ else:
+ # If cyclic, remove the duplicate point
+ start_idx = 1
+ if lon0 < lonsin[0] or lon0 > lonsin[-1]:
+ raise ValueError('lon0 outside of range of lonsin')
+ i0 = np.argmin(np.fabs(lonsin-lon0))
+ i0_shift = len(lonsin)-i0
+ if ma.isMA(datain):
+ dataout = ma.zeros(datain.shape,datain.dtype)
+ else:
+ dataout = np.zeros(datain.shape,datain.dtype)
+ if ma.isMA(lonsin):
+ lonsout = ma.zeros(lonsin.shape,lonsin.dtype)
+ else:
+ lonsout = np.zeros(lonsin.shape,lonsin.dtype)
+ if start:
+ lonsout[0:i0_shift] = lonsin[i0:]
+ else:
+ lonsout[0:i0_shift] = lonsin[i0:]-cyclic
+ dataout[...,0:i0_shift] = datain[...,i0:]
+ if start:
+ lonsout[i0_shift:] = lonsin[start_idx:i0+start_idx]+cyclic
+ else:
+ lonsout[i0_shift:] = lonsin[start_idx:i0+start_idx]
+ dataout[...,i0_shift:] = datain[...,start_idx:i0+start_idx]
+ return dataout,lonsout
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 43ca921..bb032fc 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -1,10 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
'''
# running the program
'''
import sys
import networkx as nx
-import mccSearch
+from mccSearch
import numpy as np
import numpy.ma as ma
import files
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/code/mainProgTemplate.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemplate.py b/mccsearch/code/mainProgTemplate.py
index 9be65b5..d43788e 100644
--- a/mccsearch/code/mainProgTemplate.py
+++ b/mccsearch/code/mainProgTemplate.py
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
'''
# running the program
'''
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index b7dc4eb..e691ed5 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -1,18 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
'''
# All the functions for the MCC search algorithm
# Following RCMES dataformat in format (t,lat,lon), value
'''
-import datetime
from datetime import timedelta, datetime
-import calendar
-import fileinput
import glob
import itertools
-import json
-import math
-import Nio
-from netCDF4 import Dataset, num2date, date2num
+from netCDF4 import Dataset, date2num
import numpy as np
import numpy.ma as ma
import os
@@ -25,10 +35,10 @@ import sys
import time
import networkx as nx
-import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from matplotlib.dates import DateFormatter,HourLocator
-from matplotlib import cm
+
+import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib.colors as mcolors
from matplotlib.ticker import FuncFormatter, FormatStrFormatter
@@ -48,14 +58,14 @@ XRES = 4.0 #x direction spatial resolution in km
YRES = 4.0 #y direction spatial resolution in km
TRES = 1 #temporal resolution in hrs
LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
-LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
- #have same rank of the matrix it is being compared against
+ #have same rank of the matrix it is being compared against
#criteria for determining cloud elements and edges
T_BB_MAX = 243 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
T_BB_MIN = 218 #cooler temp for the center of the system
CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
-MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
+MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
@@ -80,19 +90,19 @@ PRUNED_GRAPH = nx.DiGraph()
def readMergData(dirname, filelist = None):
'''
Purpose::
- Read MERG data into RCMES format
+ Read MERG data into RCMES format
Input::
- dirname: a string representing the directory to the MERG files in NETCDF format
- filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
+ dirname: a string representing the directory to the MERG files in NETCDF format
+ filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
Output::
- A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
- criteria for each frame
+ A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
+ criteria for each frame
Assumptions::
- The MERG data has been converted to NETCDF using LATS4D
- The data has the same lat/lon format
+ The MERG data has been converted to NETCDF using LATS4D
+ The data has the same lat/lon format
TODO:: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
@@ -129,8 +139,9 @@ def readMergData(dirname, filelist = None):
sys.exit()
else:
# Open the first file in the list to read in lats, lons and generate the grid for comparison
- tmp = Nio.open_file(filelist[0], format='nc')
-
+ #tmp = Nio.open_file(filelist[0], format='nc')
+ tmp = Dataset(filelist[0], format='NETCDF4')
+
#clip the lat/lon grid according to user input
#http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
@@ -146,11 +157,12 @@ def readMergData(dirname, filelist = None):
for files in filelist:
try:
- thisFile = Nio.open_file(files, format='nc')
+ #thisFile = Nio.open_file(files, format='nc')
+ thisFile = Dataset(files, format='NETCDF4')
#clip the dataset according to user lat,lon coordinates
#mask the data and fill with zeros for later
tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
- +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
+ +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
@@ -180,8 +192,8 @@ def readMergData(dirname, filelist = None):
def findCloudElements(mergImgs,timelist,TRMMdirName=None):
'''
Purpose::
- Determines the contiguous boxes for a given time of the satellite images i.e. each frame
- using scipy ndimage package
+ Determines the contiguous boxes for a given time of the satellite images i.e. each frame
+ using scipy ndimage package
Input::
mergImgs: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
@@ -190,8 +202,8 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
TRMMdirName (optional): string representing the path where to find the TRMM datafiles
Output::
- CLOUD_ELEMENT_GRAPH: a Networkx directed graph where each node contains the information in cloudElementDict
- The nodes are determined according to the area of contiguous squares. The nodes are linked through weighted edges.
+ CLOUD_ELEMENT_GRAPH: a Networkx directed graph where each node contains the information in cloudElementDict
+ The nodes are determined according to the area of contiguous squares. The nodes are linked through weighted edges.
cloudElementDict = {'uniqueID': unique tag for this CE,
'cloudElementTime': time of the CE,
@@ -207,9 +219,9 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
'CETRMMmax':floating-point representing the max rate in the CE if TRMMdirName entered,
'CETRMMmin':floating-point representing the min rate in the CE if TRMMdirName entered}
Assumptions::
- Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
- required according to Vila et al. (2008) is 2400km^2
- therefore, 2400/16 = 150 contiguous squares
+ Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
+ required according to Vila et al. (2008) is 2400km^2
+ therefore, 2400/16 = 150 contiguous squares
'''
frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
@@ -293,7 +305,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
#-------------------------------------------------
- #textfile name for accesing CE data using MATLAB code
+ #textfile name for accesing CE data using MATLAB code
# thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
# cloudElementsTextFile = open(thisFileName,'w')
#-------------------------------------------------
@@ -417,8 +429,8 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#-----------End most of NETCDF file stuff ------------------------------------
#populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
for index,value in np.ndenumerate(cloudElement):
if value != 0 :
lat_index,lon_index = index
@@ -443,7 +455,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#calculate the total precip associated with the feature
for index, value in np.ndenumerate(finalCETRMMvalues):
precipTotal += value
- precip.append(value)
+ precip.append(value)
rainFallacc[:] = finalCETRMMvalues[:]
currNetCDFTRMMData.close()
@@ -460,19 +472,19 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#determine if the cloud element the shape
cloudElementEpsilon = eccentricity (cloudElement)
- cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
- cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
-
- #latCenter and lonCenter are given according to the particular array defining this CE
- #so you need to convert this value to the overall domain truth
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementCenter.append(latCenter)
- cloudElementCenter.append(lonCenter)
- cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementCenter.append(latCenter)
+ cloudElementCenter.append(lonCenter)
+ cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
@@ -507,42 +519,42 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
elif areaOverlap >= MIN_OVERLAP:
CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
- else:
- #TODO: remove this else as we only wish for the CE details
- #ensure only the non-zero elements are considered
- #store intel in allCE file
- labels, _ = ndimage.label(cloudElement)
- cloudElementsFile.write("\n-----------------------------------------------")
- cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
- # cloudElementLat = LAT[loc[0],0]
- # cloudElementLon = LON[0,loc[1]]
-
- #populate cloudElementLatLons by unpacking the original values from loc
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
- for index,value in np.ndenumerate(cloudElement):
- if value != 0 :
- lat_index,lon_index = index
- lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
- cloudElementLatLons.append(lat_lon_tuple)
-
- cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
- #latCenter and lonCenter are given according to the particular array defining this CE
+ else:
+ #TODO: remove this else as we only wish for the CE details
+ #ensure only the non-zero elements are considered
+ #store intel in allCE file
+ labels, _ = ndimage.label(cloudElement)
+ cloudElementsFile.write("\n-----------------------------------------------")
+ cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ # cloudElementLat = LAT[loc[0],0]
+ # cloudElementLon = LON[0,loc[1]]
+
+ #populate cloudElementLatLons by unpacking the original values from loc
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
+ #latCenter and lonCenter are given according to the particular array defining this CE
#so you need to convert this value to the overall domain truth
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
- cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
- cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
- cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
- cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
- cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
- cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
- cloudElementsFile.write("\n-----------------------------------------------")
-
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ cloudElementsFile.write("\n-----------------------------------------------")
+
#reset list for the next CE
nodeExist = False
cloudElementCenter=[]
@@ -563,7 +575,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
prevFrameCEs =[]
prevFrameCEs = currFrameCEs
currFrameCEs =[]
-
+
cloudElementsFile.close
cloudElementsUserFile.close
#if using ARCGIS data store code, uncomment this file close line
@@ -593,11 +605,11 @@ def findPrecipRate(TRMMdirName, timelist):
TRMMdirName: a string representing the directory for the original TRMM netCDF files
timelist: a list of python datatimes
- Output:: a list of dictionary of the TRMM data
- NB: also creates netCDF with TRMM data for each CE (for post processing) index
- in MAINDIRECTORY/TRMMnetcdfCEs
+ Output:: a list of dictionary of the TRMM data
+ NB: also creates netCDF with TRMM data for each CE (for post processing) index
+ in MAINDIRECTORY/TRMMnetcdfCEs
- Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
+ Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
'''
allCEnodesTRMMdata =[]
@@ -765,14 +777,14 @@ def findCloudClusters(CEGraph):
'''
Purpose::
Determines the cloud clusters properties from the subgraphs in
- the graph i.e. prunes the graph according to the minimum depth
+ the graph i.e. prunes the graph according to the minimum depth
Input::
CEGraph: a Networkx directed graph of the CEs with weighted edges
according the area overlap between nodes (CEs) of consectuive frames
-
- Output::
- PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
+
+ Output::
+ PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
'''
@@ -828,11 +840,11 @@ def findMCC (prunedGraph):
Input::
prunedGraph: a Networkx Graph representing the CCs
- Output::
- finalMCCList: a list of list of tuples representing a MCC
-
- Assumptions:
- frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Output::
+ finalMCCList: a list of list of tuples representing a MCC
+
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
MCCList = []
@@ -999,12 +1011,12 @@ def traverseTree(subGraph,node, stack, checkedNodes=None):
stack: a list of strings representing a list of nodes in a stack functionality
i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node
checkedNodes: a list of strings representing the list of the nodes in the traversal
-
- Output::
- checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Output::
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
- Assumptions:
- frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
if len(checkedNodes) == len(subGraph):
@@ -1043,7 +1055,7 @@ def checkedNodesMCC (prunedGraph, nodeList):
'''
Purpose ::
Determine if this path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
+ preliminary information regarding the stages of the feature
Input::
prunedGraph: a Networkx Graph representing all the cloud clusters
@@ -1154,7 +1166,7 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
'''
Purpose::
Utility function to determine if a path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
+ preliminary information regarding the stages of the feature
Input::
prunedGraph: a Networkx Graph representing all the cloud clusters
@@ -1389,7 +1401,7 @@ def maxExtentAndEccentricity(eachList):
Output::
maxShieldNode: a string representing the node with the maximum maxShieldNode
- definiteMCCFlag: a boolean indicating that the MCC has met all requirements
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
'''
maxShieldNode =''
@@ -1581,7 +1593,7 @@ def allAncestors(path, aNode):
Input::
path: a list of strings representing the nodes in the path
- aNode: a string representing a node to be checked for parents
+ aNode: a string representing a node to be checked for parents
Output::
path: a list of strings representing the list of the nodes connected to aNode through its parents
@@ -1607,7 +1619,7 @@ def allDescendants(path, aNode):
Input::
path: a list of strings representing the nodes in the path
- aNode: a string representing a node to be checked for children
+ aNode: a string representing a node to be checked for children
Output::
path: a list of strings representing the list of the nodes connected to aNode through its children
@@ -1653,7 +1665,7 @@ def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
Input::
thisNode: a string representing the unique ID of a node
- nodeBehaviorIdentifier: a string representing the behavior S- split, M- merge, B- both split and merge, N- neither split or merge
+ nodeBehaviorIdentifier: a string representing the behavior S- split, M- merge, B- both split and merge, N- neither split or merge
Output :: None
@@ -1703,9 +1715,9 @@ def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
def eccentricity (cloudElementLatLon):
'''
Purpose::
- Determines the eccentricity (shape) of contiguous boxes
- Values tending to 1 are more circular by definition, whereas
- values tending to 0 are more linear
+ Determines the eccentricity (shape) of contiguous boxes
+ Values tending to 1 are more circular by definition, whereas
+ values tending to 0 are more linear
Input::
cloudElementLatLon: 2D array in (lat,lon) representing T_bb contiguous squares
@@ -1720,29 +1732,29 @@ def eccentricity (cloudElementLatLon):
#loop over all lons and determine longest (non-zero) col
#loop over all lats and determine longest (non-zero) row
for latLon in cloudElementLatLon:
- #assign a matrix to determine the legit values
-
- nonEmptyLons = sum(sum(cloudElementLatLon)>0)
- nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
-
- lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
- latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
- epsilon = min(latEigenvalues,lonEigenvalues)
-
+ #assign a matrix to determine the legit values
+
+ nonEmptyLons = sum(sum(cloudElementLatLon)>0)
+ nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
+
+ lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
+ latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
+ epsilon = min(latEigenvalues,lonEigenvalues)
+
return epsilon
#******************************************************************
def cloudElementOverlap (currentCELatLons, previousCELatLons):
'''
Purpose::
- Determines the percentage overlap between two list of lat-lons passed
+ Determines the percentage overlap between two list of lat-lons passed
Input::
- currentCELatLons: a list of tuples for the current CE
- previousCELatLons: a list of tuples for the other CE being considered
+ currentCELatLons: a list of tuples for the current CE
+ previousCELatLons: a list of tuples for the other CE being considered
Output::
- percentageOverlap: a floating-point representing the number of overlapping lat_lon tuples
- areaOverlap: a floating-point number representing the area overlapping
+ percentageOverlap: a floating-point representing the number of overlapping lat_lon tuples
+ areaOverlap: a floating-point number representing the area overlapping
'''
@@ -1829,8 +1841,8 @@ def findCESpeed(node, MCSList):
def maenumerate(mArray):
'''
Purpose::
- Utility script for returning the actual values from the masked array
- Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
+ Utility script for returning the actual values from the masked array
+ Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
Input::
mArray: the masked array returned from the ma.array() command
@@ -1844,7 +1856,7 @@ def maenumerate(mArray):
mask = ~mArray.mask.ravel()
#beware yield fast, but generates a type called "generate" that does not allow for array methods
for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
- if maskedValue:
+ if maskedValue:
yield index
#******************************************************************
def createMainDirectory(mainDirStr):
@@ -2046,20 +2058,20 @@ def find_nearest(thisArray,value):
def preprocessingMERG(MERGdirname):
'''
Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format. The files end up in a folder called mergNETCDF in the directory
- where the raw MERG data is
- NOTE: VERY RAW AND DIRTY
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
Input::
- Directory to the location of the raw MERG files, preferably zipped
+ Directory to the location of the raw MERG files, preferably zipped
Output::
none
Assumptions::
1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
+ have been installed on the system and the user can access
2 User can write files in location where script is being called
3 the files havent been unzipped
'''
@@ -2156,20 +2168,20 @@ def postProcessingNetCDF(dataset, dirName = None):
TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
Purpose::
- Utility script displaying the data in generated NETCDF4 files
- in GrADS
- NOTE: VERY RAW AND DIRTY
+ Utility script displaying the data in generated NETCDF4 files
+ in GrADS
+ NOTE: VERY RAW AND DIRTY
Input::
- dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
- string: Directory to the location of the raw (MERG) files, preferably zipped
+ dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
+ string: Directory to the location of the raw (MERG) files, preferably zipped
Output::
images in location as specfied in the code
Assumptions::
1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
+ have been installed on the system and the user can access
2 User can write files in location where script is being called
'''
@@ -2190,7 +2202,7 @@ def postProcessingNetCDF(dataset, dirName = None):
if dataset == 1:
var = 'ch4'
ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
- ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
+ ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
origsFile = coreDir+"/../GrADSscripts/cs1.gs"
gsFile = coreDir+"/../GrADSscripts/cs2.gs"
sologsFile = coreDir+"/../GrADSscripts/mergeCE.gs"
@@ -2199,7 +2211,7 @@ def postProcessingNetCDF(dataset, dirName = None):
elif dataset ==2:
var = 'precipAcc'
ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
- ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
+ ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
origsFile = coreDir+"/../GrADSscripts/cs3.gs"
gsFile = coreDir+"/../GrADSscripts/cs4.gs"
sologsFile = coreDir+"/../GrADSscripts/TRMMCE.gs"
@@ -2208,7 +2220,7 @@ def postProcessingNetCDF(dataset, dirName = None):
elif dataset ==3:
var = 'ch4'
ctlTitle = 'TITLE MERG DATA'
- ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
+ ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
origsFile = coreDir+"/../GrADSscripts/cs1.gs"
sologsFile = coreDir+"/../GrADSscripts/infrared.gs"
lineNum = 54
@@ -2415,15 +2427,15 @@ def tempMaskedImages(imgFilename):
To generate temperature-masked images for a first pass verification
Input::
- imgFilename: filename for the gif file
+ imgFilename: filename for the gif file
Output::
- None - Gif images for each file of T_bb less than 250K are generated in folder called mergImgs
+ None - Gif images for each file of T_bb less than 250K are generated in folder called mergImgs
Assumptions::
Same as for preprocessingMERG
1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
+ have been installed on the system and the user can access
2 User can write files in location where script is being called
3 the files havent been unzipped
'''
@@ -2445,7 +2457,7 @@ def tempMaskedImages(imgFilename):
return
#******************************************************************
#
-# METRICS FUNCTIONS FOR MERG.PY
+# METRICS FUNCTIONS FOR MERG.PY
# TODO: rewrite these metrics so that they use the data from the
# file instead of the graph as this reduce mem resources needed
#
@@ -2523,7 +2535,7 @@ def longestDuration(allMCCtimes):
Output::
an integer - lenMCC: representing the duration of the longest MCC found
- a list of strings - longestMCC: representing the nodes of longest MCC
+ a list of strings - longestMCC: representing the nodes of longest MCC
Assumptions::
@@ -2551,7 +2563,7 @@ def shortestDuration(allMCCtimes):
of MCC temporal details for each MCC in the period considered
Output::an integer - lenMCC: representing the duration of the shortest MCC found
- a list of strings - longestMCC: representing the nodes of shortest MCC
+ a list of strings - longestMCC: representing the nodes of shortest MCC
Assumptions::
@@ -2577,7 +2589,7 @@ def averageDuration(allMCCtimes):
of MCC temporal details for each MCC in the period considered
Output::a floating-point representing the average duration of a MCC in the period
-
+
Assumptions::
'''
@@ -2614,7 +2626,7 @@ def averageFeatureSize(finalMCCList):
Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
Output::a floating-point representing the average area of a MCC in the period
-
+
Assumptions::
'''
@@ -2643,7 +2655,7 @@ def commonFeatureSize(finalMCCList):
Output::
a floating-point representing the average area of a MCC in the period
-
+
Assumptions::
'''
@@ -2917,7 +2929,7 @@ def displayPrecip(finalMCCList):
title = 'Precipitation distribution of the MCS '
fig,ax = plt.subplots(1, facecolor='white', figsize=(20,7))
- cmap = cm.jet
+ cmap = plt.jet
ax.scatter(x, y, s=partialArea, c= colorBarTime, edgecolors='none', marker='o', cmap =cmap)
colorBarTime=[]
colorBarTime =list(set(timeList))
@@ -3012,12 +3024,12 @@ def plotPrecipHistograms(finalMCCList):
# Set the formatter
plt.gca().yaxis.set_major_formatter(formatter)
plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%0.0f'))
- imgFilename = MAINDIRECTORY+'/images/'+str(thisTime)+eachNode['uniqueID']+'TRMMMCS.gif'
-
- plt.savefig(imgFilename, transparent=True)
- precip =[]
-
- # ------ NETCDF File get info ------------------------------------
+ imgFilename = MAINDIRECTORY+'/images/'+str(thisTime)+eachNode['uniqueID']+'TRMMMCS.gif'
+
+ plt.savefig(imgFilename, transparent=True)
+ precip =[]
+
+ # ------ NETCDF File get info ------------------------------------
thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + str(thisTime).replace(" ", "_") + eachNode['uniqueID'] +'.nc'
TRMMData = Dataset(thisFileName,'r', format='NETCDF4')
precipRate = TRMMData.variables['precipitation_Accumulation'][:,:,:]
@@ -3046,8 +3058,8 @@ def plotHistogram(aList, aTitle, aLabel):
Input::
aList: the list of floating points representing values for e.g. averageArea, averageDuration, etc.
- aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
- aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
+ aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
+ aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
Output::
plots (gif files)
@@ -3143,7 +3155,7 @@ def plotAccTRMM (finalMCCList):
accuPrecipRate += precipRate
imgFilename = MAINDIRECTORY+'/images/MCSaccu'+firstPartName+str(thisNode['cloudElementTime']).replace(" ", "_")+'.gif'
- #create new netCDF file
+ #create new netCDF file
accuTRMMFile = MAINDIRECTORY+'/TRMMnetcdfCEs/accu'+firstPartName+str(thisNode['cloudElementTime']).replace(" ", "_")+'.nc'
#write the file
accuTRMMData = Dataset(accuTRMMFile, 'w', format='NETCDF4')
@@ -3190,37 +3202,37 @@ def plotAccTRMM (finalMCCList):
replaceExpXDef = 'echo XDEF '+ str(nxgrdTRMM) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrdTRMM) +' >> acc.ctl'
subprocess.call(replaceExpXDef, shell=True)
#subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
- #subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
- replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
- subprocess.call(replaceExpYDef, shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
- subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
- #subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
- subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
-
- #generate GrADS script
- subprocess.call('rm accuTRMM1.gs', shell=True)
- subprocess.call('touch accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'reinit''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'open acc.ctl ''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set grads off''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set mpdset hires''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set gxout shaded''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'set datawarn off''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'d precipacc''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'draw title TRMM Accumulated Precipitation [mm]''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'run cbarn''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800 white''\'" >> accuTRMM1.gs', shell=True)
- subprocess.call('echo "''\'quit''\'" >> accuTRMM1.gs', shell=True)
- gradscmd = 'grads -blc ' + '\'run accuTRMM1.gs''\''
- subprocess.call(gradscmd, shell=True)
- sys.exit()
-
- #clean up
- subprocess.call('rm accuTRMM1.gs', shell=True)
- subprocess.call('rm acc.ctl', shell=True)
+ #subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
+ subprocess.call(replaceExpYDef, shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
+ subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
+ #subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
+ subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
+
+ #generate GrADS script
+ subprocess.call('rm accuTRMM1.gs', shell=True)
+ subprocess.call('touch accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'reinit''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'open acc.ctl ''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set grads off''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set mpdset hires''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set gxout shaded''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'set datawarn off''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'d precipacc''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'draw title TRMM Accumulated Precipitation [mm]''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'run cbarn''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800 white''\'" >> accuTRMM1.gs', shell=True)
+ subprocess.call('echo "''\'quit''\'" >> accuTRMM1.gs', shell=True)
+ gradscmd = 'grads -blc ' + '\'run accuTRMM1.gs''\''
+ subprocess.call(gradscmd, shell=True)
+ sys.exit()
+
+ #clean up
+ subprocess.call('rm accuTRMM1.gs', shell=True)
+ subprocess.call('rm acc.ctl', shell=True)
return
#******************************************************************
@@ -3328,7 +3340,7 @@ def plotAccuInTimeRange(starttime, endtime):
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
- subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
+ subprocess.call('echo "precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu" >> acc.ctl', shell=True)
subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
#generate GrADS script
imgFilename = MAINDIRECTORY+'/images/accu'+starttime+'-'+endtime+'.gif'
@@ -3715,33 +3727,32 @@ def colorbar_index(ncolors, nlabels, cmap):
return
#******************************************************************
def cmap_discretize(cmap, N):
- '''
- Taken from: http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks
- http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
- Return a discrete colormap from the continuous colormap cmap.
-
- cmap: colormap instance, eg. cm.jet.
- N: number of colors.
-
- Example
- x = resize(arange(100), (5,100))
- djet = cmap_discretize(cm.jet, 5)
- imshow(x, cmap=djet)
- '''
-
- if type(cmap) == str:
- cmap = plt.get_cmap(cmap)
- colors_i = np.concatenate((np.linspace(0, 1., N), (0.,0.,0.,0.)))
- colors_rgba = cmap(colors_i)
- indices = np.linspace(0, 1., N+1)
- cdict = {}
- for ki,key in enumerate(('red','green','blue')):
- cdict[key] = [ (indices[i], colors_rgba[i-1,ki], colors_rgba[i,ki])
- for i in xrange(N+1) ]
- # Return colormap object.
- return mcolors.LinearSegmentedColormap(cmap.name + "_%d"%N, cdict, 1024)
+ '''
+ Taken from: http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks
+ http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
+ Return a discrete colormap from the continuous colormap cmap.
+
+ cmap: colormap instance, eg. cm.jet.
+ N: number of colors.
+
+ Example
+ x = resize(arange(100), (5,100))
+ djet = cmap_discretize(cm.jet, 5)
+ imshow(x, cmap=djet)
+ '''
+
+ if type(cmap) == str:
+ cmap = plt.get_cmap(cmap)
+ colors_i = np.concatenate((np.linspace(0, 1., N), (0.,0.,0.,0.)))
+ colors_rgba = cmap(colors_i)
+ indices = np.linspace(0, 1., N+1)
+ cdict = {}
+ for ki,key in enumerate(('red','green','blue')):
+ cdict[key] = [ (indices[i], colors_rgba[i-1,ki], colors_rgba[i,ki])
+ for i in xrange(N+1) ]
+ # Return colormap object.
+ return mcolors.LinearSegmentedColormap(cmap.name + "_%d"%N, cdict, 1024)
#******************************************************************
-
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/code/mccSearchUI.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchUI.py b/mccsearch/code/mccSearchUI.py
index 45dc73a..885ff43 100644
--- a/mccsearch/code/mccSearchUI.py
+++ b/mccsearch/code/mccSearchUI.py
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
'''
# Wizard for running the mccSearch program
'''
[26/50] [abbrv] git commit: Remove unused imports from mainProg.py
Posted by le...@apache.org.
Remove unused imports from mainProg.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/587b2033
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/587b2033
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/587b2033
Branch: refs/heads/master
Commit: 587b2033e71f2102c27441b45023a133f92a05e5
Parents: ebd2ef2
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 17:12:11 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Mon Aug 25 17:12:11 2014 -0700
----------------------------------------------------------------------
mccsearch/code/mainProg.py | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/587b2033/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index bb032fc..71d6278 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -19,13 +19,8 @@
# running the program
'''
-import sys
import networkx as nx
-from mccSearch
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
+import mccSearch
import subprocess
@@ -82,7 +77,6 @@ def main():
print ("-"*80)
print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
print ("-"*80)
- #sys.exit()
print "\n -------------- TESTING findMCCs ----------"
MCCList,MCSList = mccSearch.findMCC(prunedGraph)
print ("-"*80)
[38/50] [abbrv] git commit: remove unnecessary files
Posted by le...@apache.org.
remove unnecessary files
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/c1e22a93
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/c1e22a93
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/c1e22a93
Branch: refs/heads/master
Commit: c1e22a93f8b9a00ae9eb2826d084594065996099
Parents: 99f6e36
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 20:59:22 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:04 2014 -0700
----------------------------------------------------------------------
mccsearch/code/files.pyc | Bin 18453 -> 0 bytes
mccsearch/code/mainProgGoyens.py | 126 -
mccsearch/code/mainProgTemp.py | 99 -
mccsearch/code/mainProgWorks.py | 124 -
mccsearch/code/mccSearch.pyc | Bin 98227 -> 0 bytes
mccsearch/code/mccSearchGoyens.py | 3606 ---------------------------
mccsearch/code/newPreprocessingMERG.py | 175 --
mccsearch/code/preprocessingMERG.py | 125 -
mccsearch/code/process.pyc | Bin 29786 -> 0 bytes
9 files changed, 4255 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/files.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.pyc b/mccsearch/code/files.pyc
deleted file mode 100644
index f8a5577..0000000
Binary files a/mccsearch/code/files.pyc and /dev/null differ
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/mainProgGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgGoyens.py b/mccsearch/code/mainProgGoyens.py
deleted file mode 100644
index ad54ab1..0000000
--- a/mccsearch/code/mainProgGoyens.py
+++ /dev/null
@@ -1,126 +0,0 @@
-'''
-# running the program
-# Kim Whitehall Nov 2012
-# Last updated: 15 May 2013
-'''
-
-#import rcmes
-import sys
-import networkx as nx
-import mccSearchGoyens
-#import cloudclusters
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
-
-def main():
- CEGraph = nx.DiGraph()
- prunedGraph = nx.DiGraph()
- MCCList =[]
- MCSList=[]
- MCSMCCNodesList =[]
- allMCSsList =[]
- allCETRMMList =[]
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccSearchGoyens/caseStudy1"
- #directories for the original data
- TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
- CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
- #allMCSsDict ={{}}
-
- # mccSearchGoyens.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # sys.exit()
-
- # # mccSearchGoyens.plotAccTRMM(TRMMCEdirName)
- # sys.exit()
-
- #create main directory and file structure for storing intel
- mccSearchGoyens.createMainDirectory(mainDirStr)
- TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
- CEdirName = mainDirStr+'/MERGnetcdfCEs'
-
- # #mccSearchGoyens.postProcessingNetCDF(3,CEoriDirName)
- # mccSearchGoyens.postProcessingNetCDF(2)
- # sys.exit()
-
- #using merg data ***********
- print "\n -------------- Read MERG Data ----------"
- mergImgs, timeList = mccSearchGoyens.readMergData(CEoriDirName)
- print ("-"*80)
-
- print 'in main', len(mergImgs)
- print "\n -------------- TESTING findCloudElements ----------"
- CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName isnt entered for what ever reason
- # CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearchGoyens.findPrecipRate(TRMMdirName)
- # sys.exit()
- print ("-"*80)
- print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
- print ("-"*80)
- print "\n -------------- TESTING findCloudClusters ----------"
- prunedGraph = mccSearchGoyens.findCloudClusters(CEGraph)
- print ("-"*80)
- print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING findMCCs ----------"
- MCCList,MCSList = mccSearchGoyens.findMCC(prunedGraph)
- print ("-"*80)
- print "MCC List has been acquired ", len(MCCList)
- # for eachList in MCCList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
- print "MCS List has been acquired ", len(MCSList)
- # for eachList in MCSList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING TRMM ----------"
- #now ready to perform various calculations/metrics
- print "\n -------------- TESTING METRICS ----------"
- #MCCTimes, tdelta = mccSearchGoyens.temporalAndAreaInfoMetric(MCCList)
- print ("-"*80)
- #print "MCCTimes is: ", MCCTimes
- print "creating the MCS userfile ", mccSearchGoyens.createTextFile(MCCList)
-
- # print "number of MCCs is: ", mccSearchGoyens.numberOfFeatures(MCCList)
- # print "longest duration is: ", mccSearchGoyens.longestDuration(MCCTimes), "hrs"
- # print "shortest duration is: ", mccSearchGoyens.shortestDuration(MCCTimes), "hrs"
- # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
- # print "Average duration is: ", mccSearchGoyens.averageDuration(MCCTimes), "hrs"
- # print "Average size is: ", mccSearchGoyens.averageFeatureSize(MCCList), "km^2"
-
- # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
- # hist, bin_edges = mccSearchGoyens.commonFeatureSize(allMCSsList)
- # print "Histogram is ", hist, bin_edges
- # plt.bar(bin_edges[:-1], hist, width = 10)
- # plt.xlim(min(bin_edges), max(bin_edges))
- # plt.show()
-
- #some plots that work
- # mccSearchGoyens.plotAccTRMM(MCCList)
- # mccSearchGoyens.displayPrecip(MCCList)
- # mccSearchGoyens.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- # mccSearchGoyens.displaySize(MCCList)
- # mccSearchGoyens.displayPrecip(MCCList)
- # mccSearchGoyens.plotHistograms(MCCList)
- #sys.exit()
- #
- print ("-"*80)
-
- # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
- # print "********** findPrecipRate found"
- # allCETRMMList = mccSearchGoyens.findPrecipRate(CEdirName,TRMMdirName)
- # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
-
- # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
- # print ("-"*80)
- # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
- # print ("-"*80)
- # # sys.exit()
-
-main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/mainProgTemp.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemp.py b/mccsearch/code/mainProgTemp.py
deleted file mode 100644
index 9be65b5..0000000
--- a/mccsearch/code/mainProgTemp.py
+++ /dev/null
@@ -1,99 +0,0 @@
-'''
-# running the program
-'''
-
-import sys
-import networkx as nx
-import mccSearch
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
-import subprocess
-
-
-def main():
- CEGraph = nx.DiGraph()
- prunedGraph = nx.DiGraph()
- MCCList =[]
- MCSList=[]
- MCSMCCNodesList =[]
- allMCSsList =[]
- allCETRMMList =[]
-
- #for GrADs
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- mainDirStr= "/directory/to/where/to/store/outputs"
- TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
- CEoriDirName = "/directory/to/the/MERG/netCDF/files"
-
- #for first time working with the raw MERG zipped files
- # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # ---------------------------------------------------------------------------------
-
-
- #create main directory and file structure for storing intel
- mccSearch.createMainDirectory(mainDirStr)
- TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
- CEdirName = mainDirStr+'/MERGnetcdfCEs'
-
- # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
- # mccSearch.postProcessingNetCDF(3,CEoriDirName)
- # mccSearch.postProcessingNetCDF(2)
- # -------------------------------------------------------------------------------------------------
-
- #let's go!
- print "\n -------------- Read MERG Data ----------"
- mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
- print ("-"*80)
-
- print 'in main', len(mergImgs)
- #print 'timeList', timeList
- print 'TRMMdirName ', TRMMdirName
- print "\n -------------- TESTING findCloudElements ----------"
- CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
- # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
- # ----------------------------------------------------------------------------------------------
- print ("-"*80)
- print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
- print ("-"*80)
- print "\n -------------- TESTING findCloudClusters ----------"
- prunedGraph = mccSearch.findCloudClusters(CEGraph)
- print ("-"*80)
- print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING findMCCs ----------"
- MCCList,MCSList = mccSearch.findMCC(prunedGraph)
- print ("-"*80)
- print "MCC List has been acquired ", len(MCCList)
- print "MCS List has been acquired ", len(MCSList)
- print ("-"*80)
- #now ready to perform various calculations/metrics
- print "\n -------------- TESTING METRICS ----------"
-
- #some calculations/metrics that work that work
- # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
- # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
- # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
- # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
- # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
- # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
- # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
- # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
-
- #some plots that work
- # mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
- # mccSearch.displaySize(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistogram(MCCList)
- #
- print ("-"*80)
-
-main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/mainProgWorks.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgWorks.py b/mccsearch/code/mainProgWorks.py
deleted file mode 100644
index f6eb55d..0000000
--- a/mccsearch/code/mainProgWorks.py
+++ /dev/null
@@ -1,124 +0,0 @@
-'''
-# running the program
-# Kim Whitehall Nov 2012
-# Last updated: 15 May 2013
-'''
-
-#import rcmes
-import sys
-import networkx as nx
-import mccSearch
-#import cloudclusters
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
-
-def main():
- CEGraph = nx.DiGraph()
- prunedGraph = nx.DiGraph()
- MCCList =[]
- MCSList=[]
- MCSMCCNodesList =[]
- allMCSsList =[]
- allCETRMMList =[]
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
- #directories for the original data
- TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
- CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
- #allMCSsDict ={{}}
-
- # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # sys.exit()
-
- # # mccSearch.plotAccTRMM(TRMMCEdirName)
- # sys.exit()
-
- #create main directory and file structure for storing intel
- mccSearch.createMainDirectory(mainDirStr)
- TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
- CEdirName = mainDirStr+'/MERGnetcdfCEs'
-
- # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
- # mccSearch.postProcessingNetCDF(2)
- # sys.exit()
-
- #using merg data ***********
- print "\n -------------- Read MERG Data ----------"
- mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
- print ("-"*80)
-
- print 'in main', len(mergImgs)
- print "\n -------------- TESTING findCloudElements ----------"
- CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName isnt entered for what ever reason
- # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
- # sys.exit()
- print ("-"*80)
- print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
- print ("-"*80)
- print "\n -------------- TESTING findCloudClusters ----------"
- prunedGraph = mccSearch.findCloudClusters(CEGraph)
- print ("-"*80)
- print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING findMCCs ----------"
- MCCList,MCSList = mccSearch.findMCC(prunedGraph)
- print ("-"*80)
- print "MCC List has been acquired ", len(MCCList)
- # for eachList in MCCList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
- print "MCS List has been acquired ", len(MCSList)
- # for eachList in MCSList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING TRMM ----------"
- #now ready to perform various calculations/metrics
- print "\n -------------- TESTING METRICS ----------"
- MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
- print ("-"*80)
- print "MCCTimes is: ", MCCTimes
- print ("-"*80)
- print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
- print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
- print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
- #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
- print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
-
- # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
- # print "Histogram is ", hist, bin_edges
- # plt.bar(bin_edges[:-1], hist, width = 10)
- # plt.xlim(min(bin_edges), max(bin_edges))
- # plt.show()
-
- #some plots that work
- # mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- # mccSearch.displaySize(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistograms(MCCList)
- #sys.exit()
- #
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print "********** findPrecipRate found"
- # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print ("-"*80)
- # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
- # print ("-"*80)
- # # sys.exit()
-
-main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/mccSearch.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.pyc b/mccsearch/code/mccSearch.pyc
deleted file mode 100644
index dc462d1..0000000
Binary files a/mccsearch/code/mccSearch.pyc and /dev/null differ
[07/50] [abbrv] Some linting and refactoring
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/aed34f0b/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index 177aef8..7c647d5 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -1,8 +1,6 @@
'''
# All the functions for the MCC search algorithm
# Following RCMES dataformat in format (t,lat,lon), value
-# Kim Whitehall
-# Mimumum MCS is 3 hours
'''
import datetime
@@ -42,38 +40,34 @@ import process
# --------------------- User defined variables ---------------------
#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
#the first point closest the the value (for the min) from the MERG data is used, etc.
-LATMIN = '5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
-LATMAX = '19.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
-LONMIN = '-9.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
-LONMAX = '5.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+LATMIN = '5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '19.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
+LONMIN = '-6.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '3.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
XRES = 4.0 #x direction spatial resolution in km
YRES = 4.0 #y direction spatial resolution in km
TRES = 1 #temporal resolution in hrs
-T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
-T_BB_MIN = 218 #221 #cooler temp for the center of the system
+LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
#have same rank of the matrix it is being compared against
+#criteria for determining cloud elements and edges
+T_BB_MAX = 243 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
+T_BB_MIN = 218 #cooler temp for the center of the system
CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
-MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
-MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
-LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
-LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
-DIRECTION = 45.0 #general direction of the wind flow in degrees
AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
-MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
-#---Assuming using the MCC function, these will have to be changed
+#---the MCC criteria
ECCENTRICITY_THRESHOLD_MAX = 1.0 #tending to 1 is a circle e.g. hurricane,
-ECCENTRICITY_THRESHOLD_MIN = 0.70 #0.65 #tending to 0 is a linear e.g. squall line
-OUTER_CLOUD_SHIELD_AREA = 80000.0 #100000.0 #km^2
-INNER_CLOUD_SHIELD_AREA = 30000.0 #50000.0 #km^2
+ECCENTRICITY_THRESHOLD_MIN = 0.50 #tending to 0 is a linear e.g. squall line
+OUTER_CLOUD_SHIELD_AREA = 80000.0 #km^2
+INNER_CLOUD_SHIELD_AREA = 30000.0 #km^2
OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
-MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
-MAXIMUM_DURATION = 100 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
-
-#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, MCCs is 6hrs)
+MAXIMUM_DURATION = 24#max number of framce the MCC can last for
#------------------- End user defined Variables -------------------
edgeWeight = [1,2,3] #weights for the graph edges
#graph object fo the CEs meeting the criteria
@@ -94,15 +88,13 @@ def readMergData(dirname):
Output::
A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
criteria for each frame
- **remove below**
- A dictionary of all the MERG data from the files in the directory given.
- The dictionary contains, a string representing the time (a datetime string), a
- tuple (lat,lon,value) representing only the data that meets the temperature requirements
- i.e. T_BB_MAX
Assumptions::
The MERG data has been converted to NETCDF using LATS4D
The data has the same lat/lon format
+
+ TODO:: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
+
'''
global LAT
@@ -136,8 +128,7 @@ def readMergData(dirname):
else:
# Open the first file in the list to read in lats, lons and generate the grid for comparison
tmp = Nio.open_file(filelist[0], format='nc')
- #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
-
+
#clip the lat/lon grid according to user input
#http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
@@ -149,14 +140,11 @@ def readMergData(dirname):
latsraw =[]
lonsraw = []
nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
- #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
tmp.close
for files in filelist:
try:
thisFile = Nio.open_file(files, format='nc')
- #TODO: thisFile = netCDF4.Dataset(files)
-
#clip the dataset according to user lat,lon coordinates
#mask the data and fill with zeros for later
tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
@@ -176,8 +164,6 @@ def readMergData(dirname):
#extend instead of append because getModelTimes returns a list already and we don't
#want a list of list
timelist.extend(time2store)
- #to get the actual masked data only
- #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
mergImgs.extend(tempMaskedValue)
thisFile.close
thisFile = None
@@ -195,21 +181,20 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
Determines the contiguous boxes for a given time of the satellite images i.e. each frame
using scipy ndimage package
- Input::
- 3 variables
- sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
+ Input::
+ mergImgs: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
maximum acceptable temperature, T_BB_MAX
timelist: a list of python datatimes
TRMMdirName (optional): string representing the path where to find the TRMM datafiles
Output::
- 1 variable
- cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
- The dictionary is
+ CLOUD_ELEMENT_GRAPH: a Networkx directed graph where each node contains the information in cloudElementDict
+ The nodes are determined according to the area of contiguous squares. The nodes are linked through weighted edges.
+
cloudElementDict = {'uniqueID': unique tag for this CE,
'cloudElementTime': time of the CE,
'cloudElementLatLon': (lat,lon,value) of MERG data of CE,
- 'cloudElementCenter':tuple of floating-point (lat,lon) representing the CE's center
+ 'cloudElementCenter':list of floating-point [lat,lon] representing the CE's center
'cloudElementArea':floating-point representing the area of the CE,
'cloudElementEccentricity': floating-point representing the shape of the CE,
'cloudElementTmax':integer representing the maximum Tb in CE,
@@ -248,7 +233,6 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
precip =[]
TRMMCloudElementLatLons =[]
- #edgeWeight = [1,2]
minCELatLimit = 0.0
minCELonLimit = 0.0
maxCELatLimit = 0.0
@@ -258,12 +242,9 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
#openfile for storing ALL cloudElement information
cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
-
-
#openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
- #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
-
+
#NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
for t in xrange(mergImgs.shape[0]):
#-------------------------------------------------
@@ -299,9 +280,8 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
numOfBoxes = np.count_nonzero(cloudElement)
cloudElementArea = numOfBoxes*XRES*YRES
- #If the area is greater than the area required, then start considering as CE
- #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
- #if it does, still consider as CE
+ #If the area is greater than the area required, or if the area is smaller than the suggested area, check if it meets a convective fraction requirement
+ #consider as CE
if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
@@ -514,7 +494,7 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
thisCElen = len(cloudElementLatLons)
percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
- #change weights to integers because the built in shortest path chokes on floating pts
+ #change weights to integers because the built in shortest path chokes on floating pts according to Networkx doc
#according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
if percentageOverlap >= 0.95:
CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
@@ -598,18 +578,19 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
print ("*"*80)
#hierachial graph output
- #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
+ graphTitle = "Cloud Elements observed over somewhere from 0000Z to 0000Z"
drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
return CLOUD_ELEMENT_GRAPH
#******************************************************************
-def findPrecipRate(TRMMdirName):
- '''
- TODO: needs fixing
- Purpose:: Determines the precipitation rates for MCSs found if TRMMdirName was not entered in findCloudElements this can be used
+def findPrecipRate(TRMMdirName, timelist):
+ '''
+ Purpose::
+ Determines the precipitation rates for MCSs found if TRMMdirName was not entered in findCloudElements this can be used
- Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
+ Input::
+ TRMMdirName: a string representing the directory for the original TRMM netCDF files
+ timelist: a list of python datatimes
Output:: a list of dictionary of the TRMM data
NB: also creates netCDF with TRMM data for each CE (for post processing) index
@@ -631,137 +612,141 @@ def findPrecipRate(TRMMdirName):
files.sort(key=lambda x: os.path.getmtime(x))
for afile in files:
- #try:
- print "in for ", afile
fullFname = os.path.splitext(afile)[0]
noFrameExtension = (fullFname.replace("_","")).split('F')[0]
CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
fileDateTime = fileDateTimeChar.replace("-","")
fileDate = fileDateTime[:-6]
- fileHr=fileDateTime[-6:-4]
+ fileHr1=fileDateTime[-6:-4]
cloudElementData = Dataset(afile,'r', format='NETCDF4')
- brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
+ brightnesstemp1 = cloudElementData.variables['brightnesstemp'][:,:,:]
latsrawCloudElements = cloudElementData.variables['latitude'][:]
lonsrawCloudElements = cloudElementData.variables['longitude'][:]
+
+ brightnesstemp = np.squeeze(brightnesstemp1, axis=0)
+
+ if int(fileHr1) % temporalRes == 0:
+ fileHr = fileHr1
+ else:
+ fileHr = (int(fileHr1)/temporalRes) * temporalRes
+
+ if fileHr < 10:
+ fileHr = '0'+str(fileHr)
+ else:
+ str(fileHr)
+
+ TRMMfileName = TRMMdirName+"/3B42."+ str(fileDate) + "."+str(fileHr)+".7A.nc"
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
- if int(fileHr) % 3 == 0:
- TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
- #print "TRMMfileName is ", TRMMfileName
- TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
- precipRate = TRMMData.variables['pcp'][:,:,:]
- latsrawTRMMData = TRMMData.variables['latitude'][:]
- lonsrawTRMMData = TRMMData.variables['longitude'][:]
- lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
- LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
-
- #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
-
- precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
- #---------regrid the TRMM data to the MERG dataset ----------------------------------
- #regrid using the do_regrid stuff from the Apache OCW
- regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
- regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
- #----------------------------------------------------------------------------------
-
- # #get the lat/lon info from cloudElement
- latCEStart = LAT[0][0]
- latCEEnd = LAT[-1][0]
- lonCEStart = LON[0][0]
- lonCEEnd = LON[0][-1]
-
- #get the lat/lon info for TRMM data (different resolution)
- latStartT = find_nearest(latsrawTRMMData, latCEStart)
- latEndT = find_nearest(latsrawTRMMData, latCEEnd)
- lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
- lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
- latStartIndex = np.where(latsrawTRMMData == latStartT)
- latEndIndex = np.where(latsrawTRMMData == latEndT)
- lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
- lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
-
- #get the relevant TRMM info
- CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
- TRMMData.close()
+ # ------ NETCDF File stuff ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
+ currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
+ currNetCDFTRMMData.calendar = 'standard'
+ currNetCDFTRMMData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFTRMMData.createDimension('time', None)
+ currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ fileDateTime[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp1.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+ for index,value in np.ndenumerate(brightnesstemp):
+ lat_index, lon_index = index
+ currTimeValue = 0
+ if value > 0:
+
+ finalCETRMMvalues[0,lat_index,lon_index] = regriddedTRMM[int(np.where(LAT[:,0]==LAT[lat_index,0])[0]), int(np.where(LON[0,:]==LON[0,lon_index])[0])]
-
- # ------ NETCDF File stuff ------------------------------------
- thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
- # print "thisFileName ", thisFileName
- # sys.exit()
- currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
- currNetCDFData.calendar = 'standard'
- currNetCDFData.conventions = 'COARDS'
- # dimensions
- currNetCDFData.createDimension('time', None)
- currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
- currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
- # variables
- TRMMprecip = ('time','lat', 'lon',)
- times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
- longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
- rainFallacc.units = 'mm'
-
- longitude[:] = LON[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
-
- latitude[:] = LAT[:,0]
- latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
-
- finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
- #-----------End most of NETCDF file stuff ------------------------------------
- #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
- for index,value in np.ndenumerate(brightnesstemp):
- #print "index, value ", index, value
- time_index, lat_index, lon_index = index
- currTimeValue = 0
- if value > 0:
- finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
-
+ rainFallacc[:] = finalCETRMMvalues
+ currNetCDFTRMMData.close()
+
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ precipTotal += value
+
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+
+ try:
+ minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ minCEprecipRate = 0.0
+
+ try:
+ maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ maxCEprecipRate = 0.0
+
+ #add info to CLOUDELEMENTSGRAPH
+ #TODO try block
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
+ if eachdict[1]['uniqueID'] == CEuniqueID:
+ if not 'cloudElementPrecipTotal' in eachdict[1].keys():
+ eachdict[1]['cloudElementPrecipTotal'] = precipTotal
+ if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
+ eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
+ if not 'TRMMArea' in eachdict[1].keys():
+ eachdict[1]['TRMMArea'] = TRMMArea
+ if not 'CETRMMmin' in eachdict[1].keys():
+ eachdict[1]['CETRMMmin'] = minCEprecipRate
+ if not 'CETRMMmax' in eachdict[1].keys():
+ eachdict[1]['CETRMMmax'] = maxCEprecipRate
- # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
- # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
- # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
- # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
- # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
- # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
- # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
- # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
- # # precipTotal = value*temporalRes*1.0
- # # prevLatValue = currLatvalue
- # # prevLonValue = currLonValue
-
- TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
- TRMMArea = TRMMnumOfBoxes*XRES*YRES
- rainFallacc[:] = finalCETRMMvalues
- currNetCDFData.close()
- for index, value in np.ndenumerate(finalCETRMMvalues):
- #print "index, value ", index, value
- precipTotal += value
-
- # #add info to the dictionary and to the list
- # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
- # allCEnodesTRMMdata.append(TRMMdataDict)
- # print "precipTotal is ", precipTotal
- #add info to CLOUDELEMENTSGRAPH
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
- if eachdict[1]['uniqueID'] == CEuniqueID:
- if not 'cloudElementPrecipTotal' in eachdict[1].keys():
- eachdict[1]['cloudElementPrecipTotal'] = precipTotal
- if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
- eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
- if not 'TRMMArea' in eachdict[1].keys():
- eachdict[1]['TRMMArea'] = TRMMArea
#clean up
precipTotal = 0.0
latsrawTRMMData =[]
@@ -777,13 +762,16 @@ def findPrecipRate(TRMMdirName):
#******************************************************************
def findCloudClusters(CEGraph):
'''
- Purpose:: Determines the cloud clusters properties from the subgraphs in
+ Purpose::
+ Determines the cloud clusters properties from the subgraphs in
the graph i.e. prunes the graph according to the minimum depth
- Input:: CEGraph: 1 graph - a directed graph of the CEs with weighted edges
- according the area overlap between nodes (CEs) of consectuive frames
+ Input::
+ CEGraph: a Networkx directed graph of the CEs with weighted edges
+ according the area overlap between nodes (CEs) of consectuive frames
- Output:: PRUNED_GRAPH: 1 graph - a directed graph of with CCs/ MCSs
+ Output::
+ PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
'''
@@ -825,8 +813,7 @@ def findCloudClusters(CEGraph):
print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
print ("*"*80)
- #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
+ graphTitle = "Cloud Clusters observed over somewhere during sometime"
drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
cloudClustersFile.close
@@ -834,13 +821,17 @@ def findCloudClusters(CEGraph):
#******************************************************************
def findMCC (prunedGraph):
'''
- Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
+ Purpose::
+ Determines if subtree is a MCC according to Laurent et al 1998 criteria
- Input:: prunedGraph: a Networkx Graph representing the CCs
+ Input::
+ prunedGraph: a Networkx Graph representing the CCs
- Output:: finalMCCList: a list of list of tuples representing a MCC
+ Output::
+ finalMCCList: a list of list of tuples representing a MCC
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
MCCList = []
@@ -997,18 +988,22 @@ def findMCC (prunedGraph):
#******************************************************************
def traverseTree(subGraph,node, queue, checkedNodes=None):
'''
- Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+ Purpose::
+ To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
- Input:: subGraph: a Networkx DiGraph representing a CC
+ Input::
+ subGraph: a Networkx DiGraph representing a CC
lengthOfsubGraph: an integer representing the length of the subgraph
node: a string representing the node currently being checked
queue: a list of strings representing a list of nodes in a stack functionality
i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
checkedNodes: a list of strings representing the list of the nodes in the traversal
- Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
+ Output::
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
'''
if len(checkedNodes) == len(subGraph):
@@ -1026,7 +1021,7 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
for child in downOneLevel:
if child not in checkedNodes and child not in queue:
queue.insert(0,child)
- #downCheckFlag = True
+
queue.insert(0,parent)
for child in downOneLevel:
@@ -1036,10 +1031,8 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
else:
queue.append(child)
- #print "^^^^^stack ", stack
for eachNode in queue:
if eachNode not in checkedNodes:
- #print "if eachNode ", checkedNodes, eachNode, stack
checkedNodes.append(eachNode)
return traverseTree(subGraph, eachNode, queue, checkedNodes)
@@ -1047,14 +1040,17 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
#******************************************************************
def checkedNodesMCC (prunedGraph, nodeList):
'''
- Purpose :: Determine if this path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
+ Purpose ::
+ Determine if this path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- nodeList: list of strings (CE ID) from the traversal
+ Input::
+ prunedGraph: a Networkx Graph representing all the cloud clusters
+ nodeList: list of strings (CE ID) from the traversal
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
- dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+ Output::
+ potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
'''
CounterCriteriaAFlag = False
@@ -1063,14 +1059,13 @@ def checkedNodesMCC (prunedGraph, nodeList):
MATURITYFLAG = False
DECAYFLAG = False
thisdict = {} #will have the same items as the cloudElementDict
- cloudElementArea = 0.0
+ cloudElementAreaB = 0.0
+ cloudElementAreaA = 0.0
epsilon = 0.0
frameNum =0
oldNode =''
potentialMCCList =[]
durationAandB = 0
- CounterCriteriaA = 0
- CountercriteriaB = 0
#check for if the list contains only one string/node
if type(nodeList) is str:
@@ -1085,41 +1080,62 @@ def checkedNodesMCC (prunedGraph, nodeList):
existingFrameFlag = False
if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
- #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
CounterCriteriaAFlag = True
INITIATIONFLAG = True
MATURITYFLAG = False
- #check if criteriaB is meet
- cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
-
- #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
- if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
- #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- CounterCriteriaBFlag = True
- #append this information on to the dictionary
- addInfothisDict(node, cloudElementArea, criteriaB)
- INITIATIONFLAG = False
- MATURITYFLAG = True
- stage = 'M'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- else:
- #criteria B failed
- CounterCriteriaBFlag = False
- if INITIATIONFLAG == True:
- stage = 'I'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
-
- elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
- DECAYFLAG = True
- MATURITYFLAG = False
- stage = 'D'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ #check if criteriaA is met
+ cloudElementAreaA, criteriaA = checkCriteria(thisdict['cloudElementLatLon'], OUTER_CLOUD_SHIELD_TEMPERATURE)
+ #TODO: calcuate the eccentricity at this point and read over????or create a new field in the dict
+
+ if cloudElementAreaA >= OUTER_CLOUD_SHIELD_AREA:
+ #check if criteriaB is met
+ cloudElementAreaB,criteriaB = checkCriteria(thisdict['cloudElementLatLon'], INNER_CLOUD_SHIELD_TEMPERATURE)
+
+ #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
+ if cloudElementAreaB >= INNER_CLOUD_SHIELD_AREA:
+ #TODO: add another field to the dictionary for the OUTER_AREA_SHIELD area
+ CounterCriteriaBFlag = True
+ #append this information on to the dictionary
+ addInfothisDict(node, cloudElementAreaB, criteriaB)
+ INITIATIONFLAG = False
+ MATURITYFLAG = True
+ stage = 'M'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria B failed
+ CounterCriteriaBFlag = False
+ if INITIATIONFLAG == True:
+ stage = 'I'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
+ DECAYFLAG = True
+ MATURITYFLAG = False
+ stage = 'D'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria A failed
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ #add as a CE before or after the main feature
+ if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
+ stage = "D"
+ DECAYFLAG = True
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+
else:
#criteria A failed
CounterCriteriaAFlag = False
CounterCriteriaBFlag = False
- #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- #add as a CE before or after the main feature
+ #add as a CE before or after the main feature
if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
stage ="I"
potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
@@ -1135,16 +1151,19 @@ def checkedNodesMCC (prunedGraph, nodeList):
#******************************************************************
def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
'''
- Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
+ Purpose::
+ Utility function to determine if a path is (or is part of) a MCC and provides
preliminary information regarding the stages of the feature
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- node: a string representing the cloud element currently being assessed
- CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
- CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
+ Input::
+ prunedGraph: a Networkx Graph representing all the cloud clusters
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ node: a string representing the cloud element currently being assessed
+ CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
+ CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ Output::
+ potentialMCCList: list of dictionaries representing all possible MCC within the path
dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
'''
@@ -1173,7 +1192,7 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
if predecessorsFlag == True:
- for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
+ for eachNode in potentialMCCList[index]["possMCCList"]:
if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
existingFrameFlag = True
@@ -1218,7 +1237,7 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
if successorsFlag == True:
- for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
+ for eachNode in potentialMCCList[index]["possMCCList"]:
if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
existingFrameFlag = True
@@ -1310,16 +1329,19 @@ def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag
#******************************************************************
def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
'''
- Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
+ Purpose::
+ Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
- Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
- upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
- node: a string representing the cloud element currently being assessed
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
+ Input::
+ prunedGraph:a Networkx Graph representing all the cloud clusters
+ upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
+ node: a string representing the cloud element currently being assessed
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
- Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
- index: an integer representing the location in the potentialMCCList where thisFlag occurs
+ Output::
+ thisFlag: a boolean representing whether the list passed has in the parent or child of the node
+ index: an integer representing the location in the potentialMCCList where thisFlag occurs
'''
thisFlag = False
@@ -1337,7 +1359,7 @@ def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
index = -1
for MCCDict in potentialMCCList:
index += 1
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ if aNode in list(x[0] for x in MCCDict[checkList]):
thisFlag = True
#get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
return thisFlag, index
@@ -1350,7 +1372,7 @@ def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
for MCCDict in potentialMCCList:
index += 1
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ if aNode in list(x[0] for x in MCCDict[checkList]):
thisFlag = True
return thisFlag, index
@@ -1358,12 +1380,16 @@ def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
#******************************************************************
def maxExtentAndEccentricity(eachList):
'''
- Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
+ Purpose::
+ Perform the final check for MCC based on maximum extent and eccentricity criteria
+
+ Input::
+ eachList: a list of strings representing the node of the possible MCCs within a path
- Input:: eachList: a list of strings representing the node of the possible MCCs within a path
+ Output::
+ maxShieldNode: a string representing the node with the maximum maxShieldNode
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
- Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
- definiteMCCFlag: a boolean indicating that the MCC has met all requirements
'''
maxShieldNode =''
maxShieldArea = 0.0
@@ -1385,13 +1411,16 @@ def maxExtentAndEccentricity(eachList):
#******************************************************************
def findMaxDepthAndMinPath (thisPathDistanceAndLength):
'''
- Purpose:: To determine the maximum depth and min path for the headnode
+ Purpose::
+ To determine the maximum depth and min path for the headnode
- Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
- thisPathDistanceAndLength({distance}, {path})
+ Input::
+ tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
+ thisPathDistanceAndLength({distance}, {path})
{distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
- Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
+ Output::
+ tuple of the max pathLength and min pathDistance as a tuple (like what was input)
minDistanceAndMaxPath = ({distance},{path})
'''
maxPathLength = 0
@@ -1416,30 +1445,37 @@ def findMaxDepthAndMinPath (thisPathDistanceAndLength):
if pathLength == maxPathLength :
if pathDistance <= minPath:
minPath = pathLength
- #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
+ #store details if absolute minPath and deepest
minDistanceAndMaxPath = (pathDistance, path)
return minDistanceAndMaxPath
#******************************************************************
def thisDict (thisNode):
'''
- Purpose:: return dictionary from graph if node exist in tree
+ Purpose::
+ Return dictionary from graph if node exist in tree
- Input:: String - thisNode
+ Input::
+ thisNode: a string representing the CE to get the information for
- Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
+ Output ::
+ eachdict[1]: a dictionary representing the info associated with thisNode from the graph
'''
for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
if eachdict[1]['uniqueID'] == thisNode:
return eachdict[1]
#******************************************************************
-def checkCriteriaB (thisCloudElementLatLon):
+def checkCriteria (thisCloudElementLatLon, aTemperature):
'''
- Purpose:: Determine if criteria B is met for a CEGraph
+ Purpose::
+ Determine if criteria B is met for a CEGraph
- Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
+ Input::
+ thisCloudElementLatLon: 2D array of (lat,lon) variable from the node dictionary being currently considered
+ aTemperature:a integer representing the temperature maximum for masking
- Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
+ Output ::
+ cloudElementArea: a floating-point number representing the area in the array that meet the criteria - criteriaB
'''
cloudElementCriteriaBLatLon=[]
@@ -1453,25 +1489,20 @@ def checkCriteriaB (thisCloudElementLatLon):
minLon = min(x[1]for x in thisCloudElementLatLon)
maxLon = max(x[1]for x in thisCloudElementLatLon)
- #print "minLat, maxLat, minLon, maxLon ", minLat, maxLat, minLon, maxLon
minLatIndex = np.argmax(LAT[:,0] == minLat)
maxLatIndex = np.argmax(LAT[:,0]== maxLat)
minLonIndex = np.argmax(LON[0,:] == minLon)
maxLonIndex = np.argmax(LON[0,:] == maxLon)
- #print "minLatIndex, maxLatIndex, minLonIndex, maxLonIndex ", minLatIndex, maxLatIndex, minLonIndex, maxLonIndex
-
criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
for x in thisCloudElementLatLon:
#to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
#index given in the original array to get the indices for the new array
- #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
- #print criteriaBframe
- #keep only those values < TBB_MAX
- tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
+ #keep only those values < aTemperature
+ tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= aTemperature), fill_value = 0)
#get the actual values that the mask returned
criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
@@ -1493,10 +1524,6 @@ def checkCriteriaB (thisCloudElementLatLon):
cloudElementArea = 0.0
return cloudElementArea, cloudElementCriteriaBLatLon
- print "after loc", loc
- print "************************"
- #print "criteriaB ", criteriaB.shape
- print criteriaB
try:
cloudElementCriteriaB = ma.zeros((criteriaB.shape))
cloudElementCriteriaB =criteriaB[loc]
@@ -1513,7 +1540,7 @@ def checkCriteriaB (thisCloudElementLatLon):
cloudElementCriteriaBLatLon.append(lat_lon_tuple)
cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
- #do some
+ #do some cleaning up
tempMask =[]
criteriaB =[]
cloudElementCriteriaB=[]
@@ -1522,11 +1549,13 @@ def checkCriteriaB (thisCloudElementLatLon):
#******************************************************************
def hasMergesOrSplits (nodeList):
'''
- Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
- Input:: nodeList - list of nodes from a path
- Output:: two list_vars_in_file
- splitList list of all the nodes in the path that split
- mergeList list of all the nodes in the path that merged
+ Purpose::
+ Determine if nodes within a path defined from shortest_path splittingNodeDict
+ Input::
+ nodeList: list of strings representing the nodes from a path
+ Output::
+ splitList: a list of strings representing all the nodes in the path that split
+ mergeList: a list of strings representing all the nodes in the path that merged
'''
mergeList=[]
splitList=[]
@@ -1546,13 +1575,16 @@ def hasMergesOrSplits (nodeList):
#******************************************************************
def allAncestors(path, aNode):
'''
- Purpose:: Utility script to provide the path leading up to a nodeList
+ Purpose::
+ Utility script to provide the path leading up to a nodeList
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for parents
+ Input::
+ path: a list of strings representing the nodes in the path
+ aNode: a string representing a node to be checked for parents
- Output:: list of strings - path: the list of the nodes connected to aNode through its parents
- integer - numOfChildren: the number of parents of the node passed
+ Output::
+ path: a list of strings representing the list of the nodes connected to aNode through its parents
+ numOfChildren: an integer representing the number of parents of the node passed
'''
numOfParents = PRUNED_GRAPH.in_degree(aNode)
@@ -1569,13 +1601,16 @@ def allAncestors(path, aNode):
#******************************************************************
def allDescendants(path, aNode):
'''
- Purpose:: Utility script to provide the path leading up to a nodeList
+ Purpose::
+ Utility script to provide the path leading up to a nodeList
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for children
+ Input::
+ path: a list of strings representing the nodes in the path
+ aNode: a string representing a node to be checked for children
- Output:: list of strings - path: the list of the nodes connected to aNode through its children
- integer - numOfChildren: the number of children of the node passed
+ Output::
+ path: a list of strings representing the list of the nodes connected to aNode through its children
+ numOfChildren: an integer representing the number of children of the node passed
'''
numOfChildren = PRUNED_GRAPH.out_degree(aNode)
@@ -1594,13 +1629,15 @@ def allDescendants(path, aNode):
#******************************************************************
def addInfothisDict (thisNode, cloudElementArea,criteriaB):
'''
- Purpose:: update original dictionary node with information
+ Purpose::
+ Update original dictionary node with information
- Input:: String - thisNode
- float - cloudElementArea,
- masked array of floats meeting the criteria - criteriaB
+ Input::
+ thisNode: a string representing the unique ID of a node
+ cloudElementArea: a floating-point number representing the area of the cloud element
+ criteriaB: a masked array of floating-point numbers representing the lat,lons meeting the criteria
- Output ::
+ Output:: None
'''
for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
@@ -1613,8 +1650,9 @@ def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
'''
Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
- Input:: String - thisNode
- String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
+ Input::
+ thisNode: a string representing the unique ID of a node
+ nodeBehaviorIdentifier: a string representing the behavior S- split, M- merge, B- both split and merge, N- neither split or merge
Output :: None
@@ -1627,11 +1665,12 @@ def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
#******************************************************************
def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
'''
- Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+ Purpose::
+ Add an identifier to the node dictionary to indicate splitting, merging or neither node
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
+ Input::
+ thisNode: a string representing the unique ID of a node
+ nodeMCSIdentifier: a string representing the stage of the MCS lifecyle 'I' for Initiation, 'M' for Maturity, 'D' for Decay
Output :: None
@@ -1644,11 +1683,12 @@ def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
#******************************************************************
def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
'''
- Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
+ Purpose::
+ Update an identifier to the node dictionary to indicate splitting, merging or neither node
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
+ Input::
+ thisNode: thisNode: a string representing the unique ID of a node
+ nodeMCSIdentifier: a string representing the stage of the MCS lifecyle 'I' for Initiation, 'M' for Maturity, 'D' for Decay
Output :: None
@@ -1667,12 +1707,10 @@ def eccentricity (cloudElementLatLon):
values tending to 0 are more linear
Input::
- 1 variable
- cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
+ cloudElementLatLon: 2D array in (lat,lon) representing T_bb contiguous squares
Output::
- 1 variable
- epsilon: a float representing the eccentricity of the matrix passed
+ epsilon: a floating-point representing the eccentricity of the matrix passed
'''
@@ -1698,14 +1736,12 @@ def cloudElementOverlap (currentCELatLons, previousCELatLons):
Determines the percentage overlap between two list of lat-lons passed
Input::
- 2 sorted list of tuples
- currentCELatLons - the list of tuples for the current CE
- previousCELatLons - the list of tuples for the other CE being considered
+ currentCELatLons: a list of tuples for the current CE
+ previousCELatLons: a list of tuples for the other CE being considered
Output::
- 2 variables
- percentageOverlap - a float representing the number of overlapping lat_lon tuples
- areaOverlap - a floating-point number representing the area overlapping
+ percentageOverlap: a floating-point representing the number of overlapping lat_lon tuples
+ areaOverlap: a floating-point number representing the area overlapping
'''
@@ -1732,13 +1768,15 @@ def cloudElementOverlap (currentCELatLons, previousCELatLons):
#******************************************************************
def findCESpeed(node, MCSList):
'''
- Purpose:: to determine the speed of the CEs
- uses vector displacement delta_lat/delta_lon (y/x)
+ Purpose::
+ To determine the speed of the CEs uses vector displacement delta_lat/delta_lon (y/x)
- Input:: node: a string representing the CE
- MCSList: a list of strings representing the feature
+ Input::
+ node: a string representing the CE
+ MCSList: a list of strings representing the feature
Output::
+ CEspeed: a floating-point number representing the speed of the CE
'''
@@ -1750,42 +1788,23 @@ def findCESpeed(node, MCSList):
theList = CLOUD_ELEMENT_GRAPH.successors(node)
nodeLatLon=thisDict(node)['cloudElementCenter']
- #print "nodeLatLon ", nodeLatLon
-
-
+
for aNode in theList:
if aNode in MCSList:
#if aNode is part of the MCSList then determine distance
aNodeLatLon = thisDict(aNode)['cloudElementCenter']
- #print "aNodeLatLon ", aNodeLatLon
#calculate CE speed
#checking the lats
nodeLatLon[0] += 90.0
aNodeLatLon[0] += 90.0
- delta_lat = (nodeLatLon[0] - aNodeLatLon[0]) #convert to m
- #delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
- #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
-
- #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
- #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
- # if nodeLatLon[1] < 0.0:
- # nodeLatLon[1] += 360.0
- # if aNodeLatLon[1] <= 0.0:
- # delta_lon = aNodeLatLon[1] + 360.0
- #0E --> 360 and E lons > west lons
+ delta_lat = (nodeLatLon[0] - aNodeLatLon[0])
nodeLatLon[1] += 360.0
aNodeLatLon[1] += 360.0
- delta_lon = (nodeLatLon[1] - aNodeLatLon[1]) #convert to m
- #delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
-
- #print "delta_lat, delta_lon ", delta_lat, delta_lon
-
- #theSpeed = abs(((delta_lat*LAT_DISTANCE*1000)/(delta_lon*LON_DISTANCE*1000))/(TRES*3600)) #convert to s --> m/s
+ delta_lon = (nodeLatLon[1] - aNodeLatLon[1])
theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
CEspeed.append(theSpeed)
- #print "aNode CE speed is ", aNode, (((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600)), theSpeed
-
+
if not CEspeed:
return 0.0
else:
@@ -1802,13 +1821,11 @@ def maenumerate(mArray):
Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
Input::
- 1 variable
- mArray - the masked array returned from the ma.array() command
+ mArray: the masked array returned from the ma.array() command
Output::
- 1 variable
- maskedValues - 3D (t,lat,lon), value of only masked values
+ maskedValues: 3D (t,lat,lon), value of only masked values
'''
@@ -1820,9 +1837,11 @@ def maenumerate(mArray):
#******************************************************************
def createMainDirectory(mainDirStr):
'''
- Purpose:: to create the main directory for storing information and
- the subdirectories for storing information
- Input:: mainDir: a directory for where all information generated from
+ Purpose::
+ To create the main directory for storing information and
+ the subdirectories for storing information
+ Input::
+ mainDir: a directory for where all information generated from
the program are to be stored
Output:: None
@@ -2001,7 +2020,6 @@ def postProcessingNetCDF(dataset, dirName = None):
#Just incase the X11 server is giving problems
subprocess.call('export DISPLAY=:0.0', shell=True)
- print "coreDir is ", str(coreDir)
prevFrameNum = 0
if dataset == 1:
@@ -2183,16 +2201,20 @@ def postProcessingNetCDF(dataset, dirName = None):
#******************************************************************
def drawGraph (thisGraph, graphTitle, edgeWeight=None):
'''
- Purpose:: Utility function to draw graph in the hierachial format
+ Purpose::
+ Utility function to draw graph in the hierachial format
- Input:: a Networkx graph
- a string representing the graph title
- a list of integers representing the edge weights in the graph
+ Input::
+ thisGraph: a Networkx directed graph
+ graphTitle: a string representing the graph title
+ edgeWeight: (optional) a list of integers representing the edge weights in the graph
+
Output:: None
+
'''
imgFilename = MAINDIRECTORY+'/images/'+ graphTitle+".gif"
- fig=plt.figure(facecolor='white', figsize=(16,12)) #figsize=(10,8))#figsize=(16,12))
+ fig=plt.figure(facecolor='white', figsize=(16,12))
edge95 = [(u,v) for (u,v,d) in thisGraph.edges(data=True) if d['weight'] == edgeWeight[0]]
edge90 = [(u,v) for (u,v,d) in thisGraph.edges(data=True) if d['weight'] == edgeWeight[1]]
@@ -2201,41 +2223,28 @@ def drawGraph (thisGraph, graphTitle, edgeWeight=None):
nx.write_dot(thisGraph, 'test.dot')
plt.title(graphTitle)
pos = nx.graphviz_layout(thisGraph, prog='dot')
- #nx.draw(thisGraph, pos, with_labels=True, arrows=True)
#draw graph in parts
#nodes
nx.draw_networkx_nodes(thisGraph, pos, with_labels=True, arrows=False)
#edges
- nx.draw_networkx_edges(thisGraph, pos, edgelist=edge95, alpha=0.5, arrows=False) #with_labels=True, arrows=True)
+ nx.draw_networkx_edges(thisGraph, pos, edgelist=edge95, alpha=0.5, arrows=False)
nx.draw_networkx_edges(thisGraph, pos, edgelist=edge90, edge_color='b', style='dashed', arrows=False)
nx.draw_networkx_edges(thisGraph, pos, edgelist=edegeOverlap, edge_color='y', style='dashed', arrows=False)
#labels
- nx.draw_networkx_labels(thisGraph,pos, arrows=False) #,font_size=20,font_family='sans-serif')
+ nx.draw_networkx_labels(thisGraph,pos, arrows=False)
plt.axis('off')
plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
#do some clean up...and ensuring that we are in the right dir
os.chdir((MAINDIRECTORY+'/'))
subprocess.call('rm test.dot', shell=True)
- #plt.show()
-#******************************************************************
-def convert_timedelta(duration):
- '''
- Taken from http://stackoverflow.com/questions/14190045/how-to-convert-datetime-timedelta-to-minutes-hours-in-python
-
- '''
-
- days, seconds = duration.days, duration.seconds
- hours = days * 24 + seconds // 3600
- minutes = (seconds % 3600) // 60
- seconds = (seconds % 60)
- return hours, minutes, seconds
#******************************************************************
def tempMaskedImages(imgFilename):
'''
- Purpose:: To generate temperature-masked images for a first pass verification
+ Purpose::
+ To generate temperature-masked images for a first pass verification
Input::
- filename for the gif file
+ imgFilename: filename for the gif file
Output::
None - Gif images for each file of T_bb less than 250K are generated in folder called mergImgs
@@ -2258,61 +2267,47 @@ def tempMaskedImages(imgFilename):
subprocess.call('echo "''\'set clevs 190 200 210 220 230 240 250''\'" >> tempMaskedImages.gs', shell=True)
subprocess.call('echo "''\'d ch4+75''\'" >> tempMaskedImages.gs', shell=True)
subprocess.call('echo "''\'draw title Masked Temp @ '+imgFilename +'\'" >> tempMaskedImages.gs', shell=True)
- #printimCmd = 'printim '+imgFilename +' x1000 y800'
subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800''\'" >> tempMaskedImages.gs', shell=True)
subprocess.call('echo "''\'quit''\'" >> tempMaskedImages.gs', shell=True)
gradscmd = 'grads -blc ' + '\'run tempMaskedImages.gs''\''
subprocess.call(gradscmd, shell=True)
return
#******************************************************************
-#******************************************************************
#
# METRICS FUNCTIONS FOR MERG.PY
+# TODO: rewrite these metrics so that they use the data from the
+# file instead of the graph as this reduce mem resources needed
+#
#
-#------------------------------------------------------------------
-#1)Need to know the starttime and end time
-# - determine most common, least common and avg initiation time
-# - determine duration
-#
-#2)Need criteria B area and temperature
-# - determine max, min and avg area and T
-# - visualization opportunity to show all the criteriaB for time
-# on a map
-#
-#3)Need temperature and area data
-# - calculate the max, min and average temp
-#
-#4)Need lat/lon and T for all MCC
-# - visualization of all MCCs during the time
-# - calculate speed of the system
-# ****** Comparing with other datasets ******
-#5)Need lat/lon and times for all MCCs
-# - use to locate same positions in dataset2 (e.g. TRMM) to
-# determine precip
#******************************************************************
def numberOfFeatures(finalMCCList):
'''
- Purpose:: To count the number of MCCs found for the period
+ Purpose::
+ To count the number of MCCs found for the period
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
+ Input::
+ finalMCCList: a list of list of strings representing a list of list of nodes representing a MCC
- Output::an integer representing the number of MCCs found
-
- Assumptions::
+ Output::
+ an integer representing the number of MCCs found
'''
return len(finalMCCList)
#******************************************************************
def temporalAndAreaInfoMetric(finalMCCList):
'''
- Purpose:: To provide information regarding the temporal properties of the MCCs found
+ Purpose::
+ To provide information regarding the temporal properties of the MCCs found
- Input:: List of dictionaries - finalMCCList: a list of dictionaries of nodes representing a MCC
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Output:: list of dictionaries - allMCCtimes {MCCtimes, starttime, endtime, duration, area): a list of dictionaries
- of MCC temporal details for each MCC in the period considered
+ Output::
+ allMCCtimes: a list of dictionaries {MCCtimes, starttime, endtime, duration, area} representing a list of dictionaries
+ of MCC temporal details for each MCC in the period considered
- Assumptions:: the final time hour --> the event lasted throughout that hr, therefore +1 to endtime
+ Assumptions::
+ the final time hour --> the event lasted throughout that hr, therefore +1 to endtime
'''
#TODO: in real data edit this to use datetime
#starttime =0
@@ -2323,19 +2318,15 @@ def temporalAndAreaInfoMetric(finalMCCList):
MCSArea =[]
if finalMCCList:
- #print "len finalMCCList is: ", len(finalMCCList)
for eachMCC in finalMCCList:
- #print "eachMCC count in temporalInfoMetric is ", len(eachMCC)
#get the info from the node
for eachNode in eachMCC:
- #print "eachNode in temporalInfoMetric is ", eachNode['uniqueID'], eachNode['cloudElementTime']
MCCtimes.append(thisDict(eachNode)['cloudElementTime'])
MCSArea.append(thisDict(eachNode)['cloudElementArea'])
#sort and remove duplicates
MCCtimes=list(set(MCCtimes))
MCCtimes.sort()
- #print "MCCtimes are: ", MCCtimes
tdelta = MCCtimes[1] - MCCtimes[0]
starttime = MCCtimes[0]
endtime = MCCtimes[-1]
@@ -2346,19 +2337,22 @@ def temporalAndAreaInfoMetric(finalMCCList):
MCSArea=[]
else:
allMCCtimes =[]
- tdelta = 0 # 00:00:00
+ tdelta = 0
return allMCCtimes, tdelta
#******************************************************************
def longestDuration(allMCCtimes):
'''
- Purpose:: To determine the longest MCC for the period
+ Purpose::
+ To determine the longest MCC for the period
- Input:: list of dictionaries - allMCCtimes {MCCtimes, starttime, endtime, duration): a list of dictionaries
- of MCC temporal details for each MCC in the period considered
+ Input::
+ allMCCtimes: a list of dictionaries {MCCtimes, starttime, endtime, duration, area} representing a list of dictionaries
+ of MCC temporal details for each MCC in the period considered
- Output::an integer - lenMCC: representing the duration of the longest MCC found
- a list of strings - longestMCC: representing the nodes of longest MCC
+ Output::
+ an integer - lenMCC: representing the duration of the longest MCC found
+ a list of strings - longestMCC: representing the nodes of longest MCC
Assumptions::
@@ -2421,11 +2415,15 @@ def averageDuration(allMCCtimes):
#******************************************************************
def averageTime (allTimes):
'''
- Purpose:: To determine the average time in a list of datetimes
- e.g. of use is finding avg starttime,
- Input:: allTimes: a list of datetimes representing all of a given event e.g. start time
+ Purpose::
+ To determine the average time in a list of datetimes
+ e.g. of use is finding avg starttime,
+ Input::
+ allTimes: a list of datetimes representing all of a given event e.g. start time
+
+ Output::
+ a floating-point number representing the average of the times given
- Output:: a float representing the average of the times given
'''
avgTime = 0
@@ -2466,11 +2464,14 @@ def averageFeatureSize(finalMCCList):
#******************************************************************
def commonFeatureSize(finalMCCList):
'''
- Purpose:: To determine the common (mode) MCC size for the period
+ Purpose::
+ To determine the common (mode) MCC size for the period
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
+ Input::
+ finalMCCList: a list of list of strings representing the list of nodes representing a MCC
- Output::a floating-point representing the average area of a MCC in the period
+ Output::
+ a floating-point representing the average area of a MCC in the period
Assumptions::
@@ -2491,19 +2492,16 @@ def commonFeatureSize(finalMCCList):
hist, bin_edges = np.histogram(thisMCCAvg)
return hist,bin_edges
#******************************************************************
-def commonFeatureDuration (finalMCCList):
- '''
- '''
-#******************************************************************
def displaySize(finalMCCList):
'''
- Purpose:: To create a figure showing the area verse time for each MCS
+ Purpose::
+ To create a figure showing the area verse time for each MCS
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
+ Input::
+ finalMCCList: a list of list of strings representing the list of nodes representing a MCC
- Output:: None
-
- Assumptions::
+ Output::
+ None
'''
timeList =[]
@@ -2517,9 +2515,7 @@ def displaySize(finalMCCList):
#in the graph and calculate the area
if finalMCCList:
- #print "len finalMCCList is: ", len(finalMCCList)
for eachMCC in finalMCCList:
- #print "eachMCC count in temporalInfoMetric is ", len(eachMCC)
#get the info from the node
for node in eachMCC:
eachNode=thisDict(node)
@@ -2542,18 +2538,15 @@ def displaySize(finalMCCList):
#plot info
plt.close('all')
- #title = 'Area verses time for MCS'+ str(count)
- title = 'Area distribution of the MCC over Burkina Faso'
+ title = 'Area distribution of the MCC over somewhere'
fig=plt.figure(facecolor='white', figsize=(18,10)) #figsize=(10,8))#figsize=(16,12))
fig,ax = plt.subplots(1, facecolor='white', figsize=(10,10))
#the data
for node in eachMCC: #for eachNode in eachMCC:
eachNode=thisDict(node)
- #ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'ro')
if eachNode['cloudElementArea'] < 80000 : #2400.00:
ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'bo', markersize=10)
- #eachNode['cloudElementArea'] >= 2400.00 and eachNode['cloudElementArea'] < 10000.00:
elif eachNode['cloudElementArea'] >= 80000.00 and eachNode['cloudElementArea'] < 160000.00:
ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'yo',markersize=20)
else:
@@ -2569,7 +2562,6 @@ def displaySize(finalMCCList):
fig.autofmt_xdate()
plt.subplots_adjust(bottom=0.2)
- #plt.show()
imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif'
plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
@@ -2581,11 +2573,14 @@ def displaySize(finalMCCList):
#******************************************************************
def precipTotals(finalMCCList):
'''
- Purpose:: Precipitation totals associated with a cloud element
+ Purpose::
+ Precipitation totals associated with a cloud element
- Input:: List of dictionaries - finalMCCList: a list of dictionaries of nodes representing a MCC
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Output:: precipTotal - to total amount of precipitation associated
+ Output::
+ precipTotal: a floating-point number representing the total amount of precipitation associated
with the feature
'''
precipTotal = 0.0
@@ -2604,11 +2599,7 @@ def precipTotals(finalMCCList):
if count == 1:
prevHr = int(str(eachNode['cloudElementTime']).replace(" ", "")[-8:-6])
- #print "prevHr ", prevHr
currHr =int(str(eachNode['cloudElementTime']).replace(" ", "")[-8:-6])
- #print "currHr ", currHr
- print "time ",eachNode['cloudElementTime'], prevHr, currHr
- print "eachNode[cloudElementPrecipTotal] ", eachNode['cloudElementPrecipTotal']
if prevHr == currHr:
CEprecip += eachNode['cloudElementPrecipTotal']
else:
@@ -2622,8 +2613,7 @@ def precipTotals(finalMCCList):
prevHr = currHr
MCSPrecip.append(('0',precipTotal))
- print "MCSPrecip ", MCSPrecip
-
+
allMCSPrecip.append(MCSPrecip)
precipTotal =0.0
CEprecip = 0.0
@@ -2637,10 +2627,14 @@ def precipTotals(finalMCCList):
def precipMaxMin(finalMCCList):
'''
TODO: this doesnt work the np.min/max function seems to be not working with the nonzero option..possibly a problem upstream with cloudElementLatLonTRMM
- Purpose:: Precipitation maximum and min rates associated with each CE in MCS
- Input:: List of dictionaries - finalMCCList: a list of dictionaries of nodes representing a MCC
+ Purpose::
+ Precipitation maximum and min rates associated with each CE in MCS
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
+
+ Output::
+ MCSPrecip: a list indicating max and min rate for each CE identified
- Output:: a list tuples indicating max and min rate for each CE identified
'''
maxCEprecip = 0.0
minCEprecip =0.0
@@ -2660,30 +2654,17 @@ def precipMaxMin(finalMCCList):
print "maxCEprecip ", np.max(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
sys.exit()
maxCEprecip = np.max(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.max((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
minCEprecip = np.min(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.min((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
MCSPrecip.append((eachNode['uniqueID'],minCEprecip, maxCEprecip))
- # for node in finalMCCList:
- # eachNode=thisDict(node)
- # #find min and max precip
- # minCEprecip = min((eachNode['cloudElementLatLonTRMM'])[2])
- # maxCEprecip = max((eachNode['cloudElementLatLonTRMM'])[2])
- # MCSPrecip.append((eachNode['uniqueID'],minCEprecip, maxCEprecip))
-
+
else:
for eachMCC in finalMCCList:
#get the info from the node
for node in eachMCC:
eachNode=thisDict(node)
#find min and max precip
- # minCEprecip = min((eachNode['cloudElementLatLonTRMM'])[2])
- # maxCEprecip = max((eachNode['cloudElementLatLonTRMM'])[2])
- # np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
maxCEprecip = np.max(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.max((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
minCEprecip = np.min(eachNode['cloudElementLatLonTRMM'][np.nonzero(eachNode['cloudElementLatLonTRMM'])])
- #np.min((eachNode['cloudElementLatLonTRMM'])[np.nonzero((eachNode['cloudElementLatLonTRMM']))])
MCSPrecip.append((eachNode['uniqueID'],minCEprecip, maxCEprecip))
allMCSPrecip.append(MCSPrecip)
MCSPrecip =[]
@@ -2692,13 +2673,13 @@ def precipMaxMin(finalMCCList):
#******************************************************************
def displayPrecip(finalMCCList):
'''
- Purpose:: To create a figure showing the precip rate verse time for each MCS
+ Purpose::
+ To create a figure showing the precip rate verse time for each MCS
+
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Input:: a list of list of strings - finalMCCList: a list of list of nodes representing a MCC
-
Output:: None
-
- Assumptions::
'''
timeList =[]
@@ -2728,7 +2709,6 @@ def displayPrecip(finalMCCList):
#in the graph and calculate the area
if finalMCCList:
- #print "len finalMCCList is: ", len(finalMCCList)
for eachMCC in finalMCCList:
#get the info from the node
for node in eachMCC:
@@ -2736,10 +2716,8 @@ def displayPrecip(finalMCCList):
if firstTime == True:
xStart = eachNode['cloudElementCenter'][1]#lon
yStart = eachNode['cloudElementCenter'][0]#lat
- #print "eachNode in temporalInfoMetric is ", eachNode['uniqueID'], eachNode['cloudElementTime']
timeList.append(eachNode['cloudElementTime'])
percentagePrecipitating.append((eachNode['TRMMArea']/eachNode['cloudElementArea'])*100.0)
- #areaCenter.append((eachNode['cloudElementCenter'][1]-xStart, eachNode['cloudElementCenter'][0]-yStart))
CEArea.append(eachNode['cloudElementArea'])
nodes.append(eachNode['uniqueID'])
x.append(eachNode['cloudElementCenter'][1])#-xStart)
@@ -2757,13 +2735,11 @@ def displayPrecip(finalMCCList):
#plot info
plt.close('all')
- #title = 'Area verses time for MCS'+ str(count)
title = 'Precipitation distribution of the MCS '
- #fig=plt.figure(facecolor='white', figsize=(10,10)) #figsize=(10,8))#figsize=(16,12))
fig,ax = plt.subplots(1, facecolor='white', figsize=(20,7))
cmap = cm.jet
- ax.scatter(x, y, s=partialArea, c= colorBarTime, edgecolors='none', marker='o', cmap =cmap) #,vmin=colorBarTime[0],vmax=colorBarTime[-1])
+ ax.scatter(x, y, s=partialArea, c= colorBarTime, edgecolors='none', marker='o', cmap =cmap)
colorBarTime=[]
colorBarTime =list(set(timeList))
colorBarTime.sort()
@@ -2799,11 +2775,14 @@ def displayPrecip(finalMCCList):
#******************************************************************
def plotPrecipHistograms(finalMCCList):
'''
- Purpose:: to create plots (histograms) of the each TRMMnetcdfCEs files
+ Purpose::
+ To create plots (histograms) of the each TRMMnetcdfCEs files
- Input:: finalMCCList: the list of strings representing the list of MCCs
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
- Output:: plots
+ Output::
+ plots
'''
num_bins = 5
precip =[]
@@ -2857,11 +2836,9 @@ def plotPrecipHistograms(finalMCCList):
imgFilename = MAINDIRECTORY+'/images/'+str(thisTime)+eachNode['uniqueID']+'TRMMMCS.gif'
plt.savefig(imgFilename, transparent=True)
- #plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
precip =[]
# ------ NETCDF File get info ------------------------------------
- #thisFileName = '/Users/kimwhitehall/Documents/HU/research/mccsearch/TRMMnetcdf/TRMM' + str(thisTime).replace(" ", "_") + eachNode['uniqueID'] +'.nc'
thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + str(thisTime).replace(" ", "_") + eachNode['uniqueID'] +'.nc'
TRMMData = Dataset(thisFileName,'r', format='NETCDF4')
precipRate = TRMMData.variables['precipitation_Accumulation'][:,:,:]
@@ -2872,12 +2849,10 @@ def plotPrecipHistograms(finalMCCList):
totalPrecip = np.add(totalPrecip, precipRate)
# ------ End NETCDF File ------------------------------------
- #if str(thisTime) == lastTime:
for index, value in np.ndenumerate(CEprecipRate):
if value != 0.0:
precip.append(value)
- #print "*******eachNode ", eachNode['uniqueID']
lastTime = str(thisTime)
firstTime = False
else:
@@ -2887,13 +2862,16 @@ def plotPrecipHistograms(finalMCCList):
#******************************************************************
def plotHistogram(aList, aTitle, aLabel):
'''
- Purpose:: to create plots (histograms) of the data entered in aList
+ Purpose::
+ To create plots (histograms) of the data entered in aList
- Input:: aList: the list of floating points representing values for e.g. averageArea, averageDuration, etc.
- aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
- aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
+ Input::
+ aList: the list of floating points representing values for e.g. averageArea, averageDuration, etc.
+ aTitle: a string representing the title and the name of the plot e.g. "Average area [km^2]"
+ aLabel: a string representing the x axis info i.e. the data that is being passed and the units e.g. "Area km^2"
- Output:: plots (gif files)
+ Output::
+ plots (gif files)
'''
num_bins = 10
precip =[]
@@ -2921,32 +2899,32 @@ def plotHistogram(aList, aTitle, aLabel):
plt.xlim(min(binsdg), max(binsdg))
ax.set_xticks(binsdg)#, rotation=45)
ax.set_xlabel(aLabel, fontsize=12)
- #ax.set_ylabel('Frequency', fontsize=12)
ax.set_title(aTitle)
plt.xticks(rotation =45)
# Set the formatter
- #plt.gca().yaxis.set_major_formatter(formatter)
plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%0.0f'))
plt.subplots_adjust(bottom=0.2)
imgFilename = MAINDIRECTORY+'/images/'+aTitle.replace(" ","_")+'.gif'
- #plt.savefig(imgFilename, transparent=True)
plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
return
#******************************************************************
def plotAccTRMM (finalMCCList):
'''
- Purpose:: (1) generate a file with the accumulated precipiation for the MCS
- (2) generate the appropriate image
- TODO: NB: as the domain changes, will need to change XDEF and YDEF by hand to accomodate the new domain
- TODO: look into getting the info from the NETCDF file
-
- Input:: finalMCCList: a list of strings representing the nodes that make a MCC (or MCS)
-
- Output:: a netcdf file containing the accumulated precip
- a gif (generated in Grads)
+ Purpose::
+ (1) generate a file with the accumulated precipiation for the MCS
+ (2) generate the appropriate image
+ TODO: NB: as the domain changes, will need to change XDEF and YDEF by hand to accomodate the new domain
+ TODO: look into getting the info from the NETCDF file
+
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
+
+ Output::
+ a netcdf file containing the accumulated precip
+ a gif (generated in Grads)
'''
os.chdir((MAINDIRECTORY+'/TRMMnetcdfCEs'))
fname =''
@@ -3023,7 +3001,6 @@ def plotAccTRMM (finalMCCList):
subprocess.call('rm acc.ctl', shell=True)
subprocess.call('touch acc.ctl', shell=True)
replaceExpDset = 'echo DSET ' + accuTRMMFile +' >> acc.ctl'
- # replaceExpTdef = 'echo TDEF 99999 LINEAR 00Z 00000000 30mn' +' >> acc.ctl'
subprocess.call(replaceExpDset, shell=True)
subprocess.call('echo "OPTIONS yrev little_endian template" >> acc.ctl', shell=True)
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
@@ -3039,13 +3016,10 @@ def plotAccTRMM (finalMCCList):
subprocess.call('echo "ENDVARS" >> acc.ctl', shell=True)
#generate GrADS script
- #print "thisNode ", thisNode
- #imgFilename = MAINDIRECTORY+'/images/MCSaccu'+firstPartName+str(thisNode['cloudElementTime']).replace(" ", "_")+'.gif'
subprocess.call('rm accuTRMM1.gs', shell=True)
subprocess.call('touch accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'reinit''\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'open acc.ctl ''\'" >> accuTRMM1.gs', shell=True)
- #subprocess.call('echo "''\'open '+accuTRMMFile+ '\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'set grads off''\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'set mpdset hires''\'" >> accuTRMM1.gs', shell=True)
subprocess.call('echo "''\'set gxout shaded''\'" >> accuTRMM1.gs', shell=True)
@@ -3066,13 +3040,16 @@ def plotAccTRMM (finalMCCList):
#******************************************************************
def plotAccuInTimeRange(starttime, endtime):
'''
- Purpose:: create accumulated precip plot within a time range given using all CEs
+ Purpose::
+ Create accumulated precip plot within a time range given using all CEs
- Input:: starttime: a string representing the time to start the accumulations format yyyy-mm-dd_hh:mm:ss
- endtime: a string representing the time to end the accumulations format yyyy-mm-dd_hh:mm:ss
+ Input::
+ starttime: a string representing the time to start the accumulations format yyyy-mm-dd_hh:mm:ss
+ endtime: a string representing the time to end the accumulations format yyyy-mm-dd_hh:mm:ss
- Output:: a netcdf file containing the accumulated precip for specified times
- a gif (generated in Grads)
+ Output::
+ a netcdf file containing the accumulated precip for specified times
+ a gif (generated in Grads)
TODO: pass of pick up from the NETCDF file the lat, lon and resolution for generating the ctl file
'''
@@ -3081,7 +3058,6 @@ def plotAccuInTimeRange(starttime, endtime):
#Just incase the X11 server is giving problems
subprocess.call('export DISPLAY=:0.0', shell=True)
- #fname =''
imgFilename = ''
firstPartName = ''
firstTime = True
@@ -3089,7 +3065,6 @@ def plotAccuInTimeRange(starttime, endtime):
fileList = []
sTime = datetime.strptime(starttime.replace("_"," "),'%Y-%m-%d %H:%M:%S')
eTime = datetime.strptime(endtime.replace("_"," "),'%Y-%m-%d %H:%M:%S')
- #tdelta = datetime.timedelta(0, 3600)
thisTime = sTime
while thisTime <= eTime:
@@ -3115,7 +3090,8 @@ def plotAccuInTimeRange(starttime, endtime):
thisTime +=timedelta(hours=TRES)
#create new netCDF file
- accuTRMMFile = MAINDIRECTORY+'TRMMnetcdfCEs/accu'+starttime+'-'+endtime+'.nc'
+ accuTRMMFile = MAINDIRECTORY+'/TRMMnetcdfCEs/accu'+starttime+'-'+endtime+'.nc'
+ print "accuTRMMFile ", accuTRMMFile
#write the file
accuTRMMData = Dataset(accuTRMMFile, 'w', format='NETCDF4')
accuTRMMData.description = 'Accumulated precipitation data'
@@ -3157,8 +3133,8 @@ def plotAccuInTimeRange(starttime, endtime):
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
subprocess.call('echo "UNDEF 0" >> acc.ctl', shell=True)
subprocess.call('echo "TITLE TRMM MCS accumulated precipitation" >> acc.ctl', shell=True)
- subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
- subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ subprocess.call('echo "XDEF 384 LINEAR -8.96875 0.036378335 " >> acc.ctl', shell=True)
+ subprocess.call('echo "YDEF 384 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
@@ -3190,22 +3166,22 @@ def plotAccuInTimeRange(starttime, endtime):
#******************************************************************
def createTextFile(finalMCCList, identifier):
'''
- Purpose:: Create a text file with information about the MCS
- This function is expected to be especially of use regarding
- long term record checks
+ Purpose::
+ Create a text file with information about the MCS
+ This function is expected to be especially of use regarding long term record checks
- Input:: finalMCCList: list of list of strings representing list of MCSs nodes list
- identifier: an integer representing the type of list that has been entered...this is for creating file purposes
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
+ identifier: an integer representing the type of list that has been entered...this is for creating file purposes
1 - MCCList; 2- MCSList
Output::
- a user readable text file with all information about each MCS
- a user readable text file with the summary of the MCS
+ a user readable text file with all information about each MCS
+ a user readable text file with the summary of the MCS
Assumptions::
'''
- #allTimes =[]
durations=0.0
startTimes =[]
endTimes=[]
@@ -3213,7 +3189,6 @@ def createTextFile(finalMCCList, identifier):
speedCounter = 0
maxArea =0.0
amax = 0.0
- #avgMaxArea = 0.0
avgMaxArea =[]
maxAreaCounter =0.0
maxAreaTime=''
@@ -3238,7 +3213,6 @@ def createTextFile(finalMCCList, identifier):
bigPtotal = 0.0
bigPtotalCounter = 0
allPropagationSpeeds =[]
- #averageAreas = 0.0
averageAreas =[]
areaAvg = 0.0
avgPrecipTotal = 0.0
@@ -3249,6 +3223,7 @@ def createTextFile(finalMCCList, identifier):
avgMinMCSPrecipRateCounter = 0
minMax =[]
avgPrecipArea = []
+ location =[]
avgPrecipAreaPercent = 0.0
precipArea = 0.0
precipAreaPercent = 0.0
@@ -3266,9 +3241,6 @@ def createTextFile(finalMCCList, identifier):
MCSUserFile = open((MAINDIRECTORY+'/textFiles/MCSsUserFile.txt'),'wb')
MCSSummaryFile = open((MAINDIRECTORY+'/textFiles/MCSSummary.txt'),'wb')
-
- #MCCsFile = open((MAINDIRECTORY+'/textFiles/MCCsFile.txt'),'wb')
-
for eachPath in finalMCCList:
eachPath.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0], nodeID.split('CE')[1]))
@@ -3284,16 +3256,13 @@ def createTextFile(finalMCCList, identifier):
endTimes.append(endTime)
#get the precip info
- #MCSPrecip = precipMaxMin(eachPath)
-
+
for eachNode in eachPath:
thisNode = thisDict(eachNode)
#set first time min "fake" values
if firstTime == True:
- # minCEprecipRate = MCSPrecip[0][1]
- # avgMinMCSPrecipRate = MCSPrecip[0][1]
minCEprecipRate = thisNode['CETRMMmin']
avgMinMCSPrecipRate += thisNode['CETRMMmin']
firstTime = False
@@ -3306,27 +3275,19 @@ def createTextFile(finalMCCList, identifier):
#Amax: find max area
if thisNode['cloudElementArea'] > maxArea:
maxArea = thisNode['cloudElementArea']
- # avgMaxArea += maxArea
- # maxAreaCounter += 1
maxAreaTime = str(thisNode['cloudElementTime'])
eccentricity = thisNode['cloudElementEccentricity']
-
+ location = thisNode['cloudElementCenter']
+
#determine the time the feature matures
if matureFlag == True:
timeMCSMatures = str(thisNode['cloudElementTime'])
matureFlag = False
- #find min and max precip rate
- # minMax = list(value_tuple for node,value_tuple in enumerate(MCSPrecip) if value_tuple[0] == eachNode)
- # if minMax[0][1] < minCEprecipRate:
- # minCEprecipRate = minMax[0][1]
- # print "minCEprecipRate ", minCEprecipRate
- # if minMa
<TRUNCATED>
[15/50] [abbrv] git commit: added a UI
Posted by le...@apache.org.
added a UI
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/63e7d5ad
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/63e7d5ad
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/63e7d5ad
Branch: refs/heads/master
Commit: 63e7d5ad4d5ca9911f187293e1747f6b02b5e628
Parents: 1d0b5a5
Author: kwhitehall <k_...@yahoo.com>
Authored: Wed Apr 30 19:34:05 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Wed Apr 30 19:34:05 2014 -0400
----------------------------------------------------------------------
mccsearch/README.docx | Bin 195234 -> 192118 bytes
mccsearch/code/mainProg.py | 6 +-
mccsearch/code/mainProgTemplate.py | 99 +++++++
mccsearch/code/mccSearch.py | 447 ++++++++++++++++++++++----------
mccsearch/code/mccSearchUI.py | 293 +++++++++++++++++++++
5 files changed, 712 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/63e7d5ad/mccsearch/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/README.docx b/mccsearch/README.docx
index a0a7d1e..648637e 100644
Binary files a/mccsearch/README.docx and b/mccsearch/README.docx differ
http://git-wip-us.apache.org/repos/asf/climate/blob/63e7d5ad/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 9be65b5..43ca921 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -27,9 +27,9 @@ def main():
mainDirStr= "/directory/to/where/to/store/outputs"
TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
CEoriDirName = "/directory/to/the/MERG/netCDF/files"
-
+
#for first time working with the raw MERG zipped files
- # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # mccSearch.preprocessingMERG("/directory/to/where/the/raw/MERG/files/are")
# ---------------------------------------------------------------------------------
@@ -88,7 +88,7 @@ def main():
#some plots that work
# mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
+ mccSearch.displayPrecip(MCCList)
# mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
# mccSearch.displaySize(MCCList)
# mccSearch.displayPrecip(MCCList)
http://git-wip-us.apache.org/repos/asf/climate/blob/63e7d5ad/mccsearch/code/mainProgTemplate.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemplate.py b/mccsearch/code/mainProgTemplate.py
new file mode 100644
index 0000000..9be65b5
--- /dev/null
+++ b/mccsearch/code/mainProgTemplate.py
@@ -0,0 +1,99 @@
+'''
+# running the program
+'''
+
+import sys
+import networkx as nx
+import mccSearch
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+import subprocess
+
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+
+ #for GrADs
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ mainDirStr= "/directory/to/where/to/store/outputs"
+ TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+
+ #for first time working with the raw MERG zipped files
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # ---------------------------------------------------------------------------------
+
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ # mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # -------------------------------------------------------------------------------------------------
+
+ #let's go!
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ #print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
+ # ----------------------------------------------------------------------------------------------
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ print "MCS List has been acquired ", len(MCSList)
+ print ("-"*80)
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+
+ #some calculations/metrics that work that work
+ # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistogram(MCCList)
+ #
+ print ("-"*80)
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/63e7d5ad/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index 7c647d5..8c2c801 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -42,8 +42,8 @@ import process
#the first point closest the the value (for the min) from the MERG data is used, etc.
LATMIN = '5.0' #min latitude; -ve values in the SH e.g. 5S = -5
LATMAX = '19.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
-LONMIN = '-6.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
-LONMAX = '3.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+LONMIN = '-5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '9.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
XRES = 4.0 #x direction spatial resolution in km
YRES = 4.0 #y direction spatial resolution in km
TRES = 1 #temporal resolution in hrs
@@ -545,7 +545,6 @@ def findCloudElements(mergImgs,timelist,TRMMdirName=None):
nodeExist = False
cloudElementCenter=[]
cloudElement = []
- cloudElementCenter = []
cloudElementLat=[]
cloudElementLon =[]
cloudElementLatLons =[]
@@ -986,7 +985,7 @@ def findMCC (prunedGraph):
return definiteMCC, definiteMCS
#******************************************************************
-def traverseTree(subGraph,node, queue, checkedNodes=None):
+def traverseTree(subGraph,node, stack, checkedNodes=None):
'''
Purpose::
To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
@@ -995,8 +994,8 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
subGraph: a Networkx DiGraph representing a CC
lengthOfsubGraph: an integer representing the length of the subgraph
node: a string representing the node currently being checked
- queue: a list of strings representing a list of nodes in a stack functionality
- i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
+ stack: a list of strings representing a list of nodes in a stack functionality
+ i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node
checkedNodes: a list of strings representing the list of the nodes in the traversal
Output::
@@ -1010,31 +1009,31 @@ def traverseTree(subGraph,node, queue, checkedNodes=None):
return checkedNodes
if not checkedNodes:
- queue =[]
+ stack =[]
checkedNodes.append(node)
#check one level infront first...if something does exisit, stick it at the front of the stack
upOneLevel = subGraph.predecessors(node)
downOneLevel = subGraph.successors(node)
for parent in upOneLevel:
- if parent not in checkedNodes and parent not in queue:
+ if parent not in checkedNodes and parent not in stack:
for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
- queue.insert(0,child)
+ if child not in checkedNodes and child not in stack:
+ stack.insert(0,child)
- queue.insert(0,parent)
+ stack.insert(0,parent)
for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
+ if child not in checkedNodes and child not in stack:
if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
- queue.insert(0,child)
+ stack.insert(0,child)
else:
- queue.append(child)
+ stack.append(child)
- for eachNode in queue:
+ for eachNode in stack:
if eachNode not in checkedNodes:
checkedNodes.append(eachNode)
- return traverseTree(subGraph, eachNode, queue, checkedNodes)
+ return traverseTree(subGraph, eachNode, stack, checkedNodes)
return checkedNodes
#******************************************************************
@@ -1788,6 +1787,7 @@ def findCESpeed(node, MCSList):
theList = CLOUD_ELEMENT_GRAPH.successors(node)
nodeLatLon=thisDict(node)['cloudElementCenter']
+
for aNode in theList:
if aNode in MCSList:
@@ -1795,15 +1795,25 @@ def findCESpeed(node, MCSList):
aNodeLatLon = thisDict(aNode)['cloudElementCenter']
#calculate CE speed
#checking the lats
- nodeLatLon[0] += 90.0
- aNodeLatLon[0] += 90.0
- delta_lat = (nodeLatLon[0] - aNodeLatLon[0])
- nodeLatLon[1] += 360.0
- aNodeLatLon[1] += 360.0
- delta_lon = (nodeLatLon[1] - aNodeLatLon[1])
- theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
+ # nodeLatLon[0] += 90.0
+ # aNodeLatLon[0] += 90.0
+ # delta_lat = (nodeLatLon[0] - aNodeLatLon[0])
+ delta_lat = ((thisDict(node)['cloudElementCenter'][0] +90.0) - (thisDict(aNode)['cloudElementCenter'][0]+90.0))
+ # nodeLatLon[1] += 360.0
+ # aNodeLatLon[1] += 360.0
+ # delta_lon = (nodeLatLon[1] - aNodeLatLon[1])
+ delta_lon = ((thisDict(node)['cloudElementCenter'][1]+360.0) - (thisDict(aNode)['cloudElementCenter'][1]+360.0))
+
+ try:
+ theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
+ except:
+ theSpeed = 0.0
CEspeed.append(theSpeed)
+
+ # print "~~~ ", thisDict(aNode)['uniqueID']
+ # print "*** ", nodeLatLon, thisDict(node)['cloudElementCenter']
+ # print "*** ", aNodeLatLon, thisDict(aNode)['cloudElementCenter']
if not CEspeed:
return 0.0
@@ -1866,6 +1876,155 @@ def createMainDirectory(mainDirStr):
return
#******************************************************************
+def checkForFiles(startTime, endTime, thisDir, fileType):
+ '''
+ Purpose:: To ensure all the files between the starttime and endTime
+ exist in the directory supplied
+
+ Input::
+ startTime: a string yyyymmmddhh representing the starttime
+ endTime: a string yyyymmmddhh representing the endTime
+ thisDir: a string representing the directory path where to
+ look for the file
+ fileType: an integer representing the type of file in the directory
+ 1 - MERG original files, 2 - TRMM original files
+
+ Output::
+ status: a boolean representing whether all files exists
+
+ '''
+ startFilename = ''
+ endFilename =''
+ currFilename = ''
+ status = False
+ startyr = int(startTime[:4])
+ startmm = int(startTime[4:6])
+ startdd = int(startTime[6:8])
+ starthr = int(startTime[-2:])
+ endyr = int(endTime[:4])
+ endmm = int(endTime[4:6])
+ enddd = int(endTime[6:8])
+ endhh = int(endTime[-2:])
+ curryr = startyr
+ currmm = startmm
+ currdd = startdd
+ currhr = starthr
+ currmmStr = ''
+ currddStr = ''
+ currhrStr = ''
+ endmmStr = ''
+ endddStr =''
+ endhhStr = ''
+
+ #check that the startTime is before the endTime
+ if fileType == 1:
+ #print "fileType is 1"
+ startFilename = "merg_"+startTime+"_4km-pixel.nc"
+ endFilename = thisDir+"/merg_"+endTime+"_4km-pixel.nc"
+
+ if fileType == 2:
+ #TODO:: determine closest time for TRMM files for end
+ #http://disc.sci.gsfc.nasa.gov/additional/faq/precipitation_faq.shtml#convert
+ #How do I extract time information from the TRMM 3B42 file name? section
+ # startFilename = "3B42."+startTime[:8]+"."+currhr+".7A.nc"
+ # endFilename = "3B42."+endTime[:8]+"."+endTime[-2:]+".7A.nc"
+ if starthr%3 == 2:
+ currhr += 1
+ elif starthr%3 ==1:
+ currhr -= 1
+ else:
+ currhr = starthr
+
+ curryr, currmmStr, currddStr, currhrStr,_,_,_ = findTime(curryr, currmm, currdd, currhr)
+
+ startFilename = "3B42."+str(curryr)+currmmStr+currddStr+"."+currhrStr+".7A.nc"
+ if endhh%3 == 2:
+ endhh += 1
+ elif endhh%3 ==1:
+ endhh -= 1
+
+ endyr, endmmStr, endddStr, endhhStr, _, _, _ = findTime(endyr, endmm, enddd, endhh)
+
+ endFilename = thisDir+"/3B42."+str(endyr)+endmmStr+endddStr+"."+endhhStr+".7A.nc"
+
+ #check for files between startTime and endTime
+ currFilename = thisDir+"/"+startFilename
+
+ while currFilename is not endFilename:
+ if not os.path.isfile(currFilename):
+ print "file is missing! Filename: ", currFilename
+ status = False
+ return status
+
+ status = True
+ if currFilename == endFilename:
+ break
+
+ #generate new currFilename
+ if fileType == 1:
+ currhr +=1
+ elif fileType ==2:
+ currhr += 3
+
+ curryr, currmmStr, currddStr, currhrStr, currmm, currdd, currhr = findTime(curryr, currmm, currdd, currhr)
+
+ if fileType == 1:
+ currFilename = thisDir+"/"+"merg_"+str(curryr)+currmmStr+currddStr+currhrStr+"_4km-pixel.nc"
+ if fileType == 2:
+ currFilename = thisDir+"/"+"3B42."+str(curryr)+currmmStr+currddStr+"."+currhrStr+".7A.nc"
+
+ return status
+#******************************************************************
+def findTime(curryr, currmm, currdd, currhr):
+ '''
+ Purpose:: To determine the new yr, mm, dd, hr
+
+ Input:: curryr, an integer representing the year
+ currmm, an integer representing the month
+ currdd, an integer representing the day
+ currhr, an integer representing the hour
+
+ Output::curryr, an integer representing the year
+ currmm, an integer representing the month
+ currdd, an integer representing the day
+ currhr, an integer representing the hour
+ '''
+ if currhr > 23:
+ currhr = 0
+ currdd += 1
+ if currdd > 30 and (currmm == 4 or currmm == 6 or currmm == 9 or currmm == 11):
+ currmm +=1
+ elif currdd > 31 and (currmm == 1 or currmm ==3 or currmm == 5 or currmm == 7 or currmm == 8 or currmm == 10):
+ currmm +=1
+ currdd = 1
+ elif currdd > 31 and currmm == 12:
+ currmm = 1
+ currdd = 1
+ curryr += 1
+ elif currdd > 28 and currmm == 2 and (curryr%4)!=0:
+ currmm = 3
+ currdd = 1
+ elif (curryr%4)==0 and currmm == 2 and currdd>29:
+ currmm = 3
+ currdd = 1
+
+ if currmm < 10:
+ currmmStr="0"+str(currmm)
+ else:
+ currmmStr = str(currmm)
+
+ if currdd < 10:
+ currddStr = "0"+str(currdd)
+ else:
+ currddStr = str(currdd)
+
+ if currhr < 10:
+ currhrStr = "0"+str(currhr)
+ else:
+ currhrStr = str(currhr)
+
+ return curryr, currmmStr, currddStr, currhrStr, currmm, currdd, currhr
+#******************************************************************
def find_nearest(thisArray,value):
'''
Purpose :: to determine the value within an array closes to
@@ -2007,8 +2166,7 @@ def postProcessingNetCDF(dataset, dirName = None):
2 User can write files in location where script is being called
'''
- coreDir = os.path.dirname(MAINDIRECTORY)
- #coreDir = os.path.dirname(os.path.abspath(__file__))
+ coreDir = os.path.dirname(os.path.abspath(__file__))
ImgFilename = ''
frameList=[]
fileList =[]
@@ -2024,35 +2182,29 @@ def postProcessingNetCDF(dataset, dirName = None):
if dataset == 1:
var = 'ch4'
- dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- gsFile = coreDir+"/GrADSscripts/cs2.gs"
- sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
+ origsFile = coreDir+"/../GrADSscripts/cs1.gs"
+ gsFile = coreDir+"/../GrADSscripts/cs2.gs"
+ sologsFile = coreDir+"/../GrADSscripts/mergeCE.gs"
lineNum = 50
elif dataset ==2:
var = 'precipAcc'
- dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
- origsFile = coreDir+"/GrADSscripts/cs3.gs"
- gsFile = coreDir+"/GrADSscripts/cs4.gs"
- sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
+ origsFile = coreDir+"/../GrADSscripts/cs3.gs"
+ gsFile = coreDir+"/../GrADSscripts/cs4.gs"
+ sologsFile = coreDir+"/../GrADSscripts/TRMMCE.gs"
lineNum = 10
elif dataset ==3:
var = 'ch4'
ctlTitle = 'TITLE MERG DATA'
ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
- if dirName is None:
- print "Enter directory for original files"
- return
- else:
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- sologsFile = coreDir+"/GrADSscripts/infrared.gs"
- lineNum = 54
+ origsFile = coreDir+"/../GrADSscripts/cs1.gs"
+ sologsFile = coreDir+"/../GrADSscripts/infrared.gs"
+ lineNum = 54
#sort files
os.chdir((dirName+'/'))
@@ -2066,8 +2218,13 @@ def postProcessingNetCDF(dataset, dirName = None):
for eachfile in files:
fullFname = os.path.splitext(eachfile)[0]
fnameNoExtension = fullFname.split('.nc')[0]
+
+ if dataset == 2 and fnameNoExtension[:4] != "TRMM":
+ continue
+
if dataset == 1 or dataset == 2:
frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
+
#create the ctlFile
ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
#the ctl file
@@ -2083,9 +2240,20 @@ def postProcessingNetCDF(dataset, dirName = None):
subprocess.call(lineToWrite, shell=True)
lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo XDEF 413 LINEAR -9.984375 0.036378335 >> '+ctlFile1
+ fname = dirName+'/'+fnameNoExtension+'.nc'
+ if os.path.isfile(fname):
+ #open NetCDF file add info to the accu
+ print "opening file ", fname
+ fileData = Dataset(fname,'r',format='NETCDF4')
+ lats = fileData.variables['latitude'][:]
+ lons = fileData.variables['longitude'][:]
+ LONDATA, LATDATA = np.meshgrid(lons,lats)
+ nygrd = len(LATDATA[:,0])
+ nxgrd = len(LONDATA[0,:])
+ fileData.close()
+ lineToWrite = 'echo XDEF '+ str(nxgrd) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrd) +' >> ' +ctlFile1
subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo YDEF 412 LINEAR 5.03515625 0.036378335 >> '+ctlFile1
+ lineToWrite = 'echo YDEF '+ str(nygrd)+' LINEAR ' + str(min(lats)) + ' ' + str((max(lats)-min(lats))/nygrd) +' >> '+ctlFile1
subprocess.call(lineToWrite, shell=True)
lineToWrite = 'echo ZDEF 01 LEVELS 1 >> '+ctlFile1
subprocess.call(lineToWrite, shell=True)
@@ -2171,10 +2339,6 @@ def postProcessingNetCDF(dataset, dirName = None):
GrADSscript.writelines(lines1)
GrADSscript.close()
- # if frameNum == 44:
- # sys.exit()
-
-
#run the script
runGrads = 'run '+ gsFile
gradscmd = 'grads -blc ' + '\'' +runGrads + '\''+'\n'
@@ -2492,85 +2656,6 @@ def commonFeatureSize(finalMCCList):
hist, bin_edges = np.histogram(thisMCCAvg)
return hist,bin_edges
#******************************************************************
-def displaySize(finalMCCList):
- '''
- Purpose::
- To create a figure showing the area verse time for each MCS
-
- Input::
- finalMCCList: a list of list of strings representing the list of nodes representing a MCC
-
- Output::
- None
-
- '''
- timeList =[]
- count=1
- imgFilename=''
- minArea=10000.0
- maxArea=0.0
- eachNode={}
-
- #for each node in the list, get the area information from the dictionary
- #in the graph and calculate the area
-
- if finalMCCList:
- for eachMCC in finalMCCList:
- #get the info from the node
- for node in eachMCC:
- eachNode=thisDict(node)
- timeList.append(eachNode['cloudElementTime'])
-
- if eachNode['cloudElementArea'] < minArea:
- minArea = eachNode['cloudElementArea']
- if eachNode['cloudElementArea'] > maxArea:
- maxArea = eachNode['cloudElementArea']
-
-
- #sort and remove duplicates
- timeList=list(set(timeList))
- timeList.sort()
- tdelta = timeList[1] - timeList[0]
- starttime = timeList[0]-tdelta
- endtime = timeList[-1]+tdelta
- timeList.insert(0, starttime)
- timeList.append(endtime)
-
- #plot info
- plt.close('all')
- title = 'Area distribution of the MCC over somewhere'
- fig=plt.figure(facecolor='white', figsize=(18,10)) #figsize=(10,8))#figsize=(16,12))
- fig,ax = plt.subplots(1, facecolor='white', figsize=(10,10))
-
- #the data
- for node in eachMCC: #for eachNode in eachMCC:
- eachNode=thisDict(node)
- if eachNode['cloudElementArea'] < 80000 : #2400.00:
- ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'bo', markersize=10)
- elif eachNode['cloudElementArea'] >= 80000.00 and eachNode['cloudElementArea'] < 160000.00:
- ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'yo',markersize=20)
- else:
- ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'ro',markersize=30)
-
- #axes and labels
- maxArea += 1000.00
- ax.set_xlim(starttime,endtime)
- ax.set_ylim(minArea,maxArea)
- ax.set_ylabel('Area in km^2', fontsize=12)
- ax.set_title(title)
- ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d%H:%M:%S')
- fig.autofmt_xdate()
-
- plt.subplots_adjust(bottom=0.2)
-
- imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif'
- plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
-
- #if time in not already in the time list, append it
- timeList=[]
- count += 1
- return
-#******************************************************************
def precipTotals(finalMCCList):
'''
Purpose::
@@ -2671,6 +2756,89 @@ def precipMaxMin(finalMCCList):
return MCSPrecip
#******************************************************************
+#
+# PLOTS
+#
+#******************************************************************
+def displaySize(finalMCCList):
+ '''
+ Purpose::
+ To create a figure showing the area verse time for each MCS
+
+ Input::
+ finalMCCList: a list of list of strings representing the list of nodes representing a MCC
+
+ Output::
+ None
+
+ '''
+ timeList =[]
+ count=1
+ imgFilename=''
+ minArea=10000.0
+ maxArea=0.0
+ eachNode={}
+
+ #for each node in the list, get the area information from the dictionary
+ #in the graph and calculate the area
+
+ if finalMCCList:
+ for eachMCC in finalMCCList:
+ #get the info from the node
+ for node in eachMCC:
+ eachNode=thisDict(node)
+ timeList.append(eachNode['cloudElementTime'])
+
+ if eachNode['cloudElementArea'] < minArea:
+ minArea = eachNode['cloudElementArea']
+ if eachNode['cloudElementArea'] > maxArea:
+ maxArea = eachNode['cloudElementArea']
+
+
+ #sort and remove duplicates
+ timeList=list(set(timeList))
+ timeList.sort()
+ tdelta = timeList[1] - timeList[0]
+ starttime = timeList[0]-tdelta
+ endtime = timeList[-1]+tdelta
+ timeList.insert(0, starttime)
+ timeList.append(endtime)
+
+ #plot info
+ plt.close('all')
+ title = 'Area distribution of the MCC over somewhere'
+ fig=plt.figure(facecolor='white', figsize=(18,10)) #figsize=(10,8))#figsize=(16,12))
+ fig,ax = plt.subplots(1, facecolor='white', figsize=(10,10))
+
+ #the data
+ for node in eachMCC: #for eachNode in eachMCC:
+ eachNode=thisDict(node)
+ if eachNode['cloudElementArea'] < 80000 : #2400.00:
+ ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'bo', markersize=10)
+ elif eachNode['cloudElementArea'] >= 80000.00 and eachNode['cloudElementArea'] < 160000.00:
+ ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'yo',markersize=20)
+ else:
+ ax.plot(eachNode['cloudElementTime'], eachNode['cloudElementArea'],'ro',markersize=30)
+
+ #axes and labels
+ maxArea += 1000.00
+ ax.set_xlim(starttime,endtime)
+ ax.set_ylim(minArea,maxArea)
+ ax.set_ylabel('Area in km^2', fontsize=12)
+ ax.set_title(title)
+ ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d%H:%M:%S')
+ fig.autofmt_xdate()
+
+ plt.subplots_adjust(bottom=0.2)
+
+ imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif'
+ plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
+
+ #if time in not already in the time list, append it
+ timeList=[]
+ count += 1
+ return
+#******************************************************************
def displayPrecip(finalMCCList):
'''
Purpose::
@@ -2720,6 +2888,7 @@ def displayPrecip(finalMCCList):
percentagePrecipitating.append((eachNode['TRMMArea']/eachNode['cloudElementArea'])*100.0)
CEArea.append(eachNode['cloudElementArea'])
nodes.append(eachNode['uniqueID'])
+ # print eachNode['uniqueID'], eachNode['cloudElementCenter'][1], eachNode['cloudElementCenter'][0]
x.append(eachNode['cloudElementCenter'][1])#-xStart)
y.append(eachNode['cloudElementCenter'][0])#-yStart)
@@ -2731,6 +2900,9 @@ def displayPrecip(finalMCCList):
totalSize = sum(CEArea)
partialArea = [(a/totalSize)*30000 for a in CEArea]
+
+ # print "x ", x
+ # print "y ", y
#plot info
plt.close('all')
@@ -2931,6 +3103,7 @@ def plotAccTRMM (finalMCCList):
imgFilename = ''
firstPartName = ''
firstTime = True
+ replaceExpXDef = ''
#Just incase the X11 server is giving problems
subprocess.call('export DISPLAY=:0.0', shell=True)
@@ -2944,7 +3117,7 @@ def plotAccTRMM (finalMCCList):
if os.path.isfile(fname):
#open NetCDF file add info to the accu
- print "opening TRMM file ", fname
+ #print "opening TRMM file ", fname
TRMMCEData = Dataset(fname,'r',format='NETCDF4')
precipRate = TRMMCEData.variables['precipitation_Accumulation'][:]
lats = TRMMCEData.variables['latitude'][:]
@@ -2997,6 +3170,7 @@ def plotAccTRMM (finalMCCList):
accuTRMMData.close()
#generate the image with GrADS
+ #print "ny,nx ", nygrdTRMM, nxgrdTRMM, min(lats), max(lats)
#the ctl file
subprocess.call('rm acc.ctl', shell=True)
subprocess.call('touch acc.ctl', shell=True)
@@ -3006,8 +3180,12 @@ def plotAccTRMM (finalMCCList):
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
subprocess.call('echo "UNDEF 0" >> acc.ctl', shell=True)
subprocess.call('echo "TITLE TRMM MCS accumulated precipitation" >> acc.ctl', shell=True)
- subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
- subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpXDef = 'echo XDEF '+ str(nxgrdTRMM) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrdTRMM) +' >> acc.ctl'
+ subprocess.call(replaceExpXDef, shell=True)
+ #subprocess.call('echo "XDEF 413 LINEAR -9.984375 0.036378335 " >> acc.ctl', shell=True)
+ #subprocess.call('echo "YDEF 412 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
+ subprocess.call(replaceExpYDef, shell=True)
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
#subprocess.call(replaceExpTdef, shell=True)
@@ -3031,6 +3209,7 @@ def plotAccTRMM (finalMCCList):
subprocess.call('echo "''\'quit''\'" >> accuTRMM1.gs', shell=True)
gradscmd = 'grads -blc ' + '\'run accuTRMM1.gs''\''
subprocess.call(gradscmd, shell=True)
+ sys.exit()
#clean up
subprocess.call('rm accuTRMM1.gs', shell=True)
@@ -3133,8 +3312,12 @@ def plotAccuInTimeRange(starttime, endtime):
subprocess.call('echo "DTYPE netcdf" >> acc.ctl', shell=True)
subprocess.call('echo "UNDEF 0" >> acc.ctl', shell=True)
subprocess.call('echo "TITLE TRMM MCS accumulated precipitation" >> acc.ctl', shell=True)
- subprocess.call('echo "XDEF 384 LINEAR -8.96875 0.036378335 " >> acc.ctl', shell=True)
- subprocess.call('echo "YDEF 384 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
+ replaceExpXDef = 'echo XDEF '+ str(nxgrdTRMM) + ' LINEAR ' + str(min(lons)) +' '+ str((max(lons)-min(lons))/nxgrdTRMM) +' >> acc.ctl'
+ subprocess.call(replaceExpXDef, shell=True)
+ replaceExpYDef = 'echo YDEF '+str(nygrdTRMM)+' LINEAR '+str(min(lats))+ ' '+str((max(lats)-min(lats))/nygrdTRMM)+' >>acc.ctl'
+ subprocess.call(replaceExpYDef, shell=True)
+ #subprocess.call('echo "XDEF 384 LINEAR -8.96875 0.036378335 " >> acc.ctl', shell=True)
+ #subprocess.call('echo "YDEF 384 LINEAR 5.03515625 0.036378335 " >> acc.ctl', shell=True)
subprocess.call('echo "ZDEF 01 LEVELS 1" >> acc.ctl', shell=True)
subprocess.call('echo "TDEF 99999 linear 31aug2009 1hr" >> acc.ctl', shell=True)
subprocess.call('echo "VARS 1" >> acc.ctl', shell=True)
@@ -3236,13 +3419,16 @@ def createTextFile(finalMCCList, identifier):
if identifier == 1:
MCSUserFile = open((MAINDIRECTORY+'/textFiles/MCCsUserFile.txt'),'wb')
MCSSummaryFile = open((MAINDIRECTORY+'/textFiles/MCCSummary.txt'),'wb')
+ MCSPostFile = open((MAINDIRECTORY+'/textFiles/MCCPostPrecessing.txt'),'wb')
if identifier == 2:
MCSUserFile = open((MAINDIRECTORY+'/textFiles/MCSsUserFile.txt'),'wb')
MCSSummaryFile = open((MAINDIRECTORY+'/textFiles/MCSSummary.txt'),'wb')
+ MCSPostFile = open((MAINDIRECTORY+'/textFiles/MCSPostPrecessing.txt'),'wb')
for eachPath in finalMCCList:
eachPath.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0], nodeID.split('CE')[1]))
+ MCSPostFile.write("\n %s" %eachPath)
startTime = thisDict(eachPath[0])['cloudElementTime']
endTime = thisDict(eachPath[-1])['cloudElementTime']
@@ -3494,6 +3680,7 @@ def createTextFile(finalMCCList, identifier):
MCSUserFile.close
MCSSummaryFile.close
+ MCSPostFile.close
return
#******************************************************************
# PLOTTING UTIL SCRIPTS
http://git-wip-us.apache.org/repos/asf/climate/blob/63e7d5ad/mccsearch/code/mccSearchUI.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchUI.py b/mccsearch/code/mccSearchUI.py
new file mode 100644
index 0000000..9dbb727
--- /dev/null
+++ b/mccsearch/code/mccSearchUI.py
@@ -0,0 +1,293 @@
+'''
+# Wizard for running the mccSearch program
+'''
+
+import sys
+import networkx as nx
+import numpy as np
+import numpy.ma as ma
+import os
+import matplotlib.pyplot as plt
+import subprocess
+
+#mccSearch modules
+import mccSearch
+import files
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ DIRS={}
+ # DIRS={
+ # mainDirStr= "/directory/to/where/to/store/outputs"
+ # TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ # CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+ # }
+ preprocessing = ''
+ rawMERG = ''
+
+ #for GrADs
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+
+ print "Running MCCSearch ..... \n"
+ DIRS['mainDirStr'] = raw_input("> Please enter working directory: \n" ) # This is where data created will be stored
+
+ preprocessing = raw_input ("> Do you need to preprocess the MERG files? [y/n]: \n")
+ while preprocessing.lower() != 'n':
+ if preprocessing.lower() == 'y':
+ #get location for raw files
+ rawMERG = raw_input("> Please enter the directory to the RAW MERG (.Z) files: \n")
+ #run preprocessing
+ mccSearch.preprocessingMERG(rawMERG)
+ continue
+ elif preprocessing.lower() == 'n' :
+ pass
+ else:
+ print "Error! Invalid choice "
+ preprocessing = raw_input ("> Do you need to preprocess the MERG files? [y/n]: \n")
+
+
+ #get the location of the MERG and TRMM data
+ DIRS['CEoriDirName'] = raw_input("> Please enter the directory to the MERG netCDF files: \n")
+
+ try:
+ if not os.path.exists(DIRS['CEoriDirName']):
+ print "Error! MERG invalid path!"
+ DIRS['CEoriDirName'] = raw_input("> Please enter the directory to the MERG netCDF files: \n")
+ except:
+ print "..."
+
+
+ DIRS['TRMMdirName'] = raw_input("> Please enter the location to the raw TRMM netCDF files: \n")
+ try:
+ if not os.path.exists(DIRS['TRMMdirName']):
+ print "Error: TRMM invalid path!"
+ DIRS['TRMMdirName'] = raw_input("> Please enter the location to the raw TRMM netCDF files: \n")
+ except:
+ pass
+
+ #get the dates for analysis
+ startDateTime = raw_input("> Please enter the start date and time yyyymmddhr: \n")
+ #check validity of time
+ while validDate(startDateTime) == 0:
+ print "Invalid time entered for startDateTime!"
+ startDateTime = raw_input("> Please enter the start date and time yyyymmddhr: \n")
+
+ endDateTime = raw_input("> Please enter the end date and time yyyymmddhr: \n")
+ while validDate(endDateTime) == 0:
+ print "Invalid time entered for endDateTime!"
+ endDateTime = raw_input("> Please enter the end date and time yyyymmddhr: \n")
+
+ #check if all the files exisits in the MERG and TRMM directories entered
+ test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['TRMMdirName'], 2)
+ if test == False:
+ print "Error with files in the original MERG directory entered. Please check your files before restarting. "
+ return
+ test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['CEoriDirName'],1)
+ if test == False:
+ print "Error with files in the original TRMM directory entered. Please check your files before restarting. "
+ return
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(DIRS['mainDirStr'])
+ TRMMCEdirName = DIRS['mainDirStr']+'/TRMMnetcdfCEs'
+ CEdirName = DIRS['mainDirStr']+'/MERGnetcdfCEs'
+
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ postprocessing = raw_input("> Do you wish to postprocess data? [y/n] \n")
+ while postprocessing.lower() != 'n':
+ if postprocessing.lower() == 'y':
+ option = postProcessingplotMenu(DIRS)
+ return
+ elif postprocessing.lower() == 'n':
+ pass
+ else:
+ print "\n Invalid option."
+ postprocessing = raw_input("> Do you wish to postprocess data? [y/n] \n")
+ # -------------------------------------------------------------------------------------------------
+ # Getting started. Make it so number one!
+ print ("-"*80)
+ print "\t\t Starting the MCCSearch Analysis "
+ print ("-"*80)
+ print "\n -------------- Reading MERG and TRMM Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(DIRS['CEoriDirName'])
+ print "\n -------------- findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,DIRS['TRMMdirName'])
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(DIRS['TRMMdirName'],timeList)
+ # ----------------------------------------------------------------------------------------------
+ print "\n -------------- findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print "\n -------------- findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ #now ready to perform various calculations/metrics
+ print ("-"*80)
+ print "\n -------------- METRICS ----------"
+ print ("-"*80)
+ #some calculations/metrics that work that work
+ print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ plotMenu(MCCList, MCSList)
+
+ #Let's get outta here! Engage!
+ print ("-"*80)
+#*********************************************************************************************************************
+def plotMenu(MCCList, MCSList):
+ '''
+ Purpose:: The flow of plots for the user to choose
+
+ Input:: MCCList: a list of directories representing a list of nodes in the MCC
+ MCSList: a list of directories representing a list of nodes in the MCS
+
+ Output:: None
+ '''
+ option = displayPlotMenu()
+ while option != 0:
+ try:
+ if option == 1:
+ print "Generating Accumulated Rainfall from TRMM for the entire period ...\n"
+ mccSearch.plotAccTRMM(MCCList)
+ if option == 2:
+ startDateTime = raw_input("> Please enter the start date and time yyyy-mm-dd_hr:mm:ss format: \n")
+ endDateTime = raw_input("> Please enter the end date and time yyyy-mm-dd_hr:mm:ss format: \n")
+ print "Generating acccumulated rainfall between ", startDateTime," and ", endDateTime, " ... \n"
+ mccSearch.plotAccuInTimeRange(startDateTime, endDateTime)
+ if option == 3:
+ print "Generating area distribution plot ... \n"
+ mccSearch.displaySize(MCCList)
+ if option == 4:
+ print "Generating precipitation and area distribution plot ... \n"
+ mccSearch.displayPrecip(MCCList)
+ if option == 5:
+ try:
+ print "Generating histogram of precipitation for each time ... \n"
+ mccSearch.plotPrecipHistograms(MCCList)
+ except:
+ pass
+ except:
+ print "Invalid option. Please try again, enter 0 to exit \n"
+
+ option = displayPlotMenu()
+ return
+#*********************************************************************************************************************
+def displayPlotMenu():
+ '''
+ Purpose:: Display the plot Menu Options
+
+ Input:: None
+
+ Output:: option: an integer representing the choice of the user
+ '''
+ print "**************** PLOTS ************** \n"
+ print "0. Exit \n"
+ print "1. Accumulated TRMM precipitation \n"
+ print "2. Accumulated TRMM precipitation between dates \n"
+ print "3. Area distribution of the system over time \n"
+ print "4. Precipitation and area distribution of the system \n"
+ print "5. Histogram distribution of the rainfall in the area \n"
+ option = int(raw_input("> Please enter your option for plots: \n"))
+ return option
+#*********************************************************************************************************************
+def displayPostprocessingPlotMenu():
+ '''
+ Purpose:: Display the plot Menu Options
+
+ Input:: None
+
+ Output:: option: an integer representing the choice of the user
+ '''
+ print "**************** POST PROCESSING PLOTS ************** \n"
+ print "0. Exit \n"
+ print "1. Map plots of the original MERG data \n"
+ print "2. Map plots of the cloud elements using IR data \n"
+ print "3. Map plots of the cloud elements rainfall accumulations using TRMM data \n"
+ #print "4. Accumulated TRMM precipitation \n"
+ #print "5. Accumulated TRMM precipitation between dates \n"
+
+ option = int(raw_input("> Please enter your option for plots: \n"))
+ return option
+#*********************************************************************************************************************
+def postProcessingplotMenu(DIRS):
+ '''
+ Purpose:: The flow of plots for the user to choose
+
+ Input:: DIRS a dictionary of directories
+ # DIRS={
+ # mainDirStr= "/directory/to/where/to/store/outputs"
+ # TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ # CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+ # }
+
+ Output:: None
+ '''
+ TRMMCEdirName = DIRS['mainDirStr']+'/TRMMnetcdfCEs'
+ CEdirName = DIRS['mainDirStr']+'/MERGnetcdfCEs'
+
+ option = displayPostprocessingPlotMenu()
+ while option != 0:
+ try:
+ if option == 1:
+ print "Generating images from the original MERG dataset ... \n"
+ mccSearch.postProcessingNetCDF(3, DIRS['CEoriDirName'])
+ if option == 2:
+ print "Generating images from the cloud elements using MERG IR data ... \n"
+ mccSearch.postProcessingNetCDF(1, CEdirName)
+ if option == 3:
+ print "Generating precipitation accumulation images from the cloud elements using TRMM data ... \n"
+ mccSearch.postProcessingNetCDF(2, TRMMCEdirName)
+ # if option == 4:
+ # print "Generating Accumulated TRMM rainfall from cloud elements for each MCS ... \n"
+ # featureType = int(raw_input("> Please enter type of MCS MCC-1 or MCS-2: \n"))
+ # if featureType == 1:
+ # filename = DIRS['mainDirStr']+'/textFiles/MCCPostProcessing.txt'
+ # try:
+ # if os.path.isfile(filename):
+ # #read each line as a list
+ # mccSearch.plotAccTRMM()
+ # if option == 5:
+ # mccSearch.plotAccuInTimeRange()
+ except:
+ print "Invalid option, please try again"
+ option = displayPostprocessingPlotMenu()
+ return
+#*********************************************************************************************************************
+def validDate(dataString):
+ '''
+ '''
+
+ if len(dataString) > 10:
+ print "invalid time entered"
+ return 0
+
+ yr = int(dataString[:4])
+ mm = int(dataString[4:6])
+ dd = int(dataString[6:8])
+ hh = int(dataString[-2:])
+
+ if mm < 1 or mm > 12:
+ return 0
+ elif hh < 0 or hh > 23:
+ return 0
+ elif (dd< 0 or dd > 30) and (mm == 4 or mm == 6 or mm == 9 or mm == 11):
+ return 0
+ elif (dd< 0 or dd > 31) and (mm == 1 or mm ==3 or mm == 5 or mm == 7 or mm == 8 or mm == 10):
+ return 0
+ elif dd > 28 and mm == 2 and (yr%4)!=0:
+ return 0
+ elif (yr%4)==0 and mm == 2 and dd>29:
+ return 0
+ elif dd > 31 and mm == 12:
+ return 0
+ else:
+ return 1
+#*********************************************************************************************************************
+
+main()
+
[12/50] [abbrv] remove unnecessary files
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/newPreprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/newPreprocessingMERG.py b/mccsearch/code/newPreprocessingMERG.py
deleted file mode 100755
index cedc155..0000000
--- a/mccsearch/code/newPreprocessingMERG.py
+++ /dev/null
@@ -1,175 +0,0 @@
-
-'''
- The scripts for processing MERG raw data
- #******************************************************************
- '''
-
-import fileinput
-import glob
-import os
-import re
-import string
-import subprocess
-import sys
-
-#----------------------- GLOBAL VARIABLES --------------------------
-# --------------------- User defined variables ---------------------
-LATMIN = '-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
-LATMAX = '20.0' #max latitude; -ve values in the SH e.g. 5S = -5
-LONMIN = '-30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
-LONMAX = '30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
-T_BB_MAX = '250' #warmest temp to allow in K
-T_BB_MIN = '180' #coolest temp to allow in K
-#------------------- End user defined Variables -------------------
-
-#------------------------ End GLOBAL VARS -------------------------
-
-def preprocessingMERG(MERGdirname):
- '''
- Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format. The files end up in a folder called mergNETCDF in the directory
- where the raw MERG data is
- NOTE: VERY RAW AND DIRTY
-
- Input::
- Directory to the location of the raw MERG files, preferably zipped
-
- Output::
- none
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
-
- os.chdir((MERGdirname+'/'))
- imgFilename = ''
-
- #Just incase the X11 server is giving problems
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- #for files in glob.glob("*-pixel"):
- for files in glob.glob("*.Z"):
- fname = os.path.splitext(files)[0]
-
- #unzip it
- bash_cmd = 'gunzip ' + files
- subprocess.call(bash_cmd, shell=True)
-
- #determine the time from the filename
- ftime = re.search('\_(.*)\_',fname).group(1)
-
- yy = ftime[0:4]
- mm = ftime[4:6]
- day = ftime[6:8]
- hr = ftime [8:10]
-
- #TODO: must be something more pythonic!
-
- if mm=='01':
- mth = 'Jan'
- if mm == '02':
- mth = 'Feb'
- if mm == '03':
- mth = 'Mar'
- if mm == '04':
- mth = 'Apr'
- if mm == '05':
- mth = 'May'
- if mm == '06':
- mth = 'Jun'
- if mm == '07':
- mth = 'Jul'
- if mm == '08':
- mth = 'Aug'
- if mm == '09':
- mth = 'Sep'
- if mm == '10':
- mth = 'Oct'
- if mm == '11':
- mth = 'Nov'
- if mm == '12':
- mth = 'Dec'
-
-
- subprocess.call('rm merg.ctl', shell=True)
- subprocess.call('touch merg.ctl', shell=True)
- replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
- replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
- subprocess.call(replaceExpDset, shell=True)
- subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
- subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
- subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
- subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
- subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
- subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
- subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
-
- #generate the lats4D command for GrADS
- lats4D = 'lats4d -v -q -lat ' + LATMIN +' ' + LATMAX + ' ' + '-lon ' + LONMIN + ' ' + LONMAX + ' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- gradscmd = 'grads -lbc ' + '\'' +lats4D + '\''
- #run grads and lats4d command
- subprocess.call(gradscmd, shell=True)
-
- #----- Generate GrADS images--------------
- imgFilename = hr+'Z'+day+mth+yy+'.gif'
- tempMaskedImages(imgFilename)
- #-------End Generate GrADS images ---------
-
- #remove the raw data file to save some resources
- rmCmd= "rm " + fname
- subprocess.call(rmCmd, shell=True)
-
- #when all the files have been converted, mv the netcdf files
- subprocess.call('mkdir mergNETCDF', shell=True)
- subprocess.call('mv *.nc mergNETCDF', shell=True)
- #mv all images
- subprocess.call('mkdir mergImgs', shell=True)
- subprocess.call('mv *.gif mergImgs', shell=True)
-
- return
-
-#******************************************************************
-def tempMaskedImages(imgFilename):
- '''
- Purpose:: To generate temperature-masked images for a first pass verification
-
- Input::
- filename for the gif file
-
- Output::
- None - gif images for each file of IR temperature < T_BB_MAX are generated in folder called mergImgs
-
- Assumptions::
- Same as for preprocessingMERG
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
-
- subprocess.call('rm tempMaskedImages.gs', shell=True)
- subprocess.call('touch tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'open merg.ctl''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set mpdset hires''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set lat ' +LATMIN + ' ' +LATMAX +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set lon ' +LONMIN + ' ' +LONMAX +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set cint 10''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set cmax '+T_BB_MAX +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set cmin '+T_BB_MIN +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'d ch4+75''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'draw title Masked Temp @ '+imgFilename +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'quit''\'" >> tempMaskedImages.gs', shell=True)
- gradscmd = 'grads -blc ' + '\'run tempMaskedImages.gs''\''
- subprocess.call(gradscmd, shell=True)
- return
-
-#******************************************************************
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/preprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/preprocessingMERG.py b/mccsearch/code/preprocessingMERG.py
deleted file mode 100755
index 04964a8..0000000
--- a/mccsearch/code/preprocessingMERG.py
+++ /dev/null
@@ -1,125 +0,0 @@
-'''
-#preprocessing for MERG data
-# unzips files and generates the NETCDF files using GrADS lats4D
-# http://opengrads.org/doc/scripts/lats4d/
-# This script requires the merg_template.ctl resides with the folder
-# TODO: generate this file
-# Kim Whitehall
-# Last updated: June 4th 2013
-'''
-import subprocess
-import fileinput
-import json
-import os
-import glob
-import re
-import sys
-
-
-def preprocessing_merg():
- '''
- Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format
- NOTE: VERY RAW AND DIRTY
- Input::
- none
-
- Output::
- none
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
- os.chdir("/Users/whitehal/Documents/kimSum2013/mergData1/")
- searchExpDset = "DSET "
- searchExpTdef = "TDEF "
- #for files in glob.glob("*-pixel"):
- for files in glob.glob("*.Z"):
- fname = os.path.splitext(files)[0]
-
- #unzip it
- bash_cmd = 'gunzip ' + files
- subprocess.call(bash_cmd, shell=True)
-
- #determine the time from the filename
- ftime = re.search('\_(.*)\_',fname).group(1)
-
- yy = ftime[0:4]
- mm = ftime[4:6]
- day = ftime[6:8]
- hr = ftime [8:10]
-
-
- if mm=='01':
- mth = 'Jan'
- if mm == '02':
- mth = 'Feb'
- if mm == '03':
- mth = 'Mar'
- if mm == '04':
- mth = 'Apr'
- if mm == '05':
- mth = 'May'
- if mm == '06':
- mth = 'Jun'
- if mm == '07':
- mth = 'Jul'
- if mm == '08':
- mth = 'Aug'
- if mm == '09':
- mth = 'Sep'
- if mm == '10':
- mth = 'Oct'
- if mm == '11':
- mth = 'Nov'
- if mm == '12':
- mth = 'Dec'
-
-
- subprocess.call('rm merg.ctl', shell=True)
- subprocess.call('touch merg.ctl', shell=True)
- replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
- replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
- subprocess.call(replaceExpDset, shell=True)
- subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
- subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
- subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
- subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
- subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
- subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
- subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
-
-
-
- # #cp template merg.ctl
- # subprocess.call('cp -f merg_template.ctl merg.ctl', shell=True)
-
- # #edit merg.ctl dset line to contain filename by running file
- # #http://stackoverflow.com/questions/39086/search-and-replace-a-line-in-a-file-in-python
- # replaceExpDset = 'DSET ' + fname
- # replaceExpTdef = 'TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn'
- # for line in fileinput.input("merg.ctl", inplace=True):
- # if searchExpDset in line:
- # line = line.replace(searchExpDset, replaceExpDset)
- # if searchExpTdef in line:
- # line = line.replace(searchExpTdef, replaceExpTdef)
- # sys.stdout.write(line)
-
- #lats4d cmd
- lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
- #run grads and lats4d command
- subprocess.call(gradscmd, shell=True)
-
- #when all the files have benn converted, mv the netcdf files
- subprocess.call('mkdir mergNETCDF', shell=True)
- subprocess.call('mv *.nc mergNETCDF', shell=True)
- return
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/process.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.pyc b/mccsearch/code/process.pyc
deleted file mode 100644
index d0133fc..0000000
Binary files a/mccsearch/code/process.pyc and /dev/null differ
[34/50] [abbrv] initial mccsearch code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
new file mode 100644
index 0000000..177aef8
--- /dev/null
+++ b/mccsearch/code/mccSearch.py
@@ -0,0 +1,4340 @@
+'''
+# All the functions for the MCC search algorithm
+# Following RCMES dataformat in format (t,lat,lon), value
+# Kim Whitehall
+# Mimumum MCS is 3 hours
+'''
+
+import datetime
+from datetime import timedelta, datetime
+import calendar
+import fileinput
+import glob
+import itertools
+import json
+import math
+import Nio
+from netCDF4 import Dataset, num2date, date2num
+import numpy as np
+import numpy.ma as ma
+import os
+import pickle
+import re
+from scipy import ndimage
+import string
+import subprocess
+import sys
+import time
+
+import networkx as nx
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+from matplotlib.dates import DateFormatter,HourLocator
+from matplotlib import cm
+import matplotlib.cm as cm
+import matplotlib.colors as mcolors
+from matplotlib.ticker import FuncFormatter, FormatStrFormatter
+
+#existing modules in services
+import files
+import process
+#----------------------- GLOBAL VARIABLES --------------------------
+# --------------------- User defined variables ---------------------
+#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
+#the first point closest the the value (for the min) from the MERG data is used, etc.
+LATMIN = '5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '19.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
+LONMIN = '-9.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '5.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+XRES = 4.0 #x direction spatial resolution in km
+YRES = 4.0 #y direction spatial resolution in km
+TRES = 1 #temporal resolution in hrs
+T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
+T_BB_MIN = 218 #221 #cooler temp for the center of the system
+STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
+ #have same rank of the matrix it is being compared against
+CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
+MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
+MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
+MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
+LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
+DIRECTION = 45.0 #general direction of the wind flow in degrees
+AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
+MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+
+#---Assuming using the MCC function, these will have to be changed
+ECCENTRICITY_THRESHOLD_MAX = 1.0 #tending to 1 is a circle e.g. hurricane,
+ECCENTRICITY_THRESHOLD_MIN = 0.70 #0.65 #tending to 0 is a linear e.g. squall line
+OUTER_CLOUD_SHIELD_AREA = 80000.0 #100000.0 #km^2
+INNER_CLOUD_SHIELD_AREA = 30000.0 #50000.0 #km^2
+OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
+INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
+MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
+MAXIMUM_DURATION = 100 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
+
+#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+#------------------- End user defined Variables -------------------
+edgeWeight = [1,2,3] #weights for the graph edges
+#graph object fo the CEs meeting the criteria
+CLOUD_ELEMENT_GRAPH = nx.DiGraph()
+#graph meeting the CC criteria
+PRUNED_GRAPH = nx.DiGraph()
+#------------------------ End GLOBAL VARS -------------------------
+#************************ Begin Functions *************************
+#******************************************************************
+def readMergData(dirname):
+ '''
+ Purpose::
+ Read MERG data into RCMES format
+
+ Input::
+ Directory to the MERG files in NETCDF format
+
+ Output::
+ A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
+ criteria for each frame
+ **remove below**
+ A dictionary of all the MERG data from the files in the directory given.
+ The dictionary contains, a string representing the time (a datetime string), a
+ tuple (lat,lon,value) representing only the data that meets the temperature requirements
+ i.e. T_BB_MAX
+
+ Assumptions::
+ The MERG data has been converted to NETCDF using LATS4D
+ The data has the same lat/lon format
+ '''
+
+ global LAT
+ global LON
+
+ # these strings are specific to the MERG data
+ mergVarName = 'ch4'
+ mergTimeVarName = 'time'
+ mergLatVarName = 'latitude'
+ mergLonVarName = 'longitude'
+
+ filelistInstructions = dirname + '/*'
+ filelist = glob.glob(filelistInstructions)
+
+
+ #sat_img is the array that will contain all the masked frames
+ mergImgs = []
+ #timelist of python time strings
+ timelist = []
+ time2store = None
+ tempMaskedValueNp =[]
+
+
+ filelist.sort()
+ nfiles = len(filelist)
+
+ # Crash nicely if there are no netcdf files
+ if nfiles == 0:
+ print 'Error: no files in this directory! Exiting elegantly'
+ sys.exit()
+ else:
+ # Open the first file in the list to read in lats, lons and generate the grid for comparison
+ tmp = Nio.open_file(filelist[0], format='nc')
+ #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
+
+ #clip the lat/lon grid according to user input
+ #http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
+ latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
+ lonsraw = tmp.variables[mergLonVarName][mergLonVarName+"|"+LONMIN+":"+LONMAX].astype('f2')
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ LON, LAT = np.meshgrid(lonsraw, latsraw)
+ #clean up
+ latsraw =[]
+ lonsraw = []
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+ #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
+ tmp.close
+
+ for files in filelist:
+ try:
+ thisFile = Nio.open_file(files, format='nc')
+ #TODO: thisFile = netCDF4.Dataset(files)
+
+ #clip the dataset according to user lat,lon coordinates
+ #mask the data and fill with zeros for later
+ tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
+ +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
+
+ tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
+
+ #get the actual values that the mask returned
+ tempMaskedValue = ma.zeros((tempRaw.shape)).astype('int16')
+ for index, value in maenumerate(tempMask):
+ time_index, lat_index, lon_index = index
+ tempMaskedValue[time_index,lat_index, lon_index]=value
+
+ timesRaw = thisFile.variables[mergTimeVarName]
+ #convert this time to a python datastring
+ time2store, _ = process.getModelTimes(files, mergTimeVarName)
+ #extend instead of append because getModelTimes returns a list already and we don't
+ #want a list of list
+ timelist.extend(time2store)
+ #to get the actual masked data only
+ #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
+ mergImgs.extend(tempMaskedValue)
+ thisFile.close
+ thisFile = None
+
+ except:
+ print "bad file! ", file
+
+ mergImgs = ma.array(mergImgs)
+
+ return mergImgs, timelist
+#******************************************************************
+def findCloudElements(mergImgs,timelist,TRMMdirName=None):
+ '''
+ Purpose::
+ Determines the contiguous boxes for a given time of the satellite images i.e. each frame
+ using scipy ndimage package
+
+ Input::
+ 3 variables
+ sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
+ maximum acceptable temperature, T_BB_MAX
+ timelist: a list of python datatimes
+ TRMMdirName (optional): string representing the path where to find the TRMM datafiles
+
+ Output::
+ 1 variable
+ cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
+ The dictionary is
+ cloudElementDict = {'uniqueID': unique tag for this CE,
+ 'cloudElementTime': time of the CE,
+ 'cloudElementLatLon': (lat,lon,value) of MERG data of CE,
+ 'cloudElementCenter':tuple of floating-point (lat,lon) representing the CE's center
+ 'cloudElementArea':floating-point representing the area of the CE,
+ 'cloudElementEccentricity': floating-point representing the shape of the CE,
+ 'cloudElementTmax':integer representing the maximum Tb in CE,
+ 'cloudElementTmin': integer representing the minimum Tb in CE,
+ 'cloudElementPrecipTotal':floating-point representing the sum of all rainfall in CE if TRMMdirName entered,
+ 'cloudElementLatLonTRMM':(lat,lon,value) of TRMM data in CE if TRMMdirName entered,
+ 'TRMMArea': floating-point representing the CE if TRMMdirName entered,
+ 'CETRMMmax':floating-point representing the max rate in the CE if TRMMdirName entered,
+ 'CETRMMmin':floating-point representing the min rate in the CE if TRMMdirName entered}
+ Assumptions::
+ Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
+ required according to Vila et al. (2008) is 2400km^2
+ therefore, 2400/16 = 150 contiguous squares
+ '''
+
+ frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
+ CEcounter = 0
+ frameCEcounter = 0
+ frameNum = 0
+ cloudElementEpsilon = 0.0
+ cloudElementDict = {}
+ cloudElementCenter = [] #list with two elements [lat,lon] for the center of a CE
+ prevFrameCEs = [] #list for CEs in previous frame
+ currFrameCEs = [] #list for CEs in current frame
+ cloudElementLat = [] #list for a particular CE's lat values
+ cloudElementLon = [] #list for a particular CE's lon values
+ cloudElementLatLons = [] #list for a particular CE's (lat,lon) values
+
+ prevLatValue = 0.0
+ prevLonValue = 0.0
+ TIR_min = 0.0
+ TIR_max = 0.0
+ temporalRes = 3 # TRMM data is 3 hourly
+ precipTotal = 0.0
+ CETRMMList =[]
+ precip =[]
+ TRMMCloudElementLatLons =[]
+
+ #edgeWeight = [1,2]
+ minCELatLimit = 0.0
+ minCELonLimit = 0.0
+ maxCELatLimit = 0.0
+ maxCELonLimit = 0.0
+
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ #openfile for storing ALL cloudElement information
+ cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
+
+
+ #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
+ cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
+ #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
+
+ #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
+ for t in xrange(mergImgs.shape[0]):
+ #-------------------------------------------------
+ # #textfile name for saving the data for arcgis
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ #determine contiguous locations with temeperature below the warmest temp i.e. cloudElements in each frame
+ frame, CEcounter = ndimage.measurements.label(mergImgs[t,:,:], structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ frameNum += 1
+
+ #for each of the areas identified, check to determine if it a valid CE via an area and T requirement
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ try:
+ loc = ndimage.find_objects(frame==(count+1))[0]
+ except Exception, e:
+ print "Error is ", e
+ continue
+
+
+ cloudElement = mergImgs[t,:,:][loc]
+ labels, lcounter = ndimage.label(cloudElement)
+
+ #determine the true lats and lons for this particular CE
+ cloudElementLat = LAT[loc[0],0]
+ cloudElementLon = LON[0,loc[1]]
+
+ #determine number of boxes in this cloudelement
+ numOfBoxes = np.count_nonzero(cloudElement)
+ cloudElementArea = numOfBoxes*XRES*YRES
+
+ #If the area is greater than the area required, then start considering as CE
+ #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
+ #if it does, still consider as CE
+
+ if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
+
+ #get some time information and labeling info
+ frameTime = str(timelist[t])
+ frameCEcounter +=1
+ CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
+
+ #-------------------------------------------------
+ #textfile name for accesing CE data using MATLAB code
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ # ------ NETCDF File stuff for brightness temp stuff ------------------------------------
+ thisFileName = MAINDIRECTORY +'/MERGnetcdfCEs/cloudElements'+ (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFCEData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFCEData.description = 'Cloud Element '+CEuniqueID + ' temperature data'
+ currNetCDFCEData.calendar = 'standard'
+ currNetCDFCEData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFCEData.createDimension('time', None)
+ currNetCDFCEData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFCEData.createDimension('lon', len(LON[0,:]))
+ # variables
+ tempDims = ('time','lat', 'lon',)
+ times = currNetCDFCEData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitudes = currNetCDFCEData.createVariable('latitude', 'f8', ('lat',))
+ longitudes = currNetCDFCEData.createVariable('longitude', 'f8', ('lon',))
+ brightnesstemp = currNetCDFCEData.createVariable('brightnesstemp', 'i16',tempDims )
+ brightnesstemp.units = 'Kelvin'
+ # NETCDF data
+ dates=[timelist[t]+timedelta(hours=0)]
+ times[:] = date2num(dates,units=times.units)
+ longitudes[:] = LON[0,:]
+ longitudes.units = "degrees_east"
+ longitudes.long_name = "Longitude"
+
+ latitudes[:] = LAT[:,0]
+ latitudes.units = "degrees_north"
+ latitudes.long_name ="Latitude"
+
+ #generate array of zeros for brightness temperature
+ brightnesstemp1 = ma.zeros((1,len(latitudes), len(longitudes))).astype('int16')
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #if other dataset (TRMM) assumed to be a precipitation dataset was entered
+ if TRMMdirName:
+ #------------------TRMM stuff -------------------------------------------------
+ fileDate = ((str(timelist[t])).replace(" ", "")[:-8]).replace("-","")
+ fileHr1 = (str(timelist[t])).replace(" ", "")[-8:-6]
+
+ if int(fileHr1) % temporalRes == 0:
+ fileHr = fileHr1
+ else:
+ fileHr = (int(fileHr1)/temporalRes) * temporalRes
+ if fileHr < 10:
+ fileHr = '0'+str(fileHr)
+ else:
+ str(fileHr)
+
+ #open TRMM file for the resolution info and to create the appropriate sized grid
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+str(fileHr)+".7A.nc"
+
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ nygrdTRMM = len(LATTRMM[:,0]); nxgrdTRMM = len(LONTRMM[0,:])
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ #get the lat/lon info from the file
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+ # ------ NETCDF File info for writing TRMM CE rainfall ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' precipitation data'
+ currNetCDFTRMMData.calendar = 'standard'
+ currNetCDFTRMMData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFTRMMData.createDimension('time', None)
+ currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
+
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value)
+
+ #generate the comma separated file for GIS
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ #temp data for CE NETCDF file
+ brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value
+
+ if TRMMdirName:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])]
+ CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]]))
+
+
+ brightnesstemp[:] = brightnesstemp1[:]
+ currNetCDFCEData.close()
+
+ if TRMMdirName:
+
+ #calculate the total precip associated with the feature
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ precipTotal += value
+ precip.append(value)
+
+ rainFallacc[:] = finalCETRMMvalues[:]
+ currNetCDFTRMMData.close()
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ try:
+ maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ pass
+
+ #sort cloudElementLatLons by lats
+ cloudElementLatLons.sort(key=lambda tup: tup[0])
+
+ #determine if the cloud element the shape
+ cloudElementEpsilon = eccentricity (cloudElement)
+ cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementCenter.append(latCenter)
+ cloudElementCenter.append(lonCenter)
+ cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsUserFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ #populate the dictionary
+ if TRMMdirName:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min, 'cloudElementPrecipTotal':precipTotal,'cloudElementLatLonTRMM':CETRMMList, 'TRMMArea': TRMMArea,'CETRMMmax':maxCEprecipRate, 'CETRMMmin':minCEprecipRate}
+ else:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min,}
+
+ #current frame list of CEs
+ currFrameCEs.append(cloudElementDict)
+
+ #draw the graph node
+ CLOUD_ELEMENT_GRAPH.add_node(CEuniqueID, cloudElementDict)
+
+ if frameNum != 1:
+ for cloudElementDict in prevFrameCEs:
+ thisCElen = len(cloudElementLatLons)
+ percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
+
+ #change weights to integers because the built in shortest path chokes on floating pts
+ #according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
+ if percentageOverlap >= 0.95:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
+
+ elif percentageOverlap >= 0.90 and percentageOverlap < 0.95 :
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[1])
+
+ elif areaOverlap >= MIN_OVERLAP:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
+
+ else:
+ #TODO: remove this else as we only wish for the CE details
+ #ensure only the non-zero elements are considered
+ #store intel in allCE file
+ labels, _ = ndimage.label(cloudElement)
+ cloudElementsFile.write("\n-----------------------------------------------")
+ cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ # cloudElementLat = LAT[loc[0],0]
+ # cloudElementLon = LON[0,loc[1]]
+
+ #populate cloudElementLatLons by unpacking the original values from loc
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ cloudElementsFile.write("\n-----------------------------------------------")
+
+ #reset list for the next CE
+ nodeExist = False
+ cloudElementCenter=[]
+ cloudElement = []
+ cloudElementCenter = []
+ cloudElementLat=[]
+ cloudElementLon =[]
+ cloudElementLatLons =[]
+ brightnesstemp1 =[]
+ brightnesstemp =[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ CETRMMList =[]
+ precipTotal = 0.0
+ precip=[]
+ TRMMCloudElementLatLons=[]
+
+ #reset for the next time
+ prevFrameCEs =[]
+ prevFrameCEs = currFrameCEs
+ currFrameCEs =[]
+
+ cloudElementsFile.close
+ cloudElementsUserFile.close
+ #if using ARCGIS data store code, uncomment this file close line
+ #cloudElementsTextFile.close
+
+ #clean up graph - remove parent and childless nodes
+ outAndInDeg = CLOUD_ELEMENT_GRAPH.degree_iter()
+ toRemove = [node[0] for node in outAndInDeg if node[1]<1]
+ CLOUD_ELEMENT_GRAPH.remove_nodes_from(toRemove)
+
+ print "number of nodes are: ", CLOUD_ELEMENT_GRAPH.number_of_nodes()
+ print "number of edges are: ", CLOUD_ELEMENT_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #hierachial graph output
+ #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
+ drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
+
+ return CLOUD_ELEMENT_GRAPH
+#******************************************************************
+def findPrecipRate(TRMMdirName):
+ '''
+ TODO: needs fixing
+ Purpose:: Determines the precipitation rates for MCSs found if TRMMdirName was not entered in findCloudElements this can be used
+
+ Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
+
+ Output:: a list of dictionary of the TRMM data
+ NB: also creates netCDF with TRMM data for each CE (for post processing) index
+ in MAINDIRECTORY/TRMMnetcdfCEs
+
+ Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
+
+ '''
+ allCEnodesTRMMdata =[]
+ TRMMdataDict={}
+ precipTotal = 0.0
+
+ os.chdir((MAINDIRECTORY+'/MERGnetcdfCEs/'))
+ imgFilename = ''
+ temporalRes = 3 #3 hours for TRMM
+
+ #sort files
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+
+ for afile in files:
+ #try:
+ print "in for ", afile
+ fullFname = os.path.splitext(afile)[0]
+ noFrameExtension = (fullFname.replace("_","")).split('F')[0]
+ CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
+ fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
+ fileDateTime = fileDateTimeChar.replace("-","")
+ fileDate = fileDateTime[:-6]
+ fileHr=fileDateTime[-6:-4]
+
+ cloudElementData = Dataset(afile,'r', format='NETCDF4')
+ brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
+ latsrawCloudElements = cloudElementData.variables['latitude'][:]
+ lonsrawCloudElements = cloudElementData.variables['longitude'][:]
+
+
+ if int(fileHr) % 3 == 0:
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
+ #print "TRMMfileName is ", TRMMfileName
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+
+
+ # ------ NETCDF File stuff ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
+ # print "thisFileName ", thisFileName
+ # sys.exit()
+ currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
+ currNetCDFData.calendar = 'standard'
+ currNetCDFData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFData.createDimension('time', None)
+ currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
+ currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+ #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
+ for index,value in np.ndenumerate(brightnesstemp):
+ #print "index, value ", index, value
+ time_index, lat_index, lon_index = index
+ currTimeValue = 0
+ if value > 0:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
+
+
+ # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
+ # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
+ # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
+ # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
+ # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
+ # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
+ # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
+ # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
+ # # precipTotal = value*temporalRes*1.0
+ # # prevLatValue = currLatvalue
+ # # prevLonValue = currLonValue
+
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ rainFallacc[:] = finalCETRMMvalues
+ currNetCDFData.close()
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ #print "index, value ", index, value
+ precipTotal += value
+
+ # #add info to the dictionary and to the list
+ # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
+ # allCEnodesTRMMdata.append(TRMMdataDict)
+ # print "precipTotal is ", precipTotal
+ #add info to CLOUDELEMENTSGRAPH
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
+ if eachdict[1]['uniqueID'] == CEuniqueID:
+ if not 'cloudElementPrecipTotal' in eachdict[1].keys():
+ eachdict[1]['cloudElementPrecipTotal'] = precipTotal
+ if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
+ eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
+ if not 'TRMMArea' in eachdict[1].keys():
+ eachdict[1]['TRMMArea'] = TRMMArea
+ #clean up
+ precipTotal = 0.0
+ latsrawTRMMData =[]
+ lonsrawTRMMData = []
+ latsrawCloudElements=[]
+ lonsrawCloudElements=[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ brightnesstemp =[]
+ TRMMdataDict ={}
+
+ return allCEnodesTRMMdata
+#******************************************************************
+def findCloudClusters(CEGraph):
+ '''
+ Purpose:: Determines the cloud clusters properties from the subgraphs in
+ the graph i.e. prunes the graph according to the minimum depth
+
+ Input:: CEGraph: 1 graph - a directed graph of the CEs with weighted edges
+ according the area overlap between nodes (CEs) of consectuive frames
+
+ Output:: PRUNED_GRAPH: 1 graph - a directed graph of with CCs/ MCSs
+
+ '''
+
+ seenNode = []
+ allMCSLists =[]
+ pathDictList =[]
+ pathList=[]
+
+ cloudClustersFile = open((MAINDIRECTORY+'/textFiles/cloudClusters.txt'),'wb')
+
+ for eachNode in CEGraph:
+ #check if the node has been seen before
+ if eachNode not in dict(enumerate(zip(*seenNode))):
+ #look for all trees associated with node as the root
+ thisPathDistanceAndLength = nx.single_source_dijkstra(CEGraph, eachNode)
+ #determine the actual shortestPath and minimum depth/length
+ maxDepthAndMinPath = findMaxDepthAndMinPath(thisPathDistanceAndLength)
+ if maxDepthAndMinPath:
+ maxPathLength = maxDepthAndMinPath[0]
+ shortestPath = maxDepthAndMinPath[1]
+
+ #add nodes and paths to PRUNED_GRAPH
+ for i in xrange(len(shortestPath)):
+ if PRUNED_GRAPH.has_node(shortestPath[i]) is False:
+ PRUNED_GRAPH.add_node(shortestPath[i])
+
+ #add edge if necessary
+ if i < (len(shortestPath)-1) and PRUNED_GRAPH.has_edge(shortestPath[i], shortestPath[i+1]) is False:
+ prunedGraphEdgeweight = CEGraph.get_edge_data(shortestPath[i], shortestPath[i+1])['weight']
+ PRUNED_GRAPH.add_edge(shortestPath[i], shortestPath[i+1], weight=prunedGraphEdgeweight)
+
+ #note information in a file for consideration later i.e. checking to see if it works
+ cloudClustersFile.write("\nSubtree pathlength is %d and path is %s" %(maxPathLength, shortestPath))
+ #update seenNode info
+ seenNode.append(shortestPath)
+
+ print "pruned graph"
+ print "number of nodes are: ", PRUNED_GRAPH.number_of_nodes()
+ print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
+ drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
+ cloudClustersFile.close
+
+ return PRUNED_GRAPH
+#******************************************************************
+def findMCC (prunedGraph):
+ '''
+ Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
+
+ Input:: prunedGraph: a Networkx Graph representing the CCs
+
+ Output:: finalMCCList: a list of list of tuples representing a MCC
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ MCCList = []
+ MCSList = []
+ definiteMCC = []
+ definiteMCS = []
+ eachList =[]
+ eachMCCList =[]
+ maturing = False
+ decaying = False
+ fNode = ''
+ lNode = ''
+ removeList =[]
+ imgCount = 0
+ imgTitle =''
+
+ maxShieldNode = ''
+ orderedPath =[]
+ treeTraversalList =[]
+ definiteMCCFlag = False
+ unDirGraph = nx.Graph()
+ aSubGraph = nx.DiGraph()
+ definiteMCSFlag = False
+
+
+ #connected_components is not available for DiGraph, so generate graph as undirected
+ unDirGraph = PRUNED_GRAPH.to_undirected()
+ subGraph = nx.connected_component_subgraphs(unDirGraph)
+
+ #for each path in the subgraphs determined
+ for path in subGraph:
+ #definite is a subTree provided the duration is longer than 3 hours
+
+ if len(path.nodes()) > MIN_MCS_DURATION:
+ orderedPath = path.nodes()
+ orderedPath.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ #definiteMCS.append(orderedPath)
+
+ #build back DiGraph for checking purposes/paper purposes
+ aSubGraph.add_nodes_from(path.nodes())
+ for eachNode in path.nodes():
+ if prunedGraph.predecessors(eachNode):
+ for node in prunedGraph.predecessors(eachNode):
+ aSubGraph.add_edge(node,eachNode,weight=edgeWeight[0])
+
+ if prunedGraph.successors(eachNode):
+ for node in prunedGraph.successors(eachNode):
+ aSubGraph.add_edge(eachNode,node,weight=edgeWeight[0])
+ imgTitle = 'CC'+str(imgCount+1)
+ drawGraph(aSubGraph, imgTitle, edgeWeight) #for eachNode in path:
+ imgCount +=1
+ #----------end build back ---------------------------------------------
+
+ mergeList, splitList = hasMergesOrSplits(path)
+ #add node behavior regarding neutral, merge, split or both
+ for node in path:
+ if node in mergeList and node in splitList:
+ addNodeBehaviorIdentifier(node,'B')
+ elif node in mergeList and not node in splitList:
+ addNodeBehaviorIdentifier(node,'M')
+ elif node in splitList and not node in mergeList:
+ addNodeBehaviorIdentifier(node,'S')
+ else:
+ addNodeBehaviorIdentifier(node,'N')
+
+
+ #Do the first part of checking for the MCC feature
+ #find the path
+ treeTraversalList = traverseTree(aSubGraph, orderedPath[0],[],[])
+ #print "treeTraversalList is ", treeTraversalList
+ #check the nodes to determine if a MCC on just the area criteria (consecutive nodes meeting the area and temp requirements)
+ MCCList = checkedNodesMCC(prunedGraph, treeTraversalList)
+ for aDict in MCCList:
+ for eachNode in aDict["fullMCSMCC"]:
+ addNodeMCSIdentifier(eachNode[0],eachNode[1])
+
+ #do check for if MCCs overlap
+ if MCCList:
+ if len(MCCList) > 1:
+ for count in range(len(MCCList)): #for eachDict in MCCList:
+ #if there are more than two lists
+ if count >= 1:
+ #and the first node in this list
+ eachList = list(x[0] for x in MCCList[count]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ fNode = eachList[0]
+ #get the lastNode in the previous possMCC list
+ eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ lNode = eachList[-1]
+ if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ if aNode in eachList and aNode == lNode:
+ #if edge_data is equal or less than to the exisitng edge in the tree append one to the other
+ if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']:
+ MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"])
+ MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"])
+ MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"]
+ MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
+ MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
+ MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"]
+ removeList.append(count)
+ #update the MCCList
+ if removeList:
+ for i in removeList:
+ if (len(MCCList)-1) > i:
+ del MCCList[i]
+ removeList =[]
+
+ #check if the nodes also meet the duration criteria and the shape crieria
+ for eachDict in MCCList:
+ #order the fullMCSMCC list, then run maximum extent and eccentricity criteria
+ if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION:
+ eachList = list(x[0] for x in eachDict["fullMCSMCC"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ eachMCCList = list(x[0] for x in eachDict["possMCCList"])
+ eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+
+ #update the nodemcsidentifer behavior
+ #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
+ #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
+ #ensure that everything between is listed as 'M'
+ for eachNode in eachList[:(eachList.index(eachMCCList[0]))]:
+ addNodeMCSIdentifier(eachNode,'I')
+
+ addNodeMCSIdentifier(eachMCCList[0],'M')
+
+ for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #update definiteMCS list
+ for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #run maximum extent and eccentricity criteria
+ maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList)
+ #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag
+ if definiteMCCFlag == True:
+ definiteMCC.append(eachList)
+
+
+ definiteMCS.append(orderedPath)
+
+ #reset for next subGraph
+ aSubGraph.clear()
+ orderedPath=[]
+ MCCList =[]
+ MCSList =[]
+ definiteMCSFlag = False
+
+ return definiteMCC, definiteMCS
+#******************************************************************
+def traverseTree(subGraph,node, queue, checkedNodes=None):
+ '''
+ Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+
+ Input:: subGraph: a Networkx DiGraph representing a CC
+ lengthOfsubGraph: an integer representing the length of the subgraph
+ node: a string representing the node currently being checked
+ queue: a list of strings representing a list of nodes in a stack functionality
+ i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ if len(checkedNodes) == len(subGraph):
+ return checkedNodes
+
+ if not checkedNodes:
+ queue =[]
+ checkedNodes.append(node)
+
+ #check one level infront first...if something does exisit, stick it at the front of the stack
+ upOneLevel = subGraph.predecessors(node)
+ downOneLevel = subGraph.successors(node)
+ for parent in upOneLevel:
+ if parent not in checkedNodes and parent not in queue:
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ queue.insert(0,child)
+ #downCheckFlag = True
+ queue.insert(0,parent)
+
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
+ queue.insert(0,child)
+ else:
+ queue.append(child)
+
+ #print "^^^^^stack ", stack
+ for eachNode in queue:
+ if eachNode not in checkedNodes:
+ #print "if eachNode ", checkedNodes, eachNode, stack
+ checkedNodes.append(eachNode)
+ return traverseTree(subGraph, eachNode, queue, checkedNodes)
+
+ return checkedNodes
+#******************************************************************
+def checkedNodesMCC (prunedGraph, nodeList):
+ '''
+ Purpose :: Determine if this path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ nodeList: list of strings (CE ID) from the traversal
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+ '''
+
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ INITIATIONFLAG = False
+ MATURITYFLAG = False
+ DECAYFLAG = False
+ thisdict = {} #will have the same items as the cloudElementDict
+ cloudElementArea = 0.0
+ epsilon = 0.0
+ frameNum =0
+ oldNode =''
+ potentialMCCList =[]
+ durationAandB = 0
+ CounterCriteriaA = 0
+ CountercriteriaB = 0
+
+ #check for if the list contains only one string/node
+ if type(nodeList) is str:
+ oldNode=nodeList
+ nodeList =[]
+ nodeList.append(oldNode)
+
+ for node in nodeList:
+ thisdict = thisDict(node)
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ existingFrameFlag = False
+
+ if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
+ #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaAFlag = True
+ INITIATIONFLAG = True
+ MATURITYFLAG = False
+ #check if criteriaB is meet
+ cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
+
+ #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
+ if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
+ #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaBFlag = True
+ #append this information on to the dictionary
+ addInfothisDict(node, cloudElementArea, criteriaB)
+ INITIATIONFLAG = False
+ MATURITYFLAG = True
+ stage = 'M'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria B failed
+ CounterCriteriaBFlag = False
+ if INITIATIONFLAG == True:
+ stage = 'I'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
+ DECAYFLAG = True
+ MATURITYFLAG = False
+ stage = 'D'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria A failed
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ #add as a CE before or after the main feature
+ if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
+ stage = "D"
+ DECAYFLAG = True
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ return potentialMCCList
+#******************************************************************
+def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
+ '''
+ Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ node: a string representing the cloud element currently being assessed
+ CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
+ CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+
+ '''
+ existingFrameFlag = False
+ existingMCSFrameFlag = False
+ predecessorsFlag = False
+ predecessorsMCSFlag = False
+ successorsFlag = False
+ successorsMCSFlag = False
+ frameNum = 0
+
+ frameNum = int((node.split('CE')[0]).split('F')[1])
+ if potentialMCCList==[]:
+ #list empty
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ else:
+ #list not empty
+ predecessorsFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,1)
+
+ if predecessorsFlag == True:
+
+ for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
+ existingFrameFlag = True
+
+ #this MUST come after the check for the existing frame
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+
+
+ if existingFrameFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage ='M'
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0:
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ if existingFrameFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ if predecessorsFlag == False:
+ successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+
+ if successorsFlag == True:
+ for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
+ existingFrameFlag = True
+
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if existingFrameFlag == False:
+ if stage == 'M':
+ stage = 'D'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ elif CounterCriteriaAFlag == True:
+ potentialMCCList[index]["CounterCriteriaA"] += 1
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ else:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to exisiting MCCs check if it is connected to exisiting MCSs ...
+ if predecessorsFlag == False and successorsFlag == False:
+ stage = 'I'
+ predecessorsMCSFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,2)
+ if predecessorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ else:
+ successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+ if successorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to existing MCCs or MCSs, create a new one ...
+ if predecessorsFlag == False and predecessorsMCSFlag == False and successorsFlag == False and successorsMCSFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ return potentialMCCList
+#******************************************************************
+def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
+ '''
+ Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
+
+ Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
+ upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
+ node: a string representing the cloud element currently being assessed
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
+
+ Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
+ index: an integer representing the location in the potentialMCCList where thisFlag occurs
+
+ '''
+ thisFlag = False
+ index = -1
+ checkList =""
+ if whichList == 1:
+ checkList = "possMCCList"
+ elif whichList ==2:
+ checkList = "fullMCSMCC"
+
+ #check parents
+ if upOrDown == 1:
+ for aNode in prunedGraph.predecessors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ #get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
+ return thisFlag, index
+
+ #check children
+ if upOrDown == 2:
+ for aNode in prunedGraph.successors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ return thisFlag, index
+
+ return thisFlag, index
+#******************************************************************
+def maxExtentAndEccentricity(eachList):
+ '''
+ Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
+
+ Input:: eachList: a list of strings representing the node of the possible MCCs within a path
+
+ Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
+ '''
+ maxShieldNode =''
+ maxShieldArea = 0.0
+ maxShieldEccentricity = 0.0
+ definiteMCCFlag = False
+
+ if eachList:
+ for eachNode in eachList:
+ if (thisDict(eachNode)['nodeMCSIdentifier'] == 'M' or thisDict(eachNode)['nodeMCSIdentifier'] == 'D') and thisDict(eachNode)['cloudElementArea'] > maxShieldArea:
+ maxShieldNode = eachNode
+ maxShieldArea = thisDict(eachNode)['cloudElementArea']
+
+ maxShieldEccentricity = thisDict(maxShieldNode)['cloudElementEccentricity']
+ if thisDict(maxShieldNode)['cloudElementEccentricity'] >= ECCENTRICITY_THRESHOLD_MIN and thisDict(maxShieldNode)['cloudElementEccentricity'] <= ECCENTRICITY_THRESHOLD_MAX :
+ #criteria met
+ definiteMCCFlag = True
+
+ return maxShieldNode, definiteMCCFlag
+#******************************************************************
+def findMaxDepthAndMinPath (thisPathDistanceAndLength):
+ '''
+ Purpose:: To determine the maximum depth and min path for the headnode
+
+ Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
+ thisPathDistanceAndLength({distance}, {path})
+ {distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
+
+ Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
+ minDistanceAndMaxPath = ({distance},{path})
+ '''
+ maxPathLength = 0
+ minPath = 0
+
+ #maxPathLength for the node in question
+ maxPathLength = max(len (values) for values in thisPathDistanceAndLength[1].values())
+
+ #if the duration is shorter then the min MCS length, then don't store!
+ if maxPathLength < MIN_MCS_DURATION: #MINIMUM_DURATION :
+ minDistanceAndMaxPath = ()
+
+ #else find the min path and max depth
+ else:
+ #max path distance for the node in question
+ minPath = max(values for values in thisPathDistanceAndLength[0].values())
+
+ #check to determine the shortest path from the longest paths returned
+ for pathDistance, path in itertools.izip(thisPathDistanceAndLength[0].values(), thisPathDistanceAndLength[1].values()):
+ pathLength = len(path)
+ #if pathLength is the same as the maxPathLength, then look the pathDistance to determine if the min
+ if pathLength == maxPathLength :
+ if pathDistance <= minPath:
+ minPath = pathLength
+ #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
+ minDistanceAndMaxPath = (pathDistance, path)
+ return minDistanceAndMaxPath
+#******************************************************************
+def thisDict (thisNode):
+ '''
+ Purpose:: return dictionary from graph if node exist in tree
+
+ Input:: String - thisNode
+
+ Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ return eachdict[1]
+#******************************************************************
+def checkCriteriaB (thisCloudElementLatLon):
+ '''
+ Purpose:: Determine if criteria B is met for a CEGraph
+
+ Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
+
+ Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
+
+ '''
+ cloudElementCriteriaBLatLon=[]
+
+ frame, CEcounter = ndimage.measurements.label(thisCloudElementLatLon, structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ #determine min and max values in lat and lon, then use this to generate teh array from LAT,LON meshgrid
+
+ minLat = min(x[0] for x in thisCloudElementLatLon)
+ maxLat = max(x[0]for x in thisCloudElementLatLon)
+ minLon = min(x[1]for x in thisCloudElementLatLon)
+ maxLon = max(x[1]for x in thisCloudElementLatLon)
+
+ #print "minLat, maxLat, minLon, maxLon ", minLat, maxLat, minLon, maxLon
+ minLatIndex = np.argmax(LAT[:,0] == minLat)
+ maxLatIndex = np.argmax(LAT[:,0]== maxLat)
+ minLonIndex = np.argmax(LON[0,:] == minLon)
+ maxLonIndex = np.argmax(LON[0,:] == maxLon)
+
+ #print "minLatIndex, maxLatIndex, minLonIndex, maxLonIndex ", minLatIndex, maxLatIndex, minLonIndex, maxLonIndex
+
+ criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
+
+ for x in thisCloudElementLatLon:
+ #to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
+ #index given in the original array to get the indices for the new array
+ #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
+ criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
+
+ #print criteriaBframe
+ #keep only those values < TBB_MAX
+ tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
+
+ #get the actual values that the mask returned
+ criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
+
+ for index, value in maenumerate(tempMask):
+ lat_index, lon_index = index
+ criteriaB[lat_index, lon_index]=value
+
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ #***** returns elements down then across thus (6,4) is 6 arrays deep of size 4
+ try:
+
+ loc = ndimage.find_objects(criteriaB)[0]
+ except:
+ #this would mean that no objects were found meeting criteria B
+ print "no objects at this temperature!"
+ cloudElementArea = 0.0
+ return cloudElementArea, cloudElementCriteriaBLatLon
+
+ print "after loc", loc
+ print "************************"
+ #print "criteriaB ", criteriaB.shape
+ print criteriaB
+ try:
+ cloudElementCriteriaB = ma.zeros((criteriaB.shape))
+ cloudElementCriteriaB =criteriaB[loc]
+ except:
+ print "YIKESS"
+ print "CEcounter ", CEcounter, criteriaB.shape
+ print "criteriaB ", criteriaB
+
+ for index,value in np.ndenumerate(cloudElementCriteriaB):
+ if value !=0:
+ t,lat,lon = index
+ #add back on the minLatIndex and minLonIndex to find the true lat, lon values
+ lat_lon_tuple = (LAT[(lat),0], LON[0,(lon)],value)
+ cloudElementCriteriaBLatLon.append(lat_lon_tuple)
+
+ cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
+ #do some
+ tempMask =[]
+ criteriaB =[]
+ cloudElementCriteriaB=[]
+
+ return cloudElementArea, cloudElementCriteriaBLatLon
+#******************************************************************
+def hasMergesOrSplits (nodeList):
+ '''
+ Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
+ Input:: nodeList - list of nodes from a path
+ Output:: two list_vars_in_file
+ splitList list of all the nodes in the path that split
+ mergeList list of all the nodes in the path that merged
+ '''
+ mergeList=[]
+ splitList=[]
+
+ for node,numParents in PRUNED_GRAPH.in_degree(nodeList).items():
+ if numParents > 1:
+ mergeList.append(node)
+
+ for node, numChildren in PRUNED_GRAPH.out_degree(nodeList).items():
+ if numChildren > 1:
+ splitList.append(node)
+ #sort
+ splitList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ mergeList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+
+ return mergeList,splitList
+#******************************************************************
+def allAncestors(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for parents
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its parents
+ integer - numOfChildren: the number of parents of the node passed
+ '''
+
+ numOfParents = PRUNED_GRAPH.in_degree(aNode)
+ try:
+ if PRUNED_GRAPH.predecessors(aNode) and numOfParents <= 1:
+ path = path + PRUNED_GRAPH.predecessors(aNode)
+ thisNode = PRUNED_GRAPH.predecessors(aNode)[0]
+ return allAncestors(path,thisNode)
+ else:
+ path = path+aNode
+ return path, numOfParents
+ except:
+ return path, numOfParents
+#******************************************************************
+def allDescendants(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for children
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its children
+ integer - numOfChildren: the number of children of the node passed
+ '''
+
+ numOfChildren = PRUNED_GRAPH.out_degree(aNode)
+ try:
+ if PRUNED_GRAPH.successors(aNode) and numOfChildren <= 1:
+ path = path + PRUNED_GRAPH.successors(aNode)
+ thisNode = PRUNED_GRAPH.successors(aNode)[0]
+ return allDescendants(path,thisNode)
+ else:
+ path = path + aNode
+ #i.e. PRUNED_GRAPH.predecessors(aNode) is empty
+ return path, numOfChildren
+ except:
+ #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
+ return path, numOfChildren
+#******************************************************************
+def addInfothisDict (thisNode, cloudElementArea,criteriaB):
+ '''
+ Purpose:: update original dictionary node with information
+
+ Input:: String - thisNode
+ float - cloudElementArea,
+ masked array of floats meeting the criteria - criteriaB
+
+ Output ::
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['CriteriaBArea'] = cloudElementArea
+ eachdict[1]['CriteriaBLatLon'] = criteriaB
+ return
+#******************************************************************
+def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeBehaviorIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeBehaviorIdentifier'] = nodeBehaviorIdentifier
+ return
+#******************************************************************
+def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeMCSIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeMCSIdentifier'] = nodeMCSIdentifier
+ return
+#******************************************************************
+def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['nodeMCSIdentifier'] = nodeBehaviorIdentifier
+
+ return
+#******************************************************************
+def eccentricity (cloudElementLatLon):
+ '''
+ Purpose::
+ Determines the eccentricity (shape) of contiguous boxes
+ Values tending to 1 are more circular by definition, whereas
+ values tending to 0 are more linear
+
+ Input::
+ 1 variable
+ cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
+
+ Output::
+ 1 variable
+ epsilon: a float representing the eccentricity of the matrix passed
+
+ '''
+
+ epsilon = 0.0
+
+ #loop over all lons and determine longest (non-zero) col
+ #loop over all lats and determine longest (non-zero) row
+ for latLon in cloudElementLatLon:
+ #assign a matrix to determine the legit values
+
+ nonEmptyLons = sum(sum(cloudElementLatLon)>0)
+ nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
+
+ lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
+ latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
+ epsilon = min(latEigenvalues,lonEigenvalues)
+
+ return epsilon
+#******************************************************************
+def cloudElementOverlap (currentCELatLons, previousCELatLons):
+ '''
+ Purpose::
+ Determines the percentage overlap between two list of lat-lons passed
+
+ Input::
+ 2 sorted list of tuples
+ currentCELatLons - the list of tuples for the current CE
+ previousCELatLons - the list of tuples for the other CE being considered
+
+ Output::
+ 2 variables
+ percentageOverlap - a float representing the number of overlapping lat_lon tuples
+ areaOverlap - a floating-point number representing the area overlapping
+
+ '''
+
+ latlonprev =[]
+ latloncurr = []
+ count = 0
+ percentageOverlap = 0.0
+ areaOverlap = 0.0
+
+ #remove the temperature from the tuples for currentCELatLons and previousCELatLons then check for overlap
+ latlonprev = [(x[0],x[1]) for x in previousCELatLons]
+ latloncurr = [(x[0],x[1]) for x in currentCELatLons]
+
+ #find overlap
+ count = len(list(set(latloncurr)&set(latlonprev)))
+
+ #find area overlap
+ areaOverlap = count*XRES*YRES
+
+ #find percentage
+ percentageOverlap = max(((count*1.0)/(len(latloncurr)*1.0)),((count*1.0)/(len(latlonprev)*1.0)))
+
+ return percentageOverlap, areaOverlap
+#******************************************************************
+def findCESpeed(node, MCSList):
+ '''
+ Purpose:: to determine the speed of the CEs
+ uses vector displacement delta_lat/delta_lon (y/x)
+
+ Input:: node: a string representing the CE
+ MCSList: a list of strings representing the feature
+
+ Output::
+
+ '''
+
+ delta_lon =0.0
+ delta_lat =0.0
+ CEspeed =[]
+ theSpeed = 0.0
+
+
+ theList = CLOUD_ELEMENT_GRAPH.successors(node)
+ nodeLatLon=thisDict(node)['cloudElementCenter']
+ #print "nodeLatLon ", nodeLatLon
+
+
+ for aNode in theList:
+ if aNode in MCSList:
+ #if aNode is part of the MCSList then determine distance
+ aNodeLatLon = thisDict(aNode)['cloudElementCenter']
+ #print "aNodeLatLon ", aNodeLatLon
+ #calculate CE speed
+ #checking the lats
+ nodeLatLon[0] += 90.0
+ aNodeLatLon[0] += 90.0
+ delta_lat = (nodeLatLon[0] - aNodeLatLon[0]) #convert to m
+ #delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
+ #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
+
+ #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
+ # if nodeLatLon[1] < 0.0:
+ # nodeLatLon[1] += 360.0
+ # if aNodeLatLon[1] <= 0.0:
+ # delta_lon = aNodeLatLon[1] + 360.0
+ #0E --> 360 and E lons > west lons
+ nodeLatLon[1] += 360.0
+ aNodeLatLon[1] += 360.0
+ delta_lon = (nodeLatLon[1] - aNodeLatLon[1]) #convert to m
+ #delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
+
+ #print "delta_lat, delta_lon ", delta_lat, delta_lon
+
+ #theSpeed = abs(((delta_lat*LAT_DISTANCE*1000)/(delta_lon*LON_DISTANCE*1000))/(TRES*3600)) #convert to s --> m/s
+ theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
+
+ CEspeed.append(theSpeed)
+ #print "aNode CE speed is ", aNode, (((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600)), theSpeed
+
+ if not CEspeed:
+ return 0.0
+ else:
+ return min(CEspeed)
+#******************************************************************
+#
+# UTILITY SCRIPTS FOR MCCSEARCH.PY
+#
+#******************************************************************
+def maenumerate(mArray):
+ '''
+ Purpose::
+ Utility script for returning the actual values from the masked array
+ Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
+
+ Input::
+ 1 variable
+ mArray - the masked array returned from the ma.array() command
+
+
+ Output::
+ 1 variable
+ maskedValues - 3D (t,lat,lon), value of only masked values
+
+ '''
+
+ mask = ~mArray.mask.ravel()
+ #beware yield fast, but generates a type called "generate" that does not allow for array methods
+ for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
+ if maskedValue:
+ yield index
+#******************************************************************
+def createMainDirectory(mainDirStr):
+ '''
+ Purpose:: to create the main directory for storing information and
+ the subdirectories for storing information
+ Input:: mainDir: a directory for where all information generated from
+ the program are to be stored
+ Output:: None
+
+ '''
+ global MAINDIRECTORY
+
+ MAINDIRECTORY = mainDirStr
+ #if directory doesnt exist, creat it
+ if not os.path.exists(MAINDIRECTORY):
+ os.makedirs(MAINDIRECTORY)
+
+ os.chdir((MAINDIRECTORY))
+ #create the subdirectories
+ try:
+ os.makedirs('images')
+ os.makedirs('textFiles')
+ os.makedirs('MERGnetcdfCEs')
+ os.makedirs('TRMMnetcdfCEs')
+ except:
+ print "Directory exists already!!!"
+ #TODO: some nice way of prompting if it is ok to continue...or just leave
+
+ return
+#******************************************************************
+def find_nearest(thisArray,value):
+ '''
+ Purpose :: to determine the value within an array closes to
+ another value
+
+ Input ::
+ Output::
+ '''
+ idx = (np.abs(thisArray-value)).argmin()
+ return thisArray[idx]
+#******************************************************************
+def preprocessingMERG(MERGdirname):
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ Directory to the location of the raw MERG files, preferably zipped
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ os.chdir((MERGdirname+'/'))
+ imgFilename = ''
+
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ for files in glob.glob("*-pixel"):
+ #for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+ #TODO: must be something more efficient!
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+ #generate the lats4D command for GrADS
+ lats4D = 'lats4d -v -q -lat '+LATMIN + ' ' +LATMAX +' -lon ' +LONMIN +' ' +LONMAX +' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ #lats4D = 'lats4d -v -q -lat -40 -15 -lon 10 40 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+ #lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+ imgFilename = hr+'Z'+day+mth+yy+'.gif'
+ tempMaskedImages(imgFilename)
+
+ #when all the files have benn converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ #mv all images
+ subprocess.call('mkdir mergImgs', shell=True)
+ subprocess.call('mv *.gif mergImgs', shell=True)
+ return
+#******************************************************************
+def postProcessingNetCDF(dataset, dirName = None):
+ '''
+
+ TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
+
+ Purpose::
+ Utility script displaying the data in generated NETCDF4 files
+ in GrADS
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
+ string: Directory to the location of the raw (MERG) files, preferably zipped
+
+ Output::
+ images in location as specfied in the code
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ '''
+
+ coreDir = os.path.dirname(MAINDIRECTORY)
+ #coreDir = os.path.dirname(os.path.abspath(__file__))
+ ImgFilename = ''
+ frameList=[]
+ fileList =[]
+ lines =[]
+ var =''
+ firstTime = True
+ printLine = 0
+ lineNum = 1
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ print "coreDir is ", str(coreDir)
+ prevFrameNum = 0
+
+ if dataset == 1:
+ var = 'ch4'
+ dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
+ ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
+ ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ gsFile = coreDir+"/GrADSscripts/cs2.gs"
+ sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
+ lineNum = 50
+
+ elif dataset ==2:
+ var = 'precipAcc'
+ dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
+ ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
+ ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
+ origsFile = coreDir+"/GrADSscripts/cs3.gs"
+ gsFile = coreDir+"/GrADSscripts/cs4.gs"
+ sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
+ lineNum = 10
+
+ elif dataset ==3:
+ var = 'ch4'
+ ctlTitle = 'TITLE MERG DATA'
+ ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
+ if dirName is None:
+ print "Enter directory for original files"
+ return
+ else:
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ sologsFile = coreDir+"/GrADSscripts/infrared.gs"
+ lineNum = 54
+
+ #sort files
+ os.chdir((dirName+'/'))
+ try:
+ os.makedirs('ctlFiles')
+ except:
+ print "ctl file folder created already"
+
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+ for eachfile in files:
+ fullFname = os.path.splitext(eachfile)[0]
+ fnameNoExtension = fullFname.split('.nc')[0]
+ if dataset == 1 or dataset == 2:
+ frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
+ #create the ctlFile
+ ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
+ #the ctl file
+ subprocessCall = 'rm ' +ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ subprocessCall = 'touch '+ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ lineToWrite = 'echo DSET ' + dirName+'/'+fnameNoExtension+'.nc' +' >>' + ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo DTYPE netcdf >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo UNDEF 0 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo
<TRUNCATED>
[27/50] [abbrv] git commit: Merge branch 'master' into mccsearch
Posted by le...@apache.org.
Merge branch 'master' into mccsearch
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/8fcd2a20
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/8fcd2a20
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/8fcd2a20
Branch: refs/heads/master
Commit: 8fcd2a20eb12db233e81622f7c0771aad637f3e0
Parents: 587b203 7a0a7e2
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Fri Oct 10 18:57:17 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Fri Oct 10 18:57:17 2014 -0700
----------------------------------------------------------------------
.gitignore | 13 +
CHANGES.txt | 191 +
DISCLAIMER.txt | 15 -
KEYS | 106 +
NOTICE.txt | 6 +
README.md | 23 +
doap_CLIMATE.rdf | 72 +
docs/source/ocw/metrics.rst | 15 +
easy-ocw/bootstrap.py | 262 -
easy-ocw/buildout.cfg | 377 -
easy-ocw/etc/dependencies.cfg | 84 -
easy-ocw/install-osx.sh | 220 +
easy-ocw/install-ubuntu-12_04.sh | 189 +
easy-ocw/install.sh | 198 -
easy-ocw/ocw-conda-dependencies.txt | 22 +
easy-ocw/ocw-pip-dependencies.txt | 9 +
examples/knmi_to_cru31_full_bias.py | 27 +-
examples/simple_model_to_model_bias.py | 33 +-
examples/simple_model_tstd.py | 89 +
examples/taylor_diagram_example.py | 113 +
obs4MIPs/INSTALLATION | 3 +
obs4MIPs/README | 2 +
obs4MIPs/Toolbox/ESGFexcel.py | 125 +
.../examples/ECMWF/ecmwf_table_obs4MIPs.xls | Bin 10752 -> 55296 bytes
obs4MIPs/obs4MIPs_process.py | 29 +-
ocw-cli/ocw_cli.py | 848 +
ocw-ui/backend/processing.py | 24 +-
ocw-ui/backend/run_webservices.py | 30 +-
ocw-ui/backend/tests/__init__.py | 8 +-
ocw-ui/backend/tests/test_directory_helpers.py | 41 +-
.../test_local_file_metadata_extractors.py | 8 +-
ocw-ui/frontend/.bowerrc | 3 +
ocw-ui/frontend/.editorconfig | 21 +
ocw-ui/frontend/.gitattributes | 1 +
ocw-ui/frontend/.gitignore | 5 +
ocw-ui/frontend/.jshintrc | 24 +
ocw-ui/frontend/.travis.yml | 6 +
ocw-ui/frontend/Gruntfile.js | 412 +
ocw-ui/frontend/app/.buildignore | 1 +
ocw-ui/frontend/app/.htaccess | 543 +
ocw-ui/frontend/app/404.html | 175 +
ocw-ui/frontend/app/css/app.css | 108 -
.../app/css/lib/bootstrap-responsive.min.css | 9 -
ocw-ui/frontend/app/css/lib/bootstrap.min.css | 9 -
.../frontend/app/css/lib/font-awesome.min.css | 403 -
ocw-ui/frontend/app/css/lib/timeline.css | 169 -
ocw-ui/frontend/app/favicon.ico | Bin 0 -> 4286 bytes
ocw-ui/frontend/app/font/FontAwesome.otf | Bin 61896 -> 0 bytes
.../frontend/app/font/fontawesome-webfont.eot | Bin 37405 -> 0 bytes
.../frontend/app/font/fontawesome-webfont.svg | 399 -
.../frontend/app/font/fontawesome-webfont.ttf | Bin 79076 -> 0 bytes
.../frontend/app/font/fontawesome-webfont.woff | Bin 43572 -> 0 bytes
ocw-ui/frontend/app/img/globe.png | Bin 25857 -> 0 bytes
.../app/img/glyphicons-halflings-white.png | Bin 8777 -> 0 bytes
.../frontend/app/img/glyphicons-halflings.png | Bin 12799 -> 0 bytes
ocw-ui/frontend/app/index.html | 244 +-
ocw-ui/frontend/app/js/app.js | 91 -
.../app/js/controllers/DatasetDisplayCtrl.js | 34 -
.../app/js/controllers/DatasetSelectCtrl.js | 55 -
.../app/js/controllers/ObservationSelectCtrl.js | 194 -
.../app/js/controllers/ParameterSelectCtrl.js | 296 -
.../app/js/controllers/RcmedSelectionCtrl.js | 133 -
.../frontend/app/js/controllers/ResultCtrl.js | 38 -
.../app/js/controllers/ResultDetailCtrl.js | 38 -
.../frontend/app/js/controllers/SettingsCtrl.js | 24 -
.../frontend/app/js/controllers/TimelineCtrl.js | 92 -
.../frontend/app/js/controllers/WorldMapCtrl.js | 95 -
.../app/js/directives/BootstrapModal.js | 71 -
.../app/js/directives/BootstrapModalOpen.js | 39 -
ocw-ui/frontend/app/js/directives/LeafletMap.js | 38 -
ocw-ui/frontend/app/js/directives/OnBlur.js | 36 -
.../js/directives/PredictiveFileBrowserInput.js | 308 -
ocw-ui/frontend/app/js/directives/PreviewMap.js | 68 -
ocw-ui/frontend/app/js/directives/Timeline.js | 44 -
.../app/js/filters/ISODateToMiddleEndian.js | 48 -
.../js/lib/angular-ui/angular-ui-0.4-tpls.js | 2 -
.../js/lib/angular-ui/angular-ui-router.min.js | 7 -
.../angular-ui/ui-bootstrap-tpls-0.5.0.min.js | 2 -
.../app/js/lib/angular/angular-cookies.js | 183 -
.../app/js/lib/angular/angular-cookies.min.js | 7 -
.../app/js/lib/angular/angular-loader.js | 276 -
.../app/js/lib/angular/angular-loader.min.js | 7 -
.../app/js/lib/angular/angular-resource.js | 445 -
.../app/js/lib/angular/angular-resource.min.js | 10 -
.../app/js/lib/angular/angular-sanitize.js | 535 -
.../app/js/lib/angular/angular-sanitize.min.js | 13 -
ocw-ui/frontend/app/js/lib/angular/angular.js | 14733 ---------
.../frontend/app/js/lib/angular/angular.min.js | 161 -
ocw-ui/frontend/app/js/lib/angular/version.txt | 1 -
.../frontend/app/js/lib/bootstrap/bootstrap.js | 2276 --
.../app/js/lib/bootstrap/bootstrap.min.js | 6 -
.../frontend/app/js/lib/jquery/jquery-1.10.1.js | 9807 ------
.../jquery/jquery-ui/datepicker-wrapper/date.js | 125 -
.../jquery/jquery-ui/jquery-ui-1.10.3.min.js | 12 -
.../frontend/app/js/lib/leaflet/leaflet-0.5.js | 8 -
.../app/js/lib/timeline/timeline-locales.js | 101 -
.../app/js/lib/timeline/timeline-min.js | 199 -
ocw-ui/frontend/app/js/lib/timeline/timeline.js | 6381 ----
.../app/js/services/EvaluationSettings.js | 47 -
.../app/js/services/RegionSelectParams.js | 37 -
.../js/services/SelectedDatasetInformation.js | 49 -
ocw-ui/frontend/app/partials/main.html | 243 -
ocw-ui/frontend/app/partials/modelSelect.html | 86 -
.../frontend/app/partials/results.detail.html | 11 -
ocw-ui/frontend/app/partials/results.html | 15 -
ocw-ui/frontend/app/partials/results.list.html | 3 -
.../app/partials/selectObservation.html | 58 -
ocw-ui/frontend/app/partials/selectRcmed.html | 44 -
ocw-ui/frontend/app/robots.txt | 3 +
ocw-ui/frontend/app/scripts/app.js | 96 +
.../app/scripts/controllers/datasetdisplay.js | 43 +
.../app/scripts/controllers/datasetselect.js | 64 +
ocw-ui/frontend/app/scripts/controllers/main.js | 36 +
.../scripts/controllers/observationselect.js | 203 +
.../app/scripts/controllers/parameterselect.js | 275 +
.../app/scripts/controllers/rcmedselection.js | 144 +
.../frontend/app/scripts/controllers/result.js | 47 +
.../app/scripts/controllers/resultdetail.js | 46 +
.../app/scripts/controllers/settings.js | 34 +
.../app/scripts/controllers/timeline.js | 103 +
.../app/scripts/controllers/worldmap.js | 104 +
.../app/scripts/directives/bootstrapmodal.js | 79 +
.../scripts/directives/bootstrapmodalopen.js | 43 +
.../app/scripts/directives/leafletmap.js | 46 +
.../frontend/app/scripts/directives/onblur.js | 38 +
.../directives/predictivefilebrowserinput.js | 316 +
.../app/scripts/directives/previewmap.js | 76 +
.../frontend/app/scripts/directives/timeline.js | 54 +
.../scripts/filters/isodatetomiddleendian.js | 55 +
.../app/scripts/services/evaluationsettings.js | 56 +
.../app/scripts/services/regionselectparams.js | 46 +
.../services/selecteddatasetinformation.js | 57 +
ocw-ui/frontend/app/styles/main.css | 129 +
ocw-ui/frontend/app/views/main.html | 270 +
ocw-ui/frontend/app/views/modelselect.html | 87 +
ocw-ui/frontend/app/views/results.html | 34 +
ocw-ui/frontend/app/views/resultsdetail.html | 30 +
ocw-ui/frontend/app/views/resultslist.html | 21 +
.../frontend/app/views/selectobservation.html | 80 +
ocw-ui/frontend/app/views/selectrcmed.html | 40 +
ocw-ui/frontend/bower.json | 25 +
ocw-ui/frontend/config/karma-e2e.conf.js | 40 -
ocw-ui/frontend/config/karma.conf.js | 48 -
ocw-ui/frontend/package.json | 45 +
ocw-ui/frontend/scripts/e2e-test.sh | 27 -
ocw-ui/frontend/scripts/test.sh | 27 -
ocw-ui/frontend/scripts/web-server.js | 313 -
ocw-ui/frontend/test/.jshintrc | 36 +
ocw-ui/frontend/test/e2e/runner.html | 28 -
ocw-ui/frontend/test/e2e/scenarios.js | 63 -
ocw-ui/frontend/test/karma.conf.js | 86 +
.../frontend/test/lib/angular/angular-mocks.js | 1764 --
.../test/lib/angular/angular-scenario.js | 26195 -----------------
ocw-ui/frontend/test/lib/angular/version.txt | 1 -
.../test/spec/controllers/datasetdisplay.js | 59 +
.../test/spec/controllers/datasetselect.js | 54 +
ocw-ui/frontend/test/spec/controllers/main.js | 41 +
.../test/spec/controllers/observationselect.js | 122 +
.../test/spec/controllers/parameterselect.js | 136 +
.../test/spec/controllers/rcmedselection.js | 145 +
ocw-ui/frontend/test/spec/controllers/result.js | 37 +
.../test/spec/controllers/resultdetail.js | 37 +
.../frontend/test/spec/controllers/settings.js | 37 +
.../frontend/test/spec/controllers/timeline.js | 37 +
.../frontend/test/spec/controllers/worldmap.js | 57 +
.../test/spec/directives/bootstrapmodal.js | 33 +
.../test/spec/directives/bootstrapmodalopen.js | 33 +
.../frontend/test/spec/directives/leafletmap.js | 33 +
ocw-ui/frontend/test/spec/directives/onblur.js | 50 +
.../directives/predictivefilebrowserinput.js | 33 +
.../frontend/test/spec/directives/previewmap.js | 53 +
.../frontend/test/spec/directives/timeline.js | 33 +
.../test/spec/filters/isodatetomiddleendian.js | 76 +
.../test/spec/services/evaluationsettings.js | 44 +
.../test/spec/services/regionselectparams.js | 44 +
.../spec/services/selecteddatasetinformation.js | 92 +
.../unit/controllers/DatasetDisplayCtrlTest.js | 65 -
.../unit/controllers/DatasetSelectCtrlTest.js | 51 -
.../controllers/ObservationSelectCtrlTest.js | 151 -
.../unit/controllers/ParameterSelectCtrlTest.js | 269 -
.../unit/controllers/RcmedSelectionCtrlTest.js | 161 -
.../test/unit/controllers/SettingsCtrlTest.js | 47 -
.../test/unit/controllers/WorldMapCtrlTest.js | 48 -
.../test/unit/directives/BootstrapModalTest.js | 41 -
.../frontend/test/unit/directives/OnBlurTest.js | 42 -
.../test/unit/directives/PreviewMapTest.js | 46 -
.../unit/filters/ISODateToMiddleEndianTest.js | 67 -
.../unit/services/EvaluationSettingsTest.js | 40 -
.../unit/services/RegionSelectParamsTest.js | 41 -
.../services/SelectedDatasetInfomationTest.js | 90 -
ocw-vm/Vagrantfile | 17 +
ocw-vm/init-ocw-vm.sh | 103 +
ocw/data_source/local.py | 116 +-
ocw/data_source/rcmed.py | 6 +-
ocw/dataset.py | 41 +
ocw/dataset_processor.py | 81 +-
ocw/evaluation.py | 17 +
ocw/metrics.py | 82 +-
ocw/plotter.py | 311 +-
ocw/tests/__init__.py | 0
ocw/tests/test_dap.py | 6 +-
ocw/tests/test_metrics.py | 101 +-
ocw/tests/test_utils.py | 61 +-
ocw/utils.py | 107 +-
rcmet/src/main/vm/build/Vagrantfile | 110 -
.../main/vm/src/cookbooks/apache2/README.rdoc | 78 -
.../src/cookbooks/apache2/attributes/default.rb | 77 -
.../apache2/definitions/apache_conf.rb | 25 -
.../apache2/definitions/apache_module.rb | 43 -
.../apache2/definitions/apache_site.rb | 40 -
.../cookbooks/apache2/definitions/web_app.rb | 49 -
.../default/apache2_module_conf_generate.pl | 41 -
.../main/vm/src/cookbooks/apache2/metadata.json | 497 -
.../main/vm/src/cookbooks/apache2/metadata.rb | 197 -
.../vm/src/cookbooks/apache2/recipes/default.rb | 191 -
.../cookbooks/apache2/recipes/god_monitor.rb | 33 -
.../src/cookbooks/apache2/recipes/mod_alias.rb | 22 -
.../cookbooks/apache2/recipes/mod_auth_basic.rb | 20 -
.../apache2/recipes/mod_auth_digest.rb | 20 -
.../apache2/recipes/mod_auth_openid.rb | 59 -
.../cookbooks/apache2/recipes/mod_authn_file.rb | 20 -
.../apache2/recipes/mod_authnz_ldap.rb | 20 -
.../apache2/recipes/mod_authz_default.rb | 20 -
.../apache2/recipes/mod_authz_groupfile.rb | 20 -
.../cookbooks/apache2/recipes/mod_authz_host.rb | 20 -
.../cookbooks/apache2/recipes/mod_authz_user.rb | 20 -
.../cookbooks/apache2/recipes/mod_autoindex.rb | 22 -
.../vm/src/cookbooks/apache2/recipes/mod_cgi.rb | 20 -
.../vm/src/cookbooks/apache2/recipes/mod_dav.rb | 20 -
.../cookbooks/apache2/recipes/mod_dav_svn.rb | 22 -
.../cookbooks/apache2/recipes/mod_deflate.rb | 22 -
.../vm/src/cookbooks/apache2/recipes/mod_dir.rb | 22 -
.../vm/src/cookbooks/apache2/recipes/mod_env.rb | 20 -
.../cookbooks/apache2/recipes/mod_expires.rb | 20 -
.../src/cookbooks/apache2/recipes/mod_fcgid.rb | 46 -
.../cookbooks/apache2/recipes/mod_headers.rb | 20 -
.../src/cookbooks/apache2/recipes/mod_ldap.rb | 20 -
.../cookbooks/apache2/recipes/mod_log_config.rb | 24 -
.../src/cookbooks/apache2/recipes/mod_mime.rb | 22 -
.../apache2/recipes/mod_negotiation.rb | 22 -
.../src/cookbooks/apache2/recipes/mod_php5.rb | 32 -
.../src/cookbooks/apache2/recipes/mod_proxy.rb | 22 -
.../cookbooks/apache2/recipes/mod_proxy_ajp.rb | 20 -
.../apache2/recipes/mod_proxy_balancer.rb | 20 -
.../apache2/recipes/mod_proxy_connect.rb | 20 -
.../cookbooks/apache2/recipes/mod_proxy_http.rb | 20 -
.../src/cookbooks/apache2/recipes/mod_python.rb | 22 -
.../cookbooks/apache2/recipes/mod_rewrite.rb | 20 -
.../cookbooks/apache2/recipes/mod_setenvif.rb | 22 -
.../vm/src/cookbooks/apache2/recipes/mod_ssl.rb | 42 -
.../src/cookbooks/apache2/recipes/mod_status.rb | 22 -
.../src/cookbooks/apache2/recipes/mod_wsgi.rb | 27 -
.../apache2/templates/default/a2dismod.erb | 22 -
.../apache2/templates/default/a2dissite.erb | 29 -
.../apache2/templates/default/a2enmod.erb | 37 -
.../apache2/templates/default/a2ensite.erb | 38 -
.../apache2/templates/default/apache2.conf.erb | 230 -
.../apache2/templates/default/apache2.god.erb | 19 -
.../apache2/templates/default/charset.erb | 6 -
.../apache2/templates/default/default-site.erb | 57 -
.../templates/default/mod_auth_openid.rb.erb | 12 -
.../apache2/templates/default/mods/README | 2 -
.../templates/default/mods/alias.conf.erb | 24 -
.../templates/default/mods/authopenid.load.erb | 1 -
.../templates/default/mods/autoindex.conf.erb | 101 -
.../templates/default/mods/deflate.conf.erb | 16 -
.../apache2/templates/default/mods/dir.conf.erb | 5 -
.../templates/default/mods/fcgid.conf.erb | 10 -
.../templates/default/mods/mime.conf.erb | 191 -
.../templates/default/mods/negotiation.conf.erb | 18 -
.../templates/default/mods/proxy.conf.erb | 19 -
.../templates/default/mods/setenvif.conf.erb | 28 -
.../apache2/templates/default/mods/ssl.conf.erb | 72 -
.../templates/default/mods/status.conf.erb | 16 -
.../apache2/templates/default/port_apache.erb | 2 -
.../apache2/templates/default/ports.conf.erb | 6 -
.../apache2/templates/default/security.erb | 50 -
.../apache2/templates/default/web_app.conf.erb | 43 -
.../src/cookbooks/apt/files/default/apt-cacher | 9 -
.../cookbooks/apt/files/default/apt-cacher.conf | 144 -
.../apt/files/default/apt-proxy-v2.conf | 50 -
.../src/main/vm/src/cookbooks/apt/metadata.json | 37 -
rcmet/src/main/vm/src/cookbooks/apt/metadata.rb | 11 -
.../main/vm/src/cookbooks/apt/recipes/cacher.rb | 42 -
.../vm/src/cookbooks/apt/recipes/default.rb | 33 -
.../main/vm/src/cookbooks/apt/recipes/proxy.rb | 34 -
.../src/cookbooks/build-essential/CHANGELOG.md | 15 -
.../src/cookbooks/build-essential/CONTRIBUTING | 29 -
.../vm/src/cookbooks/build-essential/LICENSE | 201 -
.../vm/src/cookbooks/build-essential/README.md | 124 -
.../build-essential/attributes/default.rb | 33 -
.../src/cookbooks/build-essential/metadata.json | 37 -
.../src/cookbooks/build-essential/metadata.rb | 12 -
.../build-essential/recipes/default.rb | 79 -
.../main/vm/src/cookbooks/python/CHANGELOG.md | 21 -
.../main/vm/src/cookbooks/python/CONTRIBUTING | 29 -
rcmet/src/main/vm/src/cookbooks/python/LICENSE | 201 -
.../src/main/vm/src/cookbooks/python/README.md | 176 -
.../src/cookbooks/python/attributes/default.rb | 42 -
.../main/vm/src/cookbooks/python/metadata.json | 42 -
.../main/vm/src/cookbooks/python/metadata.rb | 18 -
.../vm/src/cookbooks/python/providers/pip.rb | 167 -
.../cookbooks/python/providers/virtualenv.rb | 68 -
.../vm/src/cookbooks/python/recipes/default.rb | 23 -
.../vm/src/cookbooks/python/recipes/package.rb | 58 -
.../main/vm/src/cookbooks/python/recipes/pip.rb | 41 -
.../vm/src/cookbooks/python/recipes/source.rb | 52 -
.../src/cookbooks/python/recipes/virtualenv.rb | 25 -
.../vm/src/cookbooks/python/resources/pip.rb | 27 -
.../cookbooks/python/resources/virtualenv.rb | 27 -
.../cookbooks/vagrant_main/recipes/default.rb | 56 -
.../templates/default/project.conf.erb | 3 -
.../templates/default/rcmet.conf.erb | 3 -
.../main/vm/src/scripts/ncl/wget-download.sh | 263 -
rcmet/src/main/webapp/.htaccess | 45 -
.../webapp/classes/RCMETWizardTask.class.php | 125 -
.../main/webapp/classes/WizardTask.class.php | 55 -
rcmet/src/main/webapp/config.ini | 66 -
rcmet/src/main/webapp/hooks.php | 83 -
rcmet/src/main/webapp/index.php | 48 -
rcmet/src/main/webapp/scripts/dirlist.php | 25 -
rcmet/src/main/webapp/scripts/plotView.php | 13 -
rcmet/src/main/webapp/scripts/plots.php | 35 -
rcmet/src/main/webapp/scripts/rcmed.php | 23 -
.../bootstrap/css/bootstrap-responsive.css | 1040 -
.../bootstrap/css/bootstrap-responsive.min.css | 9 -
.../webapp/static/bootstrap/css/bootstrap.css | 5624 ----
.../static/bootstrap/css/bootstrap.min.css | 9 -
.../img/glyphicons-halflings-white.png | Bin 8777 -> 0 bytes
.../bootstrap/img/glyphicons-halflings.png | Bin 12799 -> 0 bytes
.../webapp/static/bootstrap/js/bootstrap.js | 2027 --
.../webapp/static/bootstrap/js/bootstrap.min.js | 6 -
.../main/webapp/static/css/balance/balance.css | 16 -
.../main/webapp/static/css/blueprint/LICENSE | 22 -
.../src/main/webapp/static/css/blueprint/ie.css | 29 -
.../main/webapp/static/css/blueprint/print.css | 23 -
.../main/webapp/static/css/blueprint/screen.css | 252 -
.../static/css/jquery-ui-1.7.1.custom.css | 404 -
rcmet/src/main/webapp/static/css/site.css | 19 -
rcmet/src/main/webapp/static/css/style.css | 102 -
.../src/main/webapp/static/img/hexabump-bg.png | Bin 218 -> 0 bytes
rcmet/src/main/webapp/static/img/rcmes-logo.png | Bin 36578 -> 0 bytes
rcmet/src/main/webapp/static/img/wrm-badge.png | Bin 36578 -> 0 bytes
.../main/webapp/static/js/jquery-1.4.2.min.js | 154 -
.../main/webapp/static/js/jquery-1.8.0.min.js | 2 -
.../static/js/jquery-ui-1.7.1.custom.min.js | 298 -
rcmet/src/main/webapp/views/common/footer.php | 17 -
rcmet/src/main/webapp/views/common/header.php | 35 -
rcmet/src/main/webapp/views/errors/404.php | 5 -
rcmet/src/main/webapp/views/index.php | 42 -
rcmet/src/main/webapp/views/wizard/help.php | 17 -
rcmet/src/main/webapp/views/wizard/init.php | 32 -
rcmet/src/main/webapp/views/wizard/step.php | 26 -
.../webapp/views/wizard/steps/generatePlots.php | 125 -
.../views/wizard/steps/selectLatLonVars.php | 63 -
.../views/wizard/steps/selectMetricOptions.php | 49 -
.../views/wizard/steps/selectModelFiles.php | 75 -
.../views/wizard/steps/selectModelVar.php | 53 -
.../wizard/steps/selectObservationalData.php | 86 -
.../views/wizard/steps/selectOptionalTasks.php | 45 -
.../views/wizard/steps/selectPlotOptions.php | 42 -
.../webapp/views/wizard/steps/selectRegrid.php | 58 -
.../views/wizard/steps/selectTimeRange.php | 83 -
.../views/wizard/steps/selectTimeVars.php | 52 -
364 files changed, 8504 insertions(+), 86767 deletions(-)
----------------------------------------------------------------------
[22/50] [abbrv] git commit: Smarten up documentation for mccsearch
branch.
Posted by le...@apache.org.
Smarten up documentation for mccsearch branch.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/abcc73c1
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/abcc73c1
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/abcc73c1
Branch: refs/heads/master
Commit: abcc73c1e661edec5d0776f14b7001e81e4da3dc
Parents: 5070312
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 11:00:14 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Mon Aug 25 11:00:14 2014 -0700
----------------------------------------------------------------------
mccsearch/README.md | 2 +-
mccsearch/docs/mccsearch.md | 18 +++++++++---------
2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/abcc73c1/mccsearch/README.md
----------------------------------------------------------------------
diff --git a/mccsearch/README.md b/mccsearch/README.md
index 6ee45e9..e337ce9 100644
--- a/mccsearch/README.md
+++ b/mccsearch/README.md
@@ -1,3 +1,3 @@
#Graph-based Search for the Identification and Chatacterization of Mesoscale Convective Complexes
-Please see /docs directory for all current documentation relating to this module.
\ No newline at end of file
+Please see /docs/mccsearch.md document for all current documentation relating to this module.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/abcc73c1/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
index 4948228..466e027 100644
--- a/mccsearch/docs/mccsearch.md
+++ b/mccsearch/docs/mccsearch.md
@@ -19,21 +19,21 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
* LATS4D
##Download the source code and store in a folder
- (1) mccSearch.py contains all the function needed
- (2) mccSearchUI.py contains a wizard type Q&A for running the program
- (3) process.py contains some needed functions (from older version of Apache OCW)
- (4) file.py contains some needed functions (from older version of Apache OCW)
- (5) mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
+ * mccSearch.py contains all the function needed
+ * mccSearchUI.py contains a wizard type Q&A for running the program
+ * process.py contains some needed functions (from older version of Apache OCW)
+ * file.py contains some needed functions (from older version of Apache OCW)
+ * mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
* mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be store
* TRMMdirName : this is the directory where the original TRMM data in netCDF format is stored
* CEoriDirName : this is the directory where the original MERG data in netCDF format is stored
- (6) Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
+ * Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
##Download data in one folder (with only the data files)
This is a very crude program. The following assumptions are made:
- (1) input data are in one folder. For MERG data this is CEoriDirName and for the TRMM data this is TRMMdirName in mainProg.py. These directories cannot be the same.
- (2) THERE IS NO FILE CHECKING. So please ensure ALL your files are there in netCDF format.
- (3) THERE IS NO FILE ERROR HANDLING. Please ensure that the MERG data and the TRMM data files are correlated temporally and spatially
+ * input data are in one folder. For MERG data this is CEoriDirName and for the TRMM data this is TRMMdirName in mainProg.py. These directories cannot be the same.
+ * THERE IS NO FILE CHECKING. So please ensure ALL your files are there in netCDF format.
+ * THERE IS NO FILE ERROR HANDLING. Please ensure that the MERG data and the TRMM data files are correlated temporally and spatially
##Run mccSearchUI.py
As a first try to determine the workflow, run the mccSearchUI.py wizard.
[05/50] [abbrv] initial mccsearch code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
new file mode 100644
index 0000000..177aef8
--- /dev/null
+++ b/mccsearch/code/mccSearch.py
@@ -0,0 +1,4340 @@
+'''
+# All the functions for the MCC search algorithm
+# Following RCMES dataformat in format (t,lat,lon), value
+# Kim Whitehall
+# Mimumum MCS is 3 hours
+'''
+
+import datetime
+from datetime import timedelta, datetime
+import calendar
+import fileinput
+import glob
+import itertools
+import json
+import math
+import Nio
+from netCDF4 import Dataset, num2date, date2num
+import numpy as np
+import numpy.ma as ma
+import os
+import pickle
+import re
+from scipy import ndimage
+import string
+import subprocess
+import sys
+import time
+
+import networkx as nx
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+from matplotlib.dates import DateFormatter,HourLocator
+from matplotlib import cm
+import matplotlib.cm as cm
+import matplotlib.colors as mcolors
+from matplotlib.ticker import FuncFormatter, FormatStrFormatter
+
+#existing modules in services
+import files
+import process
+#----------------------- GLOBAL VARIABLES --------------------------
+# --------------------- User defined variables ---------------------
+#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
+#the first point closest the the value (for the min) from the MERG data is used, etc.
+LATMIN = '5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '19.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
+LONMIN = '-9.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '5.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+XRES = 4.0 #x direction spatial resolution in km
+YRES = 4.0 #y direction spatial resolution in km
+TRES = 1 #temporal resolution in hrs
+T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
+T_BB_MIN = 218 #221 #cooler temp for the center of the system
+STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
+ #have same rank of the matrix it is being compared against
+CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
+MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
+MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
+MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
+LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
+DIRECTION = 45.0 #general direction of the wind flow in degrees
+AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
+MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+
+#---Assuming using the MCC function, these will have to be changed
+ECCENTRICITY_THRESHOLD_MAX = 1.0 #tending to 1 is a circle e.g. hurricane,
+ECCENTRICITY_THRESHOLD_MIN = 0.70 #0.65 #tending to 0 is a linear e.g. squall line
+OUTER_CLOUD_SHIELD_AREA = 80000.0 #100000.0 #km^2
+INNER_CLOUD_SHIELD_AREA = 30000.0 #50000.0 #km^2
+OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
+INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
+MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
+MAXIMUM_DURATION = 100 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
+
+#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+#------------------- End user defined Variables -------------------
+edgeWeight = [1,2,3] #weights for the graph edges
+#graph object fo the CEs meeting the criteria
+CLOUD_ELEMENT_GRAPH = nx.DiGraph()
+#graph meeting the CC criteria
+PRUNED_GRAPH = nx.DiGraph()
+#------------------------ End GLOBAL VARS -------------------------
+#************************ Begin Functions *************************
+#******************************************************************
+def readMergData(dirname):
+ '''
+ Purpose::
+ Read MERG data into RCMES format
+
+ Input::
+ Directory to the MERG files in NETCDF format
+
+ Output::
+ A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
+ criteria for each frame
+ **remove below**
+ A dictionary of all the MERG data from the files in the directory given.
+ The dictionary contains, a string representing the time (a datetime string), a
+ tuple (lat,lon,value) representing only the data that meets the temperature requirements
+ i.e. T_BB_MAX
+
+ Assumptions::
+ The MERG data has been converted to NETCDF using LATS4D
+ The data has the same lat/lon format
+ '''
+
+ global LAT
+ global LON
+
+ # these strings are specific to the MERG data
+ mergVarName = 'ch4'
+ mergTimeVarName = 'time'
+ mergLatVarName = 'latitude'
+ mergLonVarName = 'longitude'
+
+ filelistInstructions = dirname + '/*'
+ filelist = glob.glob(filelistInstructions)
+
+
+ #sat_img is the array that will contain all the masked frames
+ mergImgs = []
+ #timelist of python time strings
+ timelist = []
+ time2store = None
+ tempMaskedValueNp =[]
+
+
+ filelist.sort()
+ nfiles = len(filelist)
+
+ # Crash nicely if there are no netcdf files
+ if nfiles == 0:
+ print 'Error: no files in this directory! Exiting elegantly'
+ sys.exit()
+ else:
+ # Open the first file in the list to read in lats, lons and generate the grid for comparison
+ tmp = Nio.open_file(filelist[0], format='nc')
+ #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
+
+ #clip the lat/lon grid according to user input
+ #http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
+ latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
+ lonsraw = tmp.variables[mergLonVarName][mergLonVarName+"|"+LONMIN+":"+LONMAX].astype('f2')
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ LON, LAT = np.meshgrid(lonsraw, latsraw)
+ #clean up
+ latsraw =[]
+ lonsraw = []
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+ #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
+ tmp.close
+
+ for files in filelist:
+ try:
+ thisFile = Nio.open_file(files, format='nc')
+ #TODO: thisFile = netCDF4.Dataset(files)
+
+ #clip the dataset according to user lat,lon coordinates
+ #mask the data and fill with zeros for later
+ tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
+ +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
+
+ tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
+
+ #get the actual values that the mask returned
+ tempMaskedValue = ma.zeros((tempRaw.shape)).astype('int16')
+ for index, value in maenumerate(tempMask):
+ time_index, lat_index, lon_index = index
+ tempMaskedValue[time_index,lat_index, lon_index]=value
+
+ timesRaw = thisFile.variables[mergTimeVarName]
+ #convert this time to a python datastring
+ time2store, _ = process.getModelTimes(files, mergTimeVarName)
+ #extend instead of append because getModelTimes returns a list already and we don't
+ #want a list of list
+ timelist.extend(time2store)
+ #to get the actual masked data only
+ #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
+ mergImgs.extend(tempMaskedValue)
+ thisFile.close
+ thisFile = None
+
+ except:
+ print "bad file! ", file
+
+ mergImgs = ma.array(mergImgs)
+
+ return mergImgs, timelist
+#******************************************************************
+def findCloudElements(mergImgs,timelist,TRMMdirName=None):
+ '''
+ Purpose::
+ Determines the contiguous boxes for a given time of the satellite images i.e. each frame
+ using scipy ndimage package
+
+ Input::
+ 3 variables
+ sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
+ maximum acceptable temperature, T_BB_MAX
+ timelist: a list of python datatimes
+ TRMMdirName (optional): string representing the path where to find the TRMM datafiles
+
+ Output::
+ 1 variable
+ cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
+ The dictionary is
+ cloudElementDict = {'uniqueID': unique tag for this CE,
+ 'cloudElementTime': time of the CE,
+ 'cloudElementLatLon': (lat,lon,value) of MERG data of CE,
+ 'cloudElementCenter':tuple of floating-point (lat,lon) representing the CE's center
+ 'cloudElementArea':floating-point representing the area of the CE,
+ 'cloudElementEccentricity': floating-point representing the shape of the CE,
+ 'cloudElementTmax':integer representing the maximum Tb in CE,
+ 'cloudElementTmin': integer representing the minimum Tb in CE,
+ 'cloudElementPrecipTotal':floating-point representing the sum of all rainfall in CE if TRMMdirName entered,
+ 'cloudElementLatLonTRMM':(lat,lon,value) of TRMM data in CE if TRMMdirName entered,
+ 'TRMMArea': floating-point representing the CE if TRMMdirName entered,
+ 'CETRMMmax':floating-point representing the max rate in the CE if TRMMdirName entered,
+ 'CETRMMmin':floating-point representing the min rate in the CE if TRMMdirName entered}
+ Assumptions::
+ Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
+ required according to Vila et al. (2008) is 2400km^2
+ therefore, 2400/16 = 150 contiguous squares
+ '''
+
+ frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
+ CEcounter = 0
+ frameCEcounter = 0
+ frameNum = 0
+ cloudElementEpsilon = 0.0
+ cloudElementDict = {}
+ cloudElementCenter = [] #list with two elements [lat,lon] for the center of a CE
+ prevFrameCEs = [] #list for CEs in previous frame
+ currFrameCEs = [] #list for CEs in current frame
+ cloudElementLat = [] #list for a particular CE's lat values
+ cloudElementLon = [] #list for a particular CE's lon values
+ cloudElementLatLons = [] #list for a particular CE's (lat,lon) values
+
+ prevLatValue = 0.0
+ prevLonValue = 0.0
+ TIR_min = 0.0
+ TIR_max = 0.0
+ temporalRes = 3 # TRMM data is 3 hourly
+ precipTotal = 0.0
+ CETRMMList =[]
+ precip =[]
+ TRMMCloudElementLatLons =[]
+
+ #edgeWeight = [1,2]
+ minCELatLimit = 0.0
+ minCELonLimit = 0.0
+ maxCELatLimit = 0.0
+ maxCELonLimit = 0.0
+
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ #openfile for storing ALL cloudElement information
+ cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
+
+
+ #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
+ cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
+ #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
+
+ #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
+ for t in xrange(mergImgs.shape[0]):
+ #-------------------------------------------------
+ # #textfile name for saving the data for arcgis
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ #determine contiguous locations with temeperature below the warmest temp i.e. cloudElements in each frame
+ frame, CEcounter = ndimage.measurements.label(mergImgs[t,:,:], structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ frameNum += 1
+
+ #for each of the areas identified, check to determine if it a valid CE via an area and T requirement
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ try:
+ loc = ndimage.find_objects(frame==(count+1))[0]
+ except Exception, e:
+ print "Error is ", e
+ continue
+
+
+ cloudElement = mergImgs[t,:,:][loc]
+ labels, lcounter = ndimage.label(cloudElement)
+
+ #determine the true lats and lons for this particular CE
+ cloudElementLat = LAT[loc[0],0]
+ cloudElementLon = LON[0,loc[1]]
+
+ #determine number of boxes in this cloudelement
+ numOfBoxes = np.count_nonzero(cloudElement)
+ cloudElementArea = numOfBoxes*XRES*YRES
+
+ #If the area is greater than the area required, then start considering as CE
+ #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
+ #if it does, still consider as CE
+
+ if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
+
+ #get some time information and labeling info
+ frameTime = str(timelist[t])
+ frameCEcounter +=1
+ CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
+
+ #-------------------------------------------------
+ #textfile name for accesing CE data using MATLAB code
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ # ------ NETCDF File stuff for brightness temp stuff ------------------------------------
+ thisFileName = MAINDIRECTORY +'/MERGnetcdfCEs/cloudElements'+ (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFCEData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFCEData.description = 'Cloud Element '+CEuniqueID + ' temperature data'
+ currNetCDFCEData.calendar = 'standard'
+ currNetCDFCEData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFCEData.createDimension('time', None)
+ currNetCDFCEData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFCEData.createDimension('lon', len(LON[0,:]))
+ # variables
+ tempDims = ('time','lat', 'lon',)
+ times = currNetCDFCEData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitudes = currNetCDFCEData.createVariable('latitude', 'f8', ('lat',))
+ longitudes = currNetCDFCEData.createVariable('longitude', 'f8', ('lon',))
+ brightnesstemp = currNetCDFCEData.createVariable('brightnesstemp', 'i16',tempDims )
+ brightnesstemp.units = 'Kelvin'
+ # NETCDF data
+ dates=[timelist[t]+timedelta(hours=0)]
+ times[:] = date2num(dates,units=times.units)
+ longitudes[:] = LON[0,:]
+ longitudes.units = "degrees_east"
+ longitudes.long_name = "Longitude"
+
+ latitudes[:] = LAT[:,0]
+ latitudes.units = "degrees_north"
+ latitudes.long_name ="Latitude"
+
+ #generate array of zeros for brightness temperature
+ brightnesstemp1 = ma.zeros((1,len(latitudes), len(longitudes))).astype('int16')
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #if other dataset (TRMM) assumed to be a precipitation dataset was entered
+ if TRMMdirName:
+ #------------------TRMM stuff -------------------------------------------------
+ fileDate = ((str(timelist[t])).replace(" ", "")[:-8]).replace("-","")
+ fileHr1 = (str(timelist[t])).replace(" ", "")[-8:-6]
+
+ if int(fileHr1) % temporalRes == 0:
+ fileHr = fileHr1
+ else:
+ fileHr = (int(fileHr1)/temporalRes) * temporalRes
+ if fileHr < 10:
+ fileHr = '0'+str(fileHr)
+ else:
+ str(fileHr)
+
+ #open TRMM file for the resolution info and to create the appropriate sized grid
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+str(fileHr)+".7A.nc"
+
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ nygrdTRMM = len(LATTRMM[:,0]); nxgrdTRMM = len(LONTRMM[0,:])
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ #get the lat/lon info from the file
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+ # ------ NETCDF File info for writing TRMM CE rainfall ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' precipitation data'
+ currNetCDFTRMMData.calendar = 'standard'
+ currNetCDFTRMMData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFTRMMData.createDimension('time', None)
+ currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
+
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value)
+
+ #generate the comma separated file for GIS
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ #temp data for CE NETCDF file
+ brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value
+
+ if TRMMdirName:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])]
+ CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]]))
+
+
+ brightnesstemp[:] = brightnesstemp1[:]
+ currNetCDFCEData.close()
+
+ if TRMMdirName:
+
+ #calculate the total precip associated with the feature
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ precipTotal += value
+ precip.append(value)
+
+ rainFallacc[:] = finalCETRMMvalues[:]
+ currNetCDFTRMMData.close()
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ try:
+ maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ pass
+
+ #sort cloudElementLatLons by lats
+ cloudElementLatLons.sort(key=lambda tup: tup[0])
+
+ #determine if the cloud element the shape
+ cloudElementEpsilon = eccentricity (cloudElement)
+ cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementCenter.append(latCenter)
+ cloudElementCenter.append(lonCenter)
+ cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsUserFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ #populate the dictionary
+ if TRMMdirName:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min, 'cloudElementPrecipTotal':precipTotal,'cloudElementLatLonTRMM':CETRMMList, 'TRMMArea': TRMMArea,'CETRMMmax':maxCEprecipRate, 'CETRMMmin':minCEprecipRate}
+ else:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min,}
+
+ #current frame list of CEs
+ currFrameCEs.append(cloudElementDict)
+
+ #draw the graph node
+ CLOUD_ELEMENT_GRAPH.add_node(CEuniqueID, cloudElementDict)
+
+ if frameNum != 1:
+ for cloudElementDict in prevFrameCEs:
+ thisCElen = len(cloudElementLatLons)
+ percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
+
+ #change weights to integers because the built in shortest path chokes on floating pts
+ #according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
+ if percentageOverlap >= 0.95:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
+
+ elif percentageOverlap >= 0.90 and percentageOverlap < 0.95 :
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[1])
+
+ elif areaOverlap >= MIN_OVERLAP:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
+
+ else:
+ #TODO: remove this else as we only wish for the CE details
+ #ensure only the non-zero elements are considered
+ #store intel in allCE file
+ labels, _ = ndimage.label(cloudElement)
+ cloudElementsFile.write("\n-----------------------------------------------")
+ cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ # cloudElementLat = LAT[loc[0],0]
+ # cloudElementLon = LON[0,loc[1]]
+
+ #populate cloudElementLatLons by unpacking the original values from loc
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ cloudElementsFile.write("\n-----------------------------------------------")
+
+ #reset list for the next CE
+ nodeExist = False
+ cloudElementCenter=[]
+ cloudElement = []
+ cloudElementCenter = []
+ cloudElementLat=[]
+ cloudElementLon =[]
+ cloudElementLatLons =[]
+ brightnesstemp1 =[]
+ brightnesstemp =[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ CETRMMList =[]
+ precipTotal = 0.0
+ precip=[]
+ TRMMCloudElementLatLons=[]
+
+ #reset for the next time
+ prevFrameCEs =[]
+ prevFrameCEs = currFrameCEs
+ currFrameCEs =[]
+
+ cloudElementsFile.close
+ cloudElementsUserFile.close
+ #if using ARCGIS data store code, uncomment this file close line
+ #cloudElementsTextFile.close
+
+ #clean up graph - remove parent and childless nodes
+ outAndInDeg = CLOUD_ELEMENT_GRAPH.degree_iter()
+ toRemove = [node[0] for node in outAndInDeg if node[1]<1]
+ CLOUD_ELEMENT_GRAPH.remove_nodes_from(toRemove)
+
+ print "number of nodes are: ", CLOUD_ELEMENT_GRAPH.number_of_nodes()
+ print "number of edges are: ", CLOUD_ELEMENT_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #hierachial graph output
+ #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
+ drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
+
+ return CLOUD_ELEMENT_GRAPH
+#******************************************************************
+def findPrecipRate(TRMMdirName):
+ '''
+ TODO: needs fixing
+ Purpose:: Determines the precipitation rates for MCSs found if TRMMdirName was not entered in findCloudElements this can be used
+
+ Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
+
+ Output:: a list of dictionary of the TRMM data
+ NB: also creates netCDF with TRMM data for each CE (for post processing) index
+ in MAINDIRECTORY/TRMMnetcdfCEs
+
+ Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
+
+ '''
+ allCEnodesTRMMdata =[]
+ TRMMdataDict={}
+ precipTotal = 0.0
+
+ os.chdir((MAINDIRECTORY+'/MERGnetcdfCEs/'))
+ imgFilename = ''
+ temporalRes = 3 #3 hours for TRMM
+
+ #sort files
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+
+ for afile in files:
+ #try:
+ print "in for ", afile
+ fullFname = os.path.splitext(afile)[0]
+ noFrameExtension = (fullFname.replace("_","")).split('F')[0]
+ CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
+ fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
+ fileDateTime = fileDateTimeChar.replace("-","")
+ fileDate = fileDateTime[:-6]
+ fileHr=fileDateTime[-6:-4]
+
+ cloudElementData = Dataset(afile,'r', format='NETCDF4')
+ brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
+ latsrawCloudElements = cloudElementData.variables['latitude'][:]
+ lonsrawCloudElements = cloudElementData.variables['longitude'][:]
+
+
+ if int(fileHr) % 3 == 0:
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
+ #print "TRMMfileName is ", TRMMfileName
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+
+
+ # ------ NETCDF File stuff ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
+ # print "thisFileName ", thisFileName
+ # sys.exit()
+ currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
+ currNetCDFData.calendar = 'standard'
+ currNetCDFData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFData.createDimension('time', None)
+ currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
+ currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+ #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
+ for index,value in np.ndenumerate(brightnesstemp):
+ #print "index, value ", index, value
+ time_index, lat_index, lon_index = index
+ currTimeValue = 0
+ if value > 0:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
+
+
+ # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
+ # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
+ # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
+ # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
+ # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
+ # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
+ # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
+ # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
+ # # precipTotal = value*temporalRes*1.0
+ # # prevLatValue = currLatvalue
+ # # prevLonValue = currLonValue
+
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ rainFallacc[:] = finalCETRMMvalues
+ currNetCDFData.close()
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ #print "index, value ", index, value
+ precipTotal += value
+
+ # #add info to the dictionary and to the list
+ # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
+ # allCEnodesTRMMdata.append(TRMMdataDict)
+ # print "precipTotal is ", precipTotal
+ #add info to CLOUDELEMENTSGRAPH
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
+ if eachdict[1]['uniqueID'] == CEuniqueID:
+ if not 'cloudElementPrecipTotal' in eachdict[1].keys():
+ eachdict[1]['cloudElementPrecipTotal'] = precipTotal
+ if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
+ eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
+ if not 'TRMMArea' in eachdict[1].keys():
+ eachdict[1]['TRMMArea'] = TRMMArea
+ #clean up
+ precipTotal = 0.0
+ latsrawTRMMData =[]
+ lonsrawTRMMData = []
+ latsrawCloudElements=[]
+ lonsrawCloudElements=[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ brightnesstemp =[]
+ TRMMdataDict ={}
+
+ return allCEnodesTRMMdata
+#******************************************************************
+def findCloudClusters(CEGraph):
+ '''
+ Purpose:: Determines the cloud clusters properties from the subgraphs in
+ the graph i.e. prunes the graph according to the minimum depth
+
+ Input:: CEGraph: 1 graph - a directed graph of the CEs with weighted edges
+ according the area overlap between nodes (CEs) of consectuive frames
+
+ Output:: PRUNED_GRAPH: 1 graph - a directed graph of with CCs/ MCSs
+
+ '''
+
+ seenNode = []
+ allMCSLists =[]
+ pathDictList =[]
+ pathList=[]
+
+ cloudClustersFile = open((MAINDIRECTORY+'/textFiles/cloudClusters.txt'),'wb')
+
+ for eachNode in CEGraph:
+ #check if the node has been seen before
+ if eachNode not in dict(enumerate(zip(*seenNode))):
+ #look for all trees associated with node as the root
+ thisPathDistanceAndLength = nx.single_source_dijkstra(CEGraph, eachNode)
+ #determine the actual shortestPath and minimum depth/length
+ maxDepthAndMinPath = findMaxDepthAndMinPath(thisPathDistanceAndLength)
+ if maxDepthAndMinPath:
+ maxPathLength = maxDepthAndMinPath[0]
+ shortestPath = maxDepthAndMinPath[1]
+
+ #add nodes and paths to PRUNED_GRAPH
+ for i in xrange(len(shortestPath)):
+ if PRUNED_GRAPH.has_node(shortestPath[i]) is False:
+ PRUNED_GRAPH.add_node(shortestPath[i])
+
+ #add edge if necessary
+ if i < (len(shortestPath)-1) and PRUNED_GRAPH.has_edge(shortestPath[i], shortestPath[i+1]) is False:
+ prunedGraphEdgeweight = CEGraph.get_edge_data(shortestPath[i], shortestPath[i+1])['weight']
+ PRUNED_GRAPH.add_edge(shortestPath[i], shortestPath[i+1], weight=prunedGraphEdgeweight)
+
+ #note information in a file for consideration later i.e. checking to see if it works
+ cloudClustersFile.write("\nSubtree pathlength is %d and path is %s" %(maxPathLength, shortestPath))
+ #update seenNode info
+ seenNode.append(shortestPath)
+
+ print "pruned graph"
+ print "number of nodes are: ", PRUNED_GRAPH.number_of_nodes()
+ print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
+ drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
+ cloudClustersFile.close
+
+ return PRUNED_GRAPH
+#******************************************************************
+def findMCC (prunedGraph):
+ '''
+ Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
+
+ Input:: prunedGraph: a Networkx Graph representing the CCs
+
+ Output:: finalMCCList: a list of list of tuples representing a MCC
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ MCCList = []
+ MCSList = []
+ definiteMCC = []
+ definiteMCS = []
+ eachList =[]
+ eachMCCList =[]
+ maturing = False
+ decaying = False
+ fNode = ''
+ lNode = ''
+ removeList =[]
+ imgCount = 0
+ imgTitle =''
+
+ maxShieldNode = ''
+ orderedPath =[]
+ treeTraversalList =[]
+ definiteMCCFlag = False
+ unDirGraph = nx.Graph()
+ aSubGraph = nx.DiGraph()
+ definiteMCSFlag = False
+
+
+ #connected_components is not available for DiGraph, so generate graph as undirected
+ unDirGraph = PRUNED_GRAPH.to_undirected()
+ subGraph = nx.connected_component_subgraphs(unDirGraph)
+
+ #for each path in the subgraphs determined
+ for path in subGraph:
+ #definite is a subTree provided the duration is longer than 3 hours
+
+ if len(path.nodes()) > MIN_MCS_DURATION:
+ orderedPath = path.nodes()
+ orderedPath.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ #definiteMCS.append(orderedPath)
+
+ #build back DiGraph for checking purposes/paper purposes
+ aSubGraph.add_nodes_from(path.nodes())
+ for eachNode in path.nodes():
+ if prunedGraph.predecessors(eachNode):
+ for node in prunedGraph.predecessors(eachNode):
+ aSubGraph.add_edge(node,eachNode,weight=edgeWeight[0])
+
+ if prunedGraph.successors(eachNode):
+ for node in prunedGraph.successors(eachNode):
+ aSubGraph.add_edge(eachNode,node,weight=edgeWeight[0])
+ imgTitle = 'CC'+str(imgCount+1)
+ drawGraph(aSubGraph, imgTitle, edgeWeight) #for eachNode in path:
+ imgCount +=1
+ #----------end build back ---------------------------------------------
+
+ mergeList, splitList = hasMergesOrSplits(path)
+ #add node behavior regarding neutral, merge, split or both
+ for node in path:
+ if node in mergeList and node in splitList:
+ addNodeBehaviorIdentifier(node,'B')
+ elif node in mergeList and not node in splitList:
+ addNodeBehaviorIdentifier(node,'M')
+ elif node in splitList and not node in mergeList:
+ addNodeBehaviorIdentifier(node,'S')
+ else:
+ addNodeBehaviorIdentifier(node,'N')
+
+
+ #Do the first part of checking for the MCC feature
+ #find the path
+ treeTraversalList = traverseTree(aSubGraph, orderedPath[0],[],[])
+ #print "treeTraversalList is ", treeTraversalList
+ #check the nodes to determine if a MCC on just the area criteria (consecutive nodes meeting the area and temp requirements)
+ MCCList = checkedNodesMCC(prunedGraph, treeTraversalList)
+ for aDict in MCCList:
+ for eachNode in aDict["fullMCSMCC"]:
+ addNodeMCSIdentifier(eachNode[0],eachNode[1])
+
+ #do check for if MCCs overlap
+ if MCCList:
+ if len(MCCList) > 1:
+ for count in range(len(MCCList)): #for eachDict in MCCList:
+ #if there are more than two lists
+ if count >= 1:
+ #and the first node in this list
+ eachList = list(x[0] for x in MCCList[count]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ fNode = eachList[0]
+ #get the lastNode in the previous possMCC list
+ eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ lNode = eachList[-1]
+ if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ if aNode in eachList and aNode == lNode:
+ #if edge_data is equal or less than to the exisitng edge in the tree append one to the other
+ if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']:
+ MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"])
+ MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"])
+ MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"]
+ MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
+ MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
+ MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"]
+ removeList.append(count)
+ #update the MCCList
+ if removeList:
+ for i in removeList:
+ if (len(MCCList)-1) > i:
+ del MCCList[i]
+ removeList =[]
+
+ #check if the nodes also meet the duration criteria and the shape crieria
+ for eachDict in MCCList:
+ #order the fullMCSMCC list, then run maximum extent and eccentricity criteria
+ if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION:
+ eachList = list(x[0] for x in eachDict["fullMCSMCC"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ eachMCCList = list(x[0] for x in eachDict["possMCCList"])
+ eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+
+ #update the nodemcsidentifer behavior
+ #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
+ #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
+ #ensure that everything between is listed as 'M'
+ for eachNode in eachList[:(eachList.index(eachMCCList[0]))]:
+ addNodeMCSIdentifier(eachNode,'I')
+
+ addNodeMCSIdentifier(eachMCCList[0],'M')
+
+ for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #update definiteMCS list
+ for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #run maximum extent and eccentricity criteria
+ maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList)
+ #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag
+ if definiteMCCFlag == True:
+ definiteMCC.append(eachList)
+
+
+ definiteMCS.append(orderedPath)
+
+ #reset for next subGraph
+ aSubGraph.clear()
+ orderedPath=[]
+ MCCList =[]
+ MCSList =[]
+ definiteMCSFlag = False
+
+ return definiteMCC, definiteMCS
+#******************************************************************
+def traverseTree(subGraph,node, queue, checkedNodes=None):
+ '''
+ Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+
+ Input:: subGraph: a Networkx DiGraph representing a CC
+ lengthOfsubGraph: an integer representing the length of the subgraph
+ node: a string representing the node currently being checked
+ queue: a list of strings representing a list of nodes in a stack functionality
+ i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ if len(checkedNodes) == len(subGraph):
+ return checkedNodes
+
+ if not checkedNodes:
+ queue =[]
+ checkedNodes.append(node)
+
+ #check one level infront first...if something does exisit, stick it at the front of the stack
+ upOneLevel = subGraph.predecessors(node)
+ downOneLevel = subGraph.successors(node)
+ for parent in upOneLevel:
+ if parent not in checkedNodes and parent not in queue:
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ queue.insert(0,child)
+ #downCheckFlag = True
+ queue.insert(0,parent)
+
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
+ queue.insert(0,child)
+ else:
+ queue.append(child)
+
+ #print "^^^^^stack ", stack
+ for eachNode in queue:
+ if eachNode not in checkedNodes:
+ #print "if eachNode ", checkedNodes, eachNode, stack
+ checkedNodes.append(eachNode)
+ return traverseTree(subGraph, eachNode, queue, checkedNodes)
+
+ return checkedNodes
+#******************************************************************
+def checkedNodesMCC (prunedGraph, nodeList):
+ '''
+ Purpose :: Determine if this path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ nodeList: list of strings (CE ID) from the traversal
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+ '''
+
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ INITIATIONFLAG = False
+ MATURITYFLAG = False
+ DECAYFLAG = False
+ thisdict = {} #will have the same items as the cloudElementDict
+ cloudElementArea = 0.0
+ epsilon = 0.0
+ frameNum =0
+ oldNode =''
+ potentialMCCList =[]
+ durationAandB = 0
+ CounterCriteriaA = 0
+ CountercriteriaB = 0
+
+ #check for if the list contains only one string/node
+ if type(nodeList) is str:
+ oldNode=nodeList
+ nodeList =[]
+ nodeList.append(oldNode)
+
+ for node in nodeList:
+ thisdict = thisDict(node)
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ existingFrameFlag = False
+
+ if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
+ #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaAFlag = True
+ INITIATIONFLAG = True
+ MATURITYFLAG = False
+ #check if criteriaB is meet
+ cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
+
+ #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
+ if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
+ #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaBFlag = True
+ #append this information on to the dictionary
+ addInfothisDict(node, cloudElementArea, criteriaB)
+ INITIATIONFLAG = False
+ MATURITYFLAG = True
+ stage = 'M'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria B failed
+ CounterCriteriaBFlag = False
+ if INITIATIONFLAG == True:
+ stage = 'I'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
+ DECAYFLAG = True
+ MATURITYFLAG = False
+ stage = 'D'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria A failed
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ #add as a CE before or after the main feature
+ if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
+ stage = "D"
+ DECAYFLAG = True
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ return potentialMCCList
+#******************************************************************
+def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
+ '''
+ Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ node: a string representing the cloud element currently being assessed
+ CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
+ CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+
+ '''
+ existingFrameFlag = False
+ existingMCSFrameFlag = False
+ predecessorsFlag = False
+ predecessorsMCSFlag = False
+ successorsFlag = False
+ successorsMCSFlag = False
+ frameNum = 0
+
+ frameNum = int((node.split('CE')[0]).split('F')[1])
+ if potentialMCCList==[]:
+ #list empty
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ else:
+ #list not empty
+ predecessorsFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,1)
+
+ if predecessorsFlag == True:
+
+ for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
+ existingFrameFlag = True
+
+ #this MUST come after the check for the existing frame
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+
+
+ if existingFrameFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage ='M'
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0:
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ if existingFrameFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ if predecessorsFlag == False:
+ successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+
+ if successorsFlag == True:
+ for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
+ existingFrameFlag = True
+
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if existingFrameFlag == False:
+ if stage == 'M':
+ stage = 'D'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ elif CounterCriteriaAFlag == True:
+ potentialMCCList[index]["CounterCriteriaA"] += 1
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ else:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to exisiting MCCs check if it is connected to exisiting MCSs ...
+ if predecessorsFlag == False and successorsFlag == False:
+ stage = 'I'
+ predecessorsMCSFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,2)
+ if predecessorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ else:
+ successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+ if successorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to existing MCCs or MCSs, create a new one ...
+ if predecessorsFlag == False and predecessorsMCSFlag == False and successorsFlag == False and successorsMCSFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ return potentialMCCList
+#******************************************************************
+def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
+ '''
+ Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
+
+ Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
+ upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
+ node: a string representing the cloud element currently being assessed
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
+
+ Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
+ index: an integer representing the location in the potentialMCCList where thisFlag occurs
+
+ '''
+ thisFlag = False
+ index = -1
+ checkList =""
+ if whichList == 1:
+ checkList = "possMCCList"
+ elif whichList ==2:
+ checkList = "fullMCSMCC"
+
+ #check parents
+ if upOrDown == 1:
+ for aNode in prunedGraph.predecessors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ #get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
+ return thisFlag, index
+
+ #check children
+ if upOrDown == 2:
+ for aNode in prunedGraph.successors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ return thisFlag, index
+
+ return thisFlag, index
+#******************************************************************
+def maxExtentAndEccentricity(eachList):
+ '''
+ Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
+
+ Input:: eachList: a list of strings representing the node of the possible MCCs within a path
+
+ Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
+ '''
+ maxShieldNode =''
+ maxShieldArea = 0.0
+ maxShieldEccentricity = 0.0
+ definiteMCCFlag = False
+
+ if eachList:
+ for eachNode in eachList:
+ if (thisDict(eachNode)['nodeMCSIdentifier'] == 'M' or thisDict(eachNode)['nodeMCSIdentifier'] == 'D') and thisDict(eachNode)['cloudElementArea'] > maxShieldArea:
+ maxShieldNode = eachNode
+ maxShieldArea = thisDict(eachNode)['cloudElementArea']
+
+ maxShieldEccentricity = thisDict(maxShieldNode)['cloudElementEccentricity']
+ if thisDict(maxShieldNode)['cloudElementEccentricity'] >= ECCENTRICITY_THRESHOLD_MIN and thisDict(maxShieldNode)['cloudElementEccentricity'] <= ECCENTRICITY_THRESHOLD_MAX :
+ #criteria met
+ definiteMCCFlag = True
+
+ return maxShieldNode, definiteMCCFlag
+#******************************************************************
+def findMaxDepthAndMinPath (thisPathDistanceAndLength):
+ '''
+ Purpose:: To determine the maximum depth and min path for the headnode
+
+ Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
+ thisPathDistanceAndLength({distance}, {path})
+ {distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
+
+ Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
+ minDistanceAndMaxPath = ({distance},{path})
+ '''
+ maxPathLength = 0
+ minPath = 0
+
+ #maxPathLength for the node in question
+ maxPathLength = max(len (values) for values in thisPathDistanceAndLength[1].values())
+
+ #if the duration is shorter then the min MCS length, then don't store!
+ if maxPathLength < MIN_MCS_DURATION: #MINIMUM_DURATION :
+ minDistanceAndMaxPath = ()
+
+ #else find the min path and max depth
+ else:
+ #max path distance for the node in question
+ minPath = max(values for values in thisPathDistanceAndLength[0].values())
+
+ #check to determine the shortest path from the longest paths returned
+ for pathDistance, path in itertools.izip(thisPathDistanceAndLength[0].values(), thisPathDistanceAndLength[1].values()):
+ pathLength = len(path)
+ #if pathLength is the same as the maxPathLength, then look the pathDistance to determine if the min
+ if pathLength == maxPathLength :
+ if pathDistance <= minPath:
+ minPath = pathLength
+ #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
+ minDistanceAndMaxPath = (pathDistance, path)
+ return minDistanceAndMaxPath
+#******************************************************************
+def thisDict (thisNode):
+ '''
+ Purpose:: return dictionary from graph if node exist in tree
+
+ Input:: String - thisNode
+
+ Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ return eachdict[1]
+#******************************************************************
+def checkCriteriaB (thisCloudElementLatLon):
+ '''
+ Purpose:: Determine if criteria B is met for a CEGraph
+
+ Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
+
+ Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
+
+ '''
+ cloudElementCriteriaBLatLon=[]
+
+ frame, CEcounter = ndimage.measurements.label(thisCloudElementLatLon, structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ #determine min and max values in lat and lon, then use this to generate teh array from LAT,LON meshgrid
+
+ minLat = min(x[0] for x in thisCloudElementLatLon)
+ maxLat = max(x[0]for x in thisCloudElementLatLon)
+ minLon = min(x[1]for x in thisCloudElementLatLon)
+ maxLon = max(x[1]for x in thisCloudElementLatLon)
+
+ #print "minLat, maxLat, minLon, maxLon ", minLat, maxLat, minLon, maxLon
+ minLatIndex = np.argmax(LAT[:,0] == minLat)
+ maxLatIndex = np.argmax(LAT[:,0]== maxLat)
+ minLonIndex = np.argmax(LON[0,:] == minLon)
+ maxLonIndex = np.argmax(LON[0,:] == maxLon)
+
+ #print "minLatIndex, maxLatIndex, minLonIndex, maxLonIndex ", minLatIndex, maxLatIndex, minLonIndex, maxLonIndex
+
+ criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
+
+ for x in thisCloudElementLatLon:
+ #to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
+ #index given in the original array to get the indices for the new array
+ #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
+ criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
+
+ #print criteriaBframe
+ #keep only those values < TBB_MAX
+ tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
+
+ #get the actual values that the mask returned
+ criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
+
+ for index, value in maenumerate(tempMask):
+ lat_index, lon_index = index
+ criteriaB[lat_index, lon_index]=value
+
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ #***** returns elements down then across thus (6,4) is 6 arrays deep of size 4
+ try:
+
+ loc = ndimage.find_objects(criteriaB)[0]
+ except:
+ #this would mean that no objects were found meeting criteria B
+ print "no objects at this temperature!"
+ cloudElementArea = 0.0
+ return cloudElementArea, cloudElementCriteriaBLatLon
+
+ print "after loc", loc
+ print "************************"
+ #print "criteriaB ", criteriaB.shape
+ print criteriaB
+ try:
+ cloudElementCriteriaB = ma.zeros((criteriaB.shape))
+ cloudElementCriteriaB =criteriaB[loc]
+ except:
+ print "YIKESS"
+ print "CEcounter ", CEcounter, criteriaB.shape
+ print "criteriaB ", criteriaB
+
+ for index,value in np.ndenumerate(cloudElementCriteriaB):
+ if value !=0:
+ t,lat,lon = index
+ #add back on the minLatIndex and minLonIndex to find the true lat, lon values
+ lat_lon_tuple = (LAT[(lat),0], LON[0,(lon)],value)
+ cloudElementCriteriaBLatLon.append(lat_lon_tuple)
+
+ cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
+ #do some
+ tempMask =[]
+ criteriaB =[]
+ cloudElementCriteriaB=[]
+
+ return cloudElementArea, cloudElementCriteriaBLatLon
+#******************************************************************
+def hasMergesOrSplits (nodeList):
+ '''
+ Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
+ Input:: nodeList - list of nodes from a path
+ Output:: two list_vars_in_file
+ splitList list of all the nodes in the path that split
+ mergeList list of all the nodes in the path that merged
+ '''
+ mergeList=[]
+ splitList=[]
+
+ for node,numParents in PRUNED_GRAPH.in_degree(nodeList).items():
+ if numParents > 1:
+ mergeList.append(node)
+
+ for node, numChildren in PRUNED_GRAPH.out_degree(nodeList).items():
+ if numChildren > 1:
+ splitList.append(node)
+ #sort
+ splitList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ mergeList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+
+ return mergeList,splitList
+#******************************************************************
+def allAncestors(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for parents
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its parents
+ integer - numOfChildren: the number of parents of the node passed
+ '''
+
+ numOfParents = PRUNED_GRAPH.in_degree(aNode)
+ try:
+ if PRUNED_GRAPH.predecessors(aNode) and numOfParents <= 1:
+ path = path + PRUNED_GRAPH.predecessors(aNode)
+ thisNode = PRUNED_GRAPH.predecessors(aNode)[0]
+ return allAncestors(path,thisNode)
+ else:
+ path = path+aNode
+ return path, numOfParents
+ except:
+ return path, numOfParents
+#******************************************************************
+def allDescendants(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for children
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its children
+ integer - numOfChildren: the number of children of the node passed
+ '''
+
+ numOfChildren = PRUNED_GRAPH.out_degree(aNode)
+ try:
+ if PRUNED_GRAPH.successors(aNode) and numOfChildren <= 1:
+ path = path + PRUNED_GRAPH.successors(aNode)
+ thisNode = PRUNED_GRAPH.successors(aNode)[0]
+ return allDescendants(path,thisNode)
+ else:
+ path = path + aNode
+ #i.e. PRUNED_GRAPH.predecessors(aNode) is empty
+ return path, numOfChildren
+ except:
+ #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
+ return path, numOfChildren
+#******************************************************************
+def addInfothisDict (thisNode, cloudElementArea,criteriaB):
+ '''
+ Purpose:: update original dictionary node with information
+
+ Input:: String - thisNode
+ float - cloudElementArea,
+ masked array of floats meeting the criteria - criteriaB
+
+ Output ::
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['CriteriaBArea'] = cloudElementArea
+ eachdict[1]['CriteriaBLatLon'] = criteriaB
+ return
+#******************************************************************
+def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeBehaviorIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeBehaviorIdentifier'] = nodeBehaviorIdentifier
+ return
+#******************************************************************
+def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeMCSIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeMCSIdentifier'] = nodeMCSIdentifier
+ return
+#******************************************************************
+def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['nodeMCSIdentifier'] = nodeBehaviorIdentifier
+
+ return
+#******************************************************************
+def eccentricity (cloudElementLatLon):
+ '''
+ Purpose::
+ Determines the eccentricity (shape) of contiguous boxes
+ Values tending to 1 are more circular by definition, whereas
+ values tending to 0 are more linear
+
+ Input::
+ 1 variable
+ cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
+
+ Output::
+ 1 variable
+ epsilon: a float representing the eccentricity of the matrix passed
+
+ '''
+
+ epsilon = 0.0
+
+ #loop over all lons and determine longest (non-zero) col
+ #loop over all lats and determine longest (non-zero) row
+ for latLon in cloudElementLatLon:
+ #assign a matrix to determine the legit values
+
+ nonEmptyLons = sum(sum(cloudElementLatLon)>0)
+ nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
+
+ lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
+ latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
+ epsilon = min(latEigenvalues,lonEigenvalues)
+
+ return epsilon
+#******************************************************************
+def cloudElementOverlap (currentCELatLons, previousCELatLons):
+ '''
+ Purpose::
+ Determines the percentage overlap between two list of lat-lons passed
+
+ Input::
+ 2 sorted list of tuples
+ currentCELatLons - the list of tuples for the current CE
+ previousCELatLons - the list of tuples for the other CE being considered
+
+ Output::
+ 2 variables
+ percentageOverlap - a float representing the number of overlapping lat_lon tuples
+ areaOverlap - a floating-point number representing the area overlapping
+
+ '''
+
+ latlonprev =[]
+ latloncurr = []
+ count = 0
+ percentageOverlap = 0.0
+ areaOverlap = 0.0
+
+ #remove the temperature from the tuples for currentCELatLons and previousCELatLons then check for overlap
+ latlonprev = [(x[0],x[1]) for x in previousCELatLons]
+ latloncurr = [(x[0],x[1]) for x in currentCELatLons]
+
+ #find overlap
+ count = len(list(set(latloncurr)&set(latlonprev)))
+
+ #find area overlap
+ areaOverlap = count*XRES*YRES
+
+ #find percentage
+ percentageOverlap = max(((count*1.0)/(len(latloncurr)*1.0)),((count*1.0)/(len(latlonprev)*1.0)))
+
+ return percentageOverlap, areaOverlap
+#******************************************************************
+def findCESpeed(node, MCSList):
+ '''
+ Purpose:: to determine the speed of the CEs
+ uses vector displacement delta_lat/delta_lon (y/x)
+
+ Input:: node: a string representing the CE
+ MCSList: a list of strings representing the feature
+
+ Output::
+
+ '''
+
+ delta_lon =0.0
+ delta_lat =0.0
+ CEspeed =[]
+ theSpeed = 0.0
+
+
+ theList = CLOUD_ELEMENT_GRAPH.successors(node)
+ nodeLatLon=thisDict(node)['cloudElementCenter']
+ #print "nodeLatLon ", nodeLatLon
+
+
+ for aNode in theList:
+ if aNode in MCSList:
+ #if aNode is part of the MCSList then determine distance
+ aNodeLatLon = thisDict(aNode)['cloudElementCenter']
+ #print "aNodeLatLon ", aNodeLatLon
+ #calculate CE speed
+ #checking the lats
+ nodeLatLon[0] += 90.0
+ aNodeLatLon[0] += 90.0
+ delta_lat = (nodeLatLon[0] - aNodeLatLon[0]) #convert to m
+ #delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
+ #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
+
+ #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
+ # if nodeLatLon[1] < 0.0:
+ # nodeLatLon[1] += 360.0
+ # if aNodeLatLon[1] <= 0.0:
+ # delta_lon = aNodeLatLon[1] + 360.0
+ #0E --> 360 and E lons > west lons
+ nodeLatLon[1] += 360.0
+ aNodeLatLon[1] += 360.0
+ delta_lon = (nodeLatLon[1] - aNodeLatLon[1]) #convert to m
+ #delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
+
+ #print "delta_lat, delta_lon ", delta_lat, delta_lon
+
+ #theSpeed = abs(((delta_lat*LAT_DISTANCE*1000)/(delta_lon*LON_DISTANCE*1000))/(TRES*3600)) #convert to s --> m/s
+ theSpeed = abs((((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600))) #convert to s --> m/s
+
+ CEspeed.append(theSpeed)
+ #print "aNode CE speed is ", aNode, (((delta_lat/delta_lon)*LAT_DISTANCE*1000)/(TRES*3600)), theSpeed
+
+ if not CEspeed:
+ return 0.0
+ else:
+ return min(CEspeed)
+#******************************************************************
+#
+# UTILITY SCRIPTS FOR MCCSEARCH.PY
+#
+#******************************************************************
+def maenumerate(mArray):
+ '''
+ Purpose::
+ Utility script for returning the actual values from the masked array
+ Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
+
+ Input::
+ 1 variable
+ mArray - the masked array returned from the ma.array() command
+
+
+ Output::
+ 1 variable
+ maskedValues - 3D (t,lat,lon), value of only masked values
+
+ '''
+
+ mask = ~mArray.mask.ravel()
+ #beware yield fast, but generates a type called "generate" that does not allow for array methods
+ for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
+ if maskedValue:
+ yield index
+#******************************************************************
+def createMainDirectory(mainDirStr):
+ '''
+ Purpose:: to create the main directory for storing information and
+ the subdirectories for storing information
+ Input:: mainDir: a directory for where all information generated from
+ the program are to be stored
+ Output:: None
+
+ '''
+ global MAINDIRECTORY
+
+ MAINDIRECTORY = mainDirStr
+ #if directory doesnt exist, creat it
+ if not os.path.exists(MAINDIRECTORY):
+ os.makedirs(MAINDIRECTORY)
+
+ os.chdir((MAINDIRECTORY))
+ #create the subdirectories
+ try:
+ os.makedirs('images')
+ os.makedirs('textFiles')
+ os.makedirs('MERGnetcdfCEs')
+ os.makedirs('TRMMnetcdfCEs')
+ except:
+ print "Directory exists already!!!"
+ #TODO: some nice way of prompting if it is ok to continue...or just leave
+
+ return
+#******************************************************************
+def find_nearest(thisArray,value):
+ '''
+ Purpose :: to determine the value within an array closes to
+ another value
+
+ Input ::
+ Output::
+ '''
+ idx = (np.abs(thisArray-value)).argmin()
+ return thisArray[idx]
+#******************************************************************
+def preprocessingMERG(MERGdirname):
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ Directory to the location of the raw MERG files, preferably zipped
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ os.chdir((MERGdirname+'/'))
+ imgFilename = ''
+
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ for files in glob.glob("*-pixel"):
+ #for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+ #TODO: must be something more efficient!
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+ #generate the lats4D command for GrADS
+ lats4D = 'lats4d -v -q -lat '+LATMIN + ' ' +LATMAX +' -lon ' +LONMIN +' ' +LONMAX +' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ #lats4D = 'lats4d -v -q -lat -40 -15 -lon 10 40 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+ #lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+ imgFilename = hr+'Z'+day+mth+yy+'.gif'
+ tempMaskedImages(imgFilename)
+
+ #when all the files have benn converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ #mv all images
+ subprocess.call('mkdir mergImgs', shell=True)
+ subprocess.call('mv *.gif mergImgs', shell=True)
+ return
+#******************************************************************
+def postProcessingNetCDF(dataset, dirName = None):
+ '''
+
+ TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
+
+ Purpose::
+ Utility script displaying the data in generated NETCDF4 files
+ in GrADS
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
+ string: Directory to the location of the raw (MERG) files, preferably zipped
+
+ Output::
+ images in location as specfied in the code
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ '''
+
+ coreDir = os.path.dirname(MAINDIRECTORY)
+ #coreDir = os.path.dirname(os.path.abspath(__file__))
+ ImgFilename = ''
+ frameList=[]
+ fileList =[]
+ lines =[]
+ var =''
+ firstTime = True
+ printLine = 0
+ lineNum = 1
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ print "coreDir is ", str(coreDir)
+ prevFrameNum = 0
+
+ if dataset == 1:
+ var = 'ch4'
+ dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
+ ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
+ ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ gsFile = coreDir+"/GrADSscripts/cs2.gs"
+ sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
+ lineNum = 50
+
+ elif dataset ==2:
+ var = 'precipAcc'
+ dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
+ ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
+ ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
+ origsFile = coreDir+"/GrADSscripts/cs3.gs"
+ gsFile = coreDir+"/GrADSscripts/cs4.gs"
+ sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
+ lineNum = 10
+
+ elif dataset ==3:
+ var = 'ch4'
+ ctlTitle = 'TITLE MERG DATA'
+ ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
+ if dirName is None:
+ print "Enter directory for original files"
+ return
+ else:
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ sologsFile = coreDir+"/GrADSscripts/infrared.gs"
+ lineNum = 54
+
+ #sort files
+ os.chdir((dirName+'/'))
+ try:
+ os.makedirs('ctlFiles')
+ except:
+ print "ctl file folder created already"
+
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+ for eachfile in files:
+ fullFname = os.path.splitext(eachfile)[0]
+ fnameNoExtension = fullFname.split('.nc')[0]
+ if dataset == 1 or dataset == 2:
+ frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
+ #create the ctlFile
+ ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
+ #the ctl file
+ subprocessCall = 'rm ' +ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ subprocessCall = 'touch '+ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ lineToWrite = 'echo DSET ' + dirName+'/'+fnameNoExtension+'.nc' +' >>' + ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo DTYPE netcdf >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo UNDEF 0 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo
<TRUNCATED>
[03/50] [abbrv] initial mccsearch code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/newPreprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/newPreprocessingMERG.py b/mccsearch/code/newPreprocessingMERG.py
new file mode 100755
index 0000000..cedc155
--- /dev/null
+++ b/mccsearch/code/newPreprocessingMERG.py
@@ -0,0 +1,175 @@
+
+'''
+ The scripts for processing MERG raw data
+ #******************************************************************
+ '''
+
+import fileinput
+import glob
+import os
+import re
+import string
+import subprocess
+import sys
+
+#----------------------- GLOBAL VARIABLES --------------------------
+# --------------------- User defined variables ---------------------
+LATMIN = '-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '20.0' #max latitude; -ve values in the SH e.g. 5S = -5
+LONMIN = '-30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
+LONMAX = '30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
+T_BB_MAX = '250' #warmest temp to allow in K
+T_BB_MIN = '180' #coolest temp to allow in K
+#------------------- End user defined Variables -------------------
+
+#------------------------ End GLOBAL VARS -------------------------
+
+def preprocessingMERG(MERGdirname):
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ Directory to the location of the raw MERG files, preferably zipped
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ os.chdir((MERGdirname+'/'))
+ imgFilename = ''
+
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ #for files in glob.glob("*-pixel"):
+ for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+ #TODO: must be something more pythonic!
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+ #generate the lats4D command for GrADS
+ lats4D = 'lats4d -v -q -lat ' + LATMIN +' ' + LATMAX + ' ' + '-lon ' + LONMIN + ' ' + LONMAX + ' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -lbc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+
+ #----- Generate GrADS images--------------
+ imgFilename = hr+'Z'+day+mth+yy+'.gif'
+ tempMaskedImages(imgFilename)
+ #-------End Generate GrADS images ---------
+
+ #remove the raw data file to save some resources
+ rmCmd= "rm " + fname
+ subprocess.call(rmCmd, shell=True)
+
+ #when all the files have been converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ #mv all images
+ subprocess.call('mkdir mergImgs', shell=True)
+ subprocess.call('mv *.gif mergImgs', shell=True)
+
+ return
+
+#******************************************************************
+def tempMaskedImages(imgFilename):
+ '''
+ Purpose:: To generate temperature-masked images for a first pass verification
+
+ Input::
+ filename for the gif file
+
+ Output::
+ None - gif images for each file of IR temperature < T_BB_MAX are generated in folder called mergImgs
+
+ Assumptions::
+ Same as for preprocessingMERG
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ subprocess.call('rm tempMaskedImages.gs', shell=True)
+ subprocess.call('touch tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'open merg.ctl''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set mpdset hires''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set lat ' +LATMIN + ' ' +LATMAX +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set lon ' +LONMIN + ' ' +LONMAX +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set cint 10''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set cmax '+T_BB_MAX +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set cmin '+T_BB_MIN +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'d ch4+75''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'draw title Masked Temp @ '+imgFilename +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'quit''\'" >> tempMaskedImages.gs', shell=True)
+ gradscmd = 'grads -blc ' + '\'run tempMaskedImages.gs''\''
+ subprocess.call(gradscmd, shell=True)
+ return
+
+#******************************************************************
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/preprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/preprocessingMERG.py b/mccsearch/code/preprocessingMERG.py
new file mode 100755
index 0000000..04964a8
--- /dev/null
+++ b/mccsearch/code/preprocessingMERG.py
@@ -0,0 +1,125 @@
+'''
+#preprocessing for MERG data
+# unzips files and generates the NETCDF files using GrADS lats4D
+# http://opengrads.org/doc/scripts/lats4d/
+# This script requires the merg_template.ctl resides with the folder
+# TODO: generate this file
+# Kim Whitehall
+# Last updated: June 4th 2013
+'''
+import subprocess
+import fileinput
+import json
+import os
+import glob
+import re
+import sys
+
+
+def preprocessing_merg():
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format
+ NOTE: VERY RAW AND DIRTY
+ Input::
+ none
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+ os.chdir("/Users/whitehal/Documents/kimSum2013/mergData1/")
+ searchExpDset = "DSET "
+ searchExpTdef = "TDEF "
+ #for files in glob.glob("*-pixel"):
+ for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+
+
+ # #cp template merg.ctl
+ # subprocess.call('cp -f merg_template.ctl merg.ctl', shell=True)
+
+ # #edit merg.ctl dset line to contain filename by running file
+ # #http://stackoverflow.com/questions/39086/search-and-replace-a-line-in-a-file-in-python
+ # replaceExpDset = 'DSET ' + fname
+ # replaceExpTdef = 'TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn'
+ # for line in fileinput.input("merg.ctl", inplace=True):
+ # if searchExpDset in line:
+ # line = line.replace(searchExpDset, replaceExpDset)
+ # if searchExpTdef in line:
+ # line = line.replace(searchExpTdef, replaceExpTdef)
+ # sys.stdout.write(line)
+
+ #lats4d cmd
+ lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+
+ #when all the files have benn converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ return
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/process.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.py b/mccsearch/code/process.py
new file mode 100644
index 0000000..5a362ec
--- /dev/null
+++ b/mccsearch/code/process.py
@@ -0,0 +1,1284 @@
+"""Collection of functions that process numpy arrays of varying shapes.
+Functions can range from subsetting to regridding"""
+
+# Python Standard Libary Imports
+import math
+import datetime
+import re
+import string
+import sys
+import time
+
+# 3rd Party Imports
+import Nio
+import numpy as np
+import numpy.ma as ma
+from scipy.ndimage import map_coordinates
+
+
+def extract_subregion_from_data_array(data, lats, lons, latmin, latmax, lonmin, lonmax):
+ '''
+ Extract a sub-region from a data array.
+
+ Example::
+ **e.g.** the user may load a global model file, but only want to examine data over North America
+ This function extracts a sub-domain from the original data.
+ The defined sub-region must be a regular lat/lon bounding box,
+ but the model data may be on a non-regular grid (e.g. rotated, or Guassian grid layout).
+ Data are kept on the original model grid and a rectangular (in model-space) region
+ is extracted which contains the rectangular (in lat/lon space) user supplied region.
+
+ INPUT::
+ data - 3d masked data array
+ lats - 2d array of latitudes corresponding to data array
+ lons - 2d array of longitudes corresponding to data array
+ latmin, latmax, lonmin, lonmax - bounding box of required region to extract
+
+ OUTPUT::
+ data2 - subset of original data array containing only data from required subregion
+ lats2 - subset of original latitude data array
+ lons2 - subset of original longitude data array
+
+ '''
+
+
+
+ # Mask model data array to find grid points inside the supplied bounding box
+ whlat = (lats > latmin) & (lats < latmax)
+ whlon = (lons > lonmin) & (lons < lonmax)
+ wh = whlat & whlon
+
+ # Find required matrix indices describing the limits of the regular lat/lon bounding box
+ jmax = np.where(wh == True)[0].max()
+ jmin = np.where(wh == True)[0].min()
+ imax = np.where(wh == True)[1].max()
+ imin = np.where(wh == True)[1].min()
+
+ # Cut out the sub-region from the data array
+ data2 = ma.zeros((data.shape[0], jmax - jmin, imax - imin))
+ data2 = data[:, jmin:jmax, imin:imax]
+
+ # Cut out sub-region from lats,lons arrays
+ lats2 = lats[jmin:jmax, imin:imax]
+ lons2 = lons[jmin:jmax, imin:imax]
+
+ return data2, lats2, lons2
+
+def calc_area_mean(data, lats, lons, mymask='not set'):
+ '''
+ Calculate Area Average of data in a masked array
+
+ INPUT::
+ data: a masked array of data (NB. only data from one time expected to be passed at once)
+ lats: 2d array of regularly gridded latitudes
+ lons: 2d array of regularly gridded longitudes
+ mymask: (optional) defines spatial region to do averaging over
+
+ OUTPUT::
+ area_mean: a value for the mean inside the area
+
+ '''
+
+
+
+ # If mask not passed in, then set maks to cover whole data domain
+ if(mymask == 'not set'):
+ mymask = np.empty(data.shape)
+ mymask[:] = False # NB. mask means (don't show), so False everywhere means use everything.
+
+ # Dimension check on lats, lons
+ # Sometimes these arrays are 3d, sometimes 2d, sometimes 1d
+ # This bit of code just converts to the required 2d array shape
+ if(len(lats.shape) == 3):
+ lats = lats[0, :, :]
+
+ if(len(lons.shape) == 3):
+ lons = lons[0, :, :]
+
+ if(np.logical_and(len(lats.shape) == 1, len(lons.shape) == 1)):
+ lons, lats = np.meshgrid(lons, lats)
+
+ # Calculate grid length (assuming regular lat/lon grid)
+ dlat = lats[1, 0] - lats[0, 0]
+ dlon = lons[0, 1] - lons[0, 0]
+
+ # Calculates weights for each grid box
+ myweights = calc_area_in_grid_box(lats, dlon, dlat)
+
+ # Create a new masked array covering just user selected area (defined in mymask)
+ # NB. this preserves missing data points in the observations data
+ subdata = ma.masked_array(data, mask=mymask)
+
+ if(myweights.shape != subdata.shape):
+ myweights.resize(subdata.shape)
+ myweights[1:, :] = myweights[0, :]
+
+ # Calculate weighted mean using ma.average (which takes weights)
+ area_mean = ma.average(subdata, weights=myweights)
+
+ return area_mean
+
+def calc_area_in_grid_box(latitude, dlat, dlon):
+ '''
+ Calculate area of regular lat-lon grid box
+
+ INPUT::
+ latitude: latitude of grid box (degrees)
+ dlat: grid length in latitude direction (degrees)
+ dlon: grid length in longitude direction (degrees)
+
+ OUTPUT::
+ A: area of the grid box
+
+ '''
+
+ R = 6371000 # radius of Earth in metres
+ C = 2 * math.pi * R
+
+ latitude = np.radians(latitude)
+
+ A = (dlon * (C / 360.)) * (dlat * (C / 360.) * np.cos(latitude))
+
+ return A
+
+def do_regrid(q, lat, lon, lat2, lon2, order=1, mdi= -999999999):
+ '''
+ Perform regridding from one set of lat,lon values onto a new set (lat2,lon2)
+
+ Input::
+ q - the variable to be regridded
+ lat,lon - original co-ordinates corresponding to q values
+ lat2,lon2 - new set of latitudes and longitudes that you want to regrid q onto
+ order - (optional) interpolation order 1=bi-linear, 3=cubic spline
+ mdi - (optional) fill value for missing data (used in creation of masked array)
+
+ Output::
+ q2 - q regridded onto the new set of lat2,lon2
+
+ '''
+
+ nlat = q.shape[0]
+ nlon = q.shape[1]
+
+ nlat2 = lat2.shape[0]
+ nlon2 = lon2.shape[1]
+
+ # To make our lives easier down the road, let's
+ # turn these into arrays of x & y coords
+ loni = lon2.ravel()
+ lati = lat2.ravel()
+
+ loni = loni.copy() # NB. it won't run unless you do this...
+ lati = lati.copy()
+
+ # Now, we'll set points outside the boundaries to lie along an edge
+ loni[loni > lon.max()] = lon.max()
+ loni[loni < lon.min()] = lon.min()
+
+ # To deal with the "hard" break, we'll have to treat y differently,
+ # so we're just setting the min here...
+ lati[lati > lat.max()] = lat.max()
+ lati[lati < lat.min()] = lat.min()
+
+
+ # We need to convert these to (float) indicies
+ # (xi should range from 0 to (nx - 1), etc)
+ loni = (nlon - 1) * (loni - lon.min()) / (lon.max() - lon.min())
+
+ # Deal with the "hard" break in the y-direction
+ lati = (nlat - 1) * (lati - lat.min()) / (lat.max() - lat.min())
+
+ # Notes on dealing with MDI when regridding data.
+ # Method adopted here:
+ # Use bilinear interpolation of data by default (but user can specify other order using order=... in call)
+ # Perform bilinear interpolation of data, and of mask.
+ # To be conservative, new grid point which contained some missing data on the old grid is set to missing data.
+ # -this is achieved by looking for any non-zero interpolated mask values.
+ # To avoid issues with bilinear interpolation producing strong gradients leading into the MDI,
+ # set values at MDI points to mean data value so little gradient visible = not ideal, but acceptable for now.
+
+ # Set values in MDI so that similar to surroundings so don't produce large gradients when interpolating
+ # Preserve MDI mask, by only changing data part of masked array object.
+ for shift in (-1, 1):
+ for axis in (0, 1):
+ q_shifted = np.roll(q, shift=shift, axis=axis)
+ idx = ~q_shifted.mask * q.mask
+ q.data[idx] = q_shifted[idx]
+
+ # Now we actually interpolate
+ # map_coordinates does cubic interpolation by default,
+ # use "order=1" to preform bilinear interpolation instead...
+ q2 = map_coordinates(q, [lati, loni], order=order)
+ q2 = q2.reshape([nlat2, nlon2])
+
+ # Set values to missing data outside of original domain
+ q2 = ma.masked_array(q2, mask=np.logical_or(np.logical_or(lat2 >= lat.max(),
+ lat2 <= lat.min()),
+ np.logical_or(lon2 <= lon.min(),
+ lon2 >= lon.max())))
+
+ # Make second map using nearest neighbour interpolation -use this to determine locations with MDI and mask these
+ qmdi = np.zeros_like(q)
+ qmdi[q.mask == True] = 1.
+ qmdi[q.mask == False] = 0.
+ qmdi_r = map_coordinates(qmdi, [lati, loni], order=order)
+ qmdi_r = qmdi_r.reshape([nlat2, nlon2])
+ mdimask = (qmdi_r != 0.0)
+
+ # Combine missing data mask, with outside domain mask define above.
+ q2.mask = np.logical_or(mdimask, q2.mask)
+
+ return q2
+
+def create_mask_using_threshold(masked_array, threshold=0.5):
+ '''
+ Routine to create a mask, depending on the proportion of times with missing data.
+ For each pixel, calculate proportion of times that are missing data,
+ **if** the proportion is above a specified threshold value,
+ then mark the pixel as missing data.
+
+ Input::
+ masked_array - a numpy masked array of data (assumes time on axis 0, and space on axes 1 and 2).
+ threshold - (optional) threshold proportion above which a pixel is marked as 'missing data'.
+ NB. default threshold = 50%
+
+ Output::
+ mymask - a numpy array describing the mask. NB. not a masked array, just the mask itself.
+
+ '''
+
+
+ # try, except used as some model files don't have a full mask, but a single bool
+ # the except catches this situation and deals with it appropriately.
+ try:
+ nT = masked_array.mask.shape[0]
+
+ # For each pixel, count how many times are masked.
+ nMasked = masked_array.mask[:, :, :].sum(axis=0)
+
+ # Define new mask as when a pixel has over a defined threshold ratio of masked data
+ # e.g. if the threshold is 75%, and there are 10 times,
+ # then a pixel will be masked if more than 5 times are masked.
+ mymask = nMasked > (nT * threshold)
+
+ except:
+ mymask = np.zeros_like(masked_array.data[0, :, :])
+
+ return mymask
+
+def calc_average_on_new_time_unit(data, dateList, unit='monthly'):
+ '''
+ Routine to calculate averages on longer time units than the data exists on.
+
+ Example::
+ If the data is 6-hourly, calculate daily, or monthly means.
+
+ Input::
+ data - data values
+ dateList - list of python datetime structures corresponding to data values
+ unit - string describing time unit to average onto.
+ *Valid values are: 'monthly', 'daily', 'pentad','annual','decadal'*
+
+ Output:
+ meanstorem - numpy masked array of data values meaned over required time period
+ newTimesList - a list of python datetime objects representing the data in the new averagin period
+ *NB.* currently set to beginning of averaging period,
+ **i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.**
+ '''
+
+ # First catch unknown values of time unit passed in by user
+ acceptable = (unit == 'full') | (unit == 'annual') | (unit == 'monthly') | (unit == 'daily') | (unit == 'pentad')
+
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging'
+ print ' Please check your code.'
+ return
+
+ # Calculate arrays of years (2007,2007),
+ # yearsmonths (200701,200702),
+ # or yearmonthdays (20070101,20070102)
+ # -depending on user selected averaging period.
+
+ # Year list
+ if unit == 'annual':
+ print 'Calculating annual mean data'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+
+ timeunits.append(str(dateList[i].year))
+
+ timeunits = np.array(timeunits, dtype=int)
+
+ # YearMonth format list
+ if unit == 'monthly':
+ print 'Calculating monthly mean data'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+
+ timeunits = np.array(timeunits, dtype=int)
+
+ # YearMonthDay format list
+ if unit == 'daily':
+ print 'Calculating daily mean data'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + \
+ str("%02d" % dateList[i].day))
+
+ timeunits = np.array(timeunits, dtype=int)
+
+
+ # TODO: add pentad setting using Julian days?
+
+
+ # Full list: a special case
+ if unit == 'full':
+ print 'Calculating means data over full time range'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
+
+ timeunits = np.array(timeunits, dtype=int)
+
+
+
+ # empty list to store new times
+ newTimesList = []
+
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and
+ # calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits) == (len(np.unique(timeunits))):
+ processing_required = False
+
+ # 1D data arrays, i.e. time series
+ if data.ndim == 1:
+ # Create array to store the resulting data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+
+ # Calculate the means across each unique time unit
+ i = 0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ datam = ma.masked_array(data, timeunits != myunit)
+ meanstore[i] = ma.average(datam)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+ i = i + 1
+
+ # 3D data arrays
+ if data.ndim == 3:
+
+ #datamask = create_mask_using_threshold(data,threshold=0.75)
+
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)), data.shape[1], data.shape[2]])
+
+ # Calculate the means across each unique time unit
+ i = 0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
+
+ mask = np.zeros_like(data)
+ mask[timeunits != myunit, :, :] = 1.0
+
+ # Calculate missing data mask within each time unit...
+ datamask_at_this_timeunit = np.zeros_like(data)
+ datamask_at_this_timeunit[:] = create_mask_using_threshold(data[timeunits == myunit, :, :], threshold=0.75)
+ # Store results for masking later
+ datamask_store.append(datamask_at_this_timeunit[0])
+
+ # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
+ datam = ma.masked_array(data, np.logical_or(mask, datamask_at_this_timeunit))
+ meanstore[i, :, :] = ma.average(datam, axis=0)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+
+ if processing_required:
+ # Create masked array (using missing data mask defined above)
+ datamask_store = np.array(datamask_store)
+ meanstorem = ma.masked_array(meanstore, datamask_store)
+
+ return meanstorem, newTimesList
+
+def calc_running_accum_from_period_accum(data):
+ '''
+ Routine to calculate running total accumulations from individual period accumulations.
+ ::
+
+ e.g. 0,0,1,0,0,2,2,1,0,0
+ -> 0,0,1,1,1,3,5,6,6,6
+
+ Input::
+ data: numpy array with time in the first axis
+
+ Output::
+ running_acc: running accumulations
+
+ '''
+
+
+ running_acc = np.zeros_like(data)
+
+ if(len(data.shape) == 1):
+ running_acc[0] = data[0]
+
+ if(len(data.shape) > 1):
+ running_acc[0, :] = data[0, :]
+
+ for i in np.arange(data.shape[0] - 1):
+ if(len(data.shape) == 1):
+ running_acc[i + 1] = running_acc[i] + data[i + 1]
+
+ if(len(data.shape) > 1):
+ running_acc[i + 1, :] = running_acc[i, :] + data[i + 1, :]
+
+ return running_acc
+
+def ignore_boundaries(data, rim=10):
+ '''
+ Routine to mask the lateral boundary regions of model data to ignore them from calculations.
+
+ Input::
+ data - a masked array of model data
+ rim - (optional) number of grid points to ignore
+
+ Output::
+ data - data array with boundary region masked
+
+ '''
+
+ nx = data.shape[1]
+ ny = data.shape[0]
+
+ rimmask = np.zeros_like(data)
+ for j in np.arange(rim):
+ rimmask[j, 0:nx - 1] = 1.0
+
+ for j in ny - 1 - np.arange(rim):
+ rimmask[j, 0:nx - 1] = 1.0
+
+ for i in np.arange(rim):
+ rimmask[0:ny - 1, i] = 1.0
+
+ for i in nx - 1 - np.arange(rim):
+ rimmask[0:ny - 1, i] = 1.0
+
+ data = ma.masked_array(data, mask=rimmask)
+
+ return data
+
+def normalizeDatetimes(datetimes, timestep):
+ """
+ Input::
+ datetimes - list of datetime objects that need to be normalized
+ timestep - string of value ('daily' | 'monthly')
+ Output::
+ normalDatetimes - list of datetime objects that have been normalized
+
+ Normalization Rules::
+ Daily data will be forced to an hour value of 00:00:00
+ Monthly data will be forced to the first of the month at midnight
+ """
+ normalDatetimes = []
+ if timestep.lower() == 'monthly':
+ for inputDatetime in datetimes:
+ if inputDatetime.day != 1:
+ # Clean the inputDatetime
+ inputDatetimeString = inputDatetime.strftime('%Y%m%d')
+ normalInputDatetimeString = inputDatetimeString[:6] + '01'
+ inputDatetime = datetime.datetime.strptime(normalInputDatetimeString, '%Y%m%d')
+
+ normalDatetimes.append(inputDatetime)
+
+ elif timestep.lower() == 'daily':
+ for inputDatetime in datetimes:
+ if inputDatetime.hour != 0 or inputDatetime.minute != 0 or inputDatetime.second != 0:
+ datetimeString = inputDatetime.strftime('%Y%m%d%H%M%S')
+ normalDatetimeString = datetimeString[:8] + '000000'
+ inputDatetime = datetime.datetime.strptime(normalDatetimeString, '%Y%m%d%H%M%S')
+
+ normalDatetimes.append(inputDatetime)
+
+ return normalDatetimes
+
+def getModelTimes(modelFile, timeVarName):
+ '''
+ TODO: Do a better job handling dates here
+ Routine to convert from model times ('hours since 1900...', 'days since ...')
+ into a python datetime structure
+
+ Input::
+ modelFile - path to the model tile you want to extract the times list and modelTimeStep from
+ timeVarName - name of the time variable in the model file
+
+ Output::
+ times - list of python datetime objects describing model data times
+ modelTimeStep - 'hourly','daily','monthly','annual'
+ '''
+
+ f = Nio.open_file(modelFile)
+ xtimes = f.variables[timeVarName]
+ timeFormat = xtimes.attributes['units']
+
+ # search to check if 'since' appears in units
+ try:
+ sinceLoc = re.search('since', timeFormat).end()
+
+
+ #KDW the below block generates and error. But the print statement, indicates that sinceLoc found something
+ except AttributeError:
+ print 'Error decoding model times: time variable attributes do not contain "since"'
+ raise
+
+ units = ''
+ TIME_UNITS = ('minutes', 'hours', 'days', 'months', 'years')
+ # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
+ for unit in TIME_UNITS:
+ if re.search(unit, timeFormat):
+ units = unit
+ break
+
+ # cut out base time (the bit following 'since')
+ base_time_string = string.lstrip(timeFormat[sinceLoc:])
+
+ # decode base time
+ base_time = decodeTimeFromString(base_time_string)
+
+ times = []
+
+
+
+ for xtime in xtimes[:]:
+
+ # Cast time as an int ***KDW remove this so fractional xtime can be read from MERG
+ xtime = int(xtime)
+
+ if units == 'minutes':
+ dt = datetime.timedelta(minutes=xtime)
+ new_time = base_time + dt
+ elif units == 'hours':
+ dt = datetime.timedelta(hours=xtime)
+ new_time = base_time + dt
+ elif units == 'days':
+ dt = datetime.timedelta(days=xtime)
+ new_time = base_time + dt
+ elif units == 'months':
+ # NB. adding months in python is complicated as month length varies and hence ambiguous.
+ # Perform date arithmatic manually
+ # Assumption: the base_date will usually be the first of the month
+ # NB. this method will fail if the base time is on the 29th or higher day of month
+ # -as can't have, e.g. Feb 31st.
+ new_month = int(base_time.month + xtime % 12)
+ new_year = int(math.floor(base_time.year + xtime / 12.))
+ new_time = datetime.datetime(new_year, new_month, base_time.day, base_time.hour, base_time.second, 0)
+ elif units == 'years':
+ dt = datetime.timedelta(years=xtime)
+ new_time = base_time + dt
+
+ #print "xtime is:", xtime, "dt is: ", dt
+
+
+
+ times.append(new_time)
+
+ try:
+ timeStepLength = int(xtimes[1] - xtimes[0] + 1.e-12)
+ modelTimeStep = getModelTimeStep(units, timeStepLength)
+
+ #KDW if timeStepLength is zero do not normalize times as this would create an empty list
+ if timeStepLength != 0:
+ times = normalizeDatetimes(times, modelTimeStep)
+ except:
+ raise
+
+ return times, modelTimeStep
+
+def getModelTimeStep(units, stepSize):
+ # Time units are now determined. Determine the time intervals of input data (mdlTimeStep)
+
+ if units == 'minutes':
+ if stepSize == 60:
+ modelTimeStep = 'hourly'
+ elif stepSize == 1440:
+ modelTimeStep = 'daily'
+ # 28 days through 31 days
+ elif 40320 <= stepSize <= 44640:
+ modelTimeStep = 'monthly'
+ # 365 days through 366 days
+ elif 525600 <= stepSize <= 527040:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'hours':
+ #need a check for fractional hrs and only one hr i.e. stepSize=0
+ if stepSize == 0 or stepSize == 1:
+ modelTimeStep = 'hourly'
+ elif stepSize == 24:
+ modelTimeStep = 'daily'
+ elif 672 <= stepSize <= 744:
+ modelTimeStep = 'monthly'
+ elif 8760 <= stepSize <= 8784:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'days':
+ if stepSize == 1:
+ modelTimeStep = 'daily'
+ elif 28 <= stepSize <= 31:
+ modelTimeStep = 'monthly'
+ elif 365 <= stepSize <= 366:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'months':
+ if stepSize == 1:
+ modelTimeStep = 'monthly'
+ elif stepSize == 12:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'years':
+ if stepSize == 1:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ else:
+ errorMessage = 'the time unit ', units, ' is not currently handled in this version.'
+ raise Exception(errorMessage)
+
+ return modelTimeStep
+
+def decodeTimeFromString(time_string):
+ '''
+ Decodes string into a python datetime object
+ *Method:* tries a bunch of different time format possibilities and hopefully one of them will hit.
+ ::
+
+ **Input:** time_string - a string that represents a date/time
+
+ **Output:** mytime - a python datetime object
+ '''
+
+ # This will deal with times that use decimal seconds
+ if '.' in time_string:
+ time_string = time_string.split('.')[0] + '0'
+ else:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y/%m/%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y%m%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y:%m:%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y%m%d%H%M%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d %H:%M')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+#KDW added
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d %H')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+
+ print 'Error decoding time string: string does not match a predefined time format'
+ return 0
+
+def regrid_wrapper(regrid_choice, obsdata, obslats, obslons, wrfdata, wrflats, wrflons):
+ '''
+ Wrapper routine for regridding.
+
+ Either regrids model to obs grid, or obs to model grid, depending on user choice
+
+ Inputs::
+ regrid_choice - [0] = Regrid obs data onto model grid or
+ [1] = Regrid model data onto obs grid
+
+ obsdata,wrfdata - data arrays
+ obslats,obslons - observation lat,lon arrays
+ wrflats,wrflons - model lat,lon arrays
+
+ Output::
+ rdata,rdata2 - regridded data
+ lats,lons - latitudes and longitudes of regridded data
+
+ '''
+
+ # Regrid obs data onto model grid
+ if(regrid_choice == '0'):
+
+ ndims = len(obsdata.shape)
+ if(ndims == 3):
+ newshape = [obsdata.shape[0], wrfdata.shape[1], wrfdata.shape[2]]
+ nT = obsdata.shape[0]
+
+ if(ndims == 2):
+ newshape = [wrfdata.shape[0], wrfdata.shape[1]]
+ nT = 0
+
+ rdata = wrfdata
+ lats, lons = wrflats, wrflons
+
+ # Create a new masked array with the required dimensions
+ tmp = np.zeros(newshape)
+ rdata2 = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
+
+ rdata2[:] = 0.0
+
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata2[t, :, :] = do_regrid(obsdata[t, :, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+
+ if(nT == 0):
+ rdata2[:, :] = do_regrid(obsdata[:, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+
+
+ # Regrid model data onto obs grid
+ if(regrid_choice == '1'):
+ ndims = len(wrfdata.shape)
+ if(ndims == 3):
+ newshape = [wrfdata.shape[0], obsdata.shape[1], obsdata.shape[2]]
+ nT = obsdata.shape[0]
+
+ if(ndims == 2):
+ newshape = [obsdata.shape[0], obsdata.shape[1]]
+ nT = 0
+
+ rdata2 = obsdata
+ lats, lons = obslats, obslons
+
+ tmp = np.zeros(newshape)
+ rdata = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
+
+ rdata[:] = 0.0
+
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata[t, :, :] = do_regrid(wrfdata[t, :, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+
+ if(nT == 0):
+ rdata[:, :] = do_regrid(wrfdata[:, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+
+
+ return rdata, rdata2, lats, lons
+
+def extract_sub_time_selection(allTimes, subTimes, data):
+ '''
+ Routine to extract a sub-selection of times from a data array.
+
+ Example::
+ Suppose a data array has 30 time values for daily data for a whole month,
+ but you only want the data from the 5th-15th of the month.
+
+ Input::
+ allTimes - list of datetimes describing what times the data in the data array correspond to
+ subTimes - the times you want to extract data from
+ data - the data array
+
+ Output::
+ subdata - subselection of data array
+
+ '''
+ # Create new array to store the subselection
+ subdata = np.zeros([len(subTimes), data.shape[1], data.shape[2]])
+
+ # Loop over all Times and when it is a member of the required subselection, copy the data across
+ i = 0 # counter for allTimes
+ subi = 0 # counter for subTimes
+ for t in allTimes:
+ if(np.setmember1d(allTimes, subTimes)):
+ subdata[subi, :, :] = data[i, :, :]
+ subi += 1
+ i += 1
+
+ return subdata
+
+def calc_average_on_new_time_unit_K(data, dateList, unit):
+ '''
+ Routine to calculate averages on longer time units than the data exists on.
+ e.g. if the data is 6-hourly, calculate daily, or monthly means.
+
+ Input:
+ data - data values
+ dateList - list of python datetime structures corresponding to data values
+ unit - string describing time unit to average onto
+ e.g. 'monthly', 'daily', 'pentad','annual','decadal'
+ Output:
+ meanstorem - numpy masked array of data values meaned over required time period
+ newTimesList - a list of python datetime objects representing the data in the new averagin period
+ NB. currently set to beginning of averaging period,
+ i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
+ '''
+
+ # Check if the user-selected temporal grid is valid. If not, EXIT
+ acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging: EXIT'
+ return -1,-1,-1,-1
+
+ # Calculate arrays of: annual timeseries: year (2007,2007),
+ # monthly time series: year-month (200701,200702),
+ # daily timeseries: year-month-day (20070101,20070102)
+ # depending on user-selected averaging period.
+
+ # Year list
+ if unit=='annual':
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year))
+ timeunits = np.array(timeunits, dtype=int)
+
+ # YearMonth format list
+ if unit=='monthly':
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+ timeunits = np.array(timeunits,dtype=int)
+
+ # YearMonthDay format list
+ if unit=='daily':
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
+ timeunits = np.array(timeunits,dtype=int)
+
+ # TODO: add pentad setting using Julian days?
+
+ # Full list: a special case
+ if unit == 'full':
+ comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
+ timeunits = np.array(timeunits, dtype=int)
+
+ # empty list to store new times
+ newTimesList = []
+
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and
+ # calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits)==(len(np.unique(timeunits))):
+ processing_required = False
+
+ # 1D data arrays, i.e. time series
+ if data.ndim==1:
+ # Create array to store the resulting data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+
+ # Calculate the means across each unique time unit
+ i=0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ datam=ma.masked_array(data,timeunits!=myunit)
+ meanstore[i] = ma.average(datam)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i = i+1
+
+ # 3D data arrays
+ if data.ndim==3:
+ # datamask = create_mask_using_threshold(data,threshold=0.75)
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
+
+ # Calculate the means across each unique time unit
+ i=0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ mask = np.zeros_like(data)
+ mask[timeunits!=myunit,:,:] = 1.0
+ # Calculate missing data mask within each time unit...
+ datamask_at_this_timeunit = np.zeros_like(data)
+ datamask_at_this_timeunit[:]= create_mask_using_threshold(data[timeunits==myunit,:,:],threshold=0.75)
+ # Store results for masking later
+ datamask_store.append(datamask_at_this_timeunit[0])
+ # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
+ datam = ma.masked_array(data,np.logical_or(mask,datamask_at_this_timeunit))
+ meanstore[i,:,:] = ma.average(datam,axis=0)
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+
+ if processing_required:
+ # Create masked array (using missing data mask defined above)
+ datamask_store = np.array(datamask_store)
+ meanstorem = ma.masked_array(meanstore, datamask_store)
+
+ return meanstorem,newTimesList
+
+def decode_model_timesK(ifile,timeVarName,file_type):
+ #################################################################################################
+ # Convert model times ('hours since 1900...', 'days since ...') into a python datetime structure
+ # Input:
+ # filelist - list of model files
+ # timeVarName - name of the time variable in the model files
+ # Output:
+ # times - list of python datetime objects describing model data times
+ # Peter Lean February 2011
+ #################################################################################################
+ f = Nio.open_file(ifile,mode='r',options=None,format=file_type)
+ xtimes = f.variables[timeVarName]
+ timeFormat = xtimes.attributes['units']
+ #timeFormat = "days since 1979-01-01 00:00:00"
+ # search to check if 'since' appears in units
+ try:
+ sinceLoc = re.search('since',timeFormat).end()
+ except:
+ print 'Error decoding model times: time var attributes do not contain "since": Exit'
+ sys.exit()
+ # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
+ # TODO: Swap out this section for a set of if...elseif statements
+ units = ''
+ try:
+ _ = re.search('minutes',timeFormat).end()
+ units = 'minutes'
+ except:
+ pass
+ try:
+ _ = re.search('hours',timeFormat).end()
+ units = 'hours'
+ except:
+ pass
+ try:
+ _ = re.search('days',timeFormat).end()
+ units = 'days'
+ except:
+ pass
+ try:
+ _ = re.search('months',timeFormat).end()
+ units = 'months'
+ except:
+ pass
+ try:
+ _ = re.search('years',timeFormat).end()
+ units = 'years'
+ except:
+ pass
+ # cut out base time (the bit following 'since')
+ base_time_string = string.lstrip(timeFormat[sinceLoc:])
+ # decode base time
+ # 9/25/2012: datetime.timedelta has problem with the argument because xtimes is NioVariable.
+ # Soln (J. Kim): use a temp variable ttmp in the for loop, then convert it into an integer xtime.
+ base_time = decodeTimeFromString(base_time_string)
+ times=[]
+ for ttmp in xtimes[:]:
+ xtime = int(ttmp)
+ if(units=='minutes'):
+ dt = datetime.timedelta(minutes=xtime); new_time = base_time + dt
+ if(units=='hours'):
+ dt = datetime.timedelta(hours=xtime); new_time = base_time + dt
+ if(units=='days'):
+ dt = datetime.timedelta(days=xtime); new_time = base_time + dt
+ if(units=='months'): # NB. adding months in python is complicated as month length varies and hence ambigous.
+ # Perform date arithmatic manually
+ # Assumption: the base_date will usually be the first of the month
+ # NB. this method will fail if the base time is on the 29th or higher day of month
+ # -as can't have, e.g. Feb 31st.
+ new_month = int(base_time.month + xtime % 12)
+ new_year = int(math.floor(base_time.year + xtime / 12.))
+ #print type(new_year),type(new_month),type(base_time.day),type(base_time.hour),type(base_time.second)
+ new_time = datetime.datetime(new_year,new_month,base_time.day,base_time.hour,base_time.second,0)
+ if(units=='years'):
+ dt = datetime.timedelta(years=xtime); new_time = base_time + dt
+ times.append(new_time)
+ return times
+
+
+def regrid_in_time(data,dateList,unit):
+ #################################################################################################
+ # Routine to calculate averages on longer time units than the data exists on.
+ # e.g. if the data is 6-hourly, calculate daily, or monthly means.
+ # Input:
+ # data - data values
+ # dateList - list of python datetime structures corresponding to data values
+ # unit - string describing time unit to average onto
+ # e.g. 'monthly', 'daily', 'pentad','annual','decadal'
+ # Output:
+ # meanstorem - numpy masked array of data values meaned over required time period
+ # newTimesList - a list of python datetime objects representing the data in the new averagin period
+ # NB. currently set to beginning of averaging period,
+ # i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
+ # ..............................
+ # Jinwon Kim, Sept 30, 2012
+ # Created from calc_average_on_new_time_unit_K, Peter's original, with the modification below:
+ # Instead of masked array used by Peter, use wh to defined data within the averaging range.
+ #################################################################################################
+
+ print '*** Begin calc_average_on_new_time_unit_KK ***'
+ # Check if the user-selected temporal grid is valid. If not, EXIT
+ acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging: EXIT'; return -1,-1,-1,-1
+
+ # Calculate time arrays of: annual (yyyy, [2007]), monthly (yyyymm, [200701,200702]), daily (yyyymmdd, [20070101,20070102])
+ # "%02d" is similar to i2.2 in Fortran
+ if unit=='annual': # YYYY
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year))
+ elif unit=='monthly': # YYYYMM
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+ elif unit=='daily': # YYYYMMDD
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
+ elif unit=='full': # Full list: a special case
+ comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
+ timeunits = np.array(timeunits,dtype=int)
+ print 'timeRegridOption= ',unit#'; output timeunits= ',timeunits
+ #print 'timeRegridOption= ',unit,'; output timeunits= ',timeunits
+
+ # empty list to store time levels after temporal regridding
+ newTimesList = []
+
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits)==(len(np.unique(timeunits))):
+ processing_required = False
+ print 'processing_required= ',processing_required,': input time steps= ',len(timeunits),': regridded output time steps = ',len(np.unique(timeunits))
+ #print 'np.unique(timeunits): ',np.unique(timeunits)
+ print 'data.ndim= ',data.ndim
+
+ if data.ndim==1: # 1D data arrays, i.e. 1D time series
+ # Create array to store the temporally regridded data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+ # Calculate the means across each unique time unit
+ i=0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ wh = timeunits==myunit
+ datam = data[wh]
+ meanstore[i] = ma.average(datam)
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i = i+1
+
+ elif data.ndim==3: # 3D data arrays, i.e. 2D time series
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
+ # Calculate the means across each unique time unit
+ i=0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ wh = timeunits==myunit
+ datam = data[wh,:,:]
+ meanstore[i,:,:] = ma.average(datam,axis=0)
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+ elif processing_required:
+ meanstorem = meanstore
+
+ print '*** End calc_average_on_new_time_unit_KK ***'
+ return meanstorem,newTimesList
[32/50] [abbrv] initial mccsearch code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/newPreprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/newPreprocessingMERG.py b/mccsearch/code/newPreprocessingMERG.py
new file mode 100755
index 0000000..cedc155
--- /dev/null
+++ b/mccsearch/code/newPreprocessingMERG.py
@@ -0,0 +1,175 @@
+
+'''
+ The scripts for processing MERG raw data
+ #******************************************************************
+ '''
+
+import fileinput
+import glob
+import os
+import re
+import string
+import subprocess
+import sys
+
+#----------------------- GLOBAL VARIABLES --------------------------
+# --------------------- User defined variables ---------------------
+LATMIN = '-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '20.0' #max latitude; -ve values in the SH e.g. 5S = -5
+LONMIN = '-30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
+LONMAX = '30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
+T_BB_MAX = '250' #warmest temp to allow in K
+T_BB_MIN = '180' #coolest temp to allow in K
+#------------------- End user defined Variables -------------------
+
+#------------------------ End GLOBAL VARS -------------------------
+
+def preprocessingMERG(MERGdirname):
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ Directory to the location of the raw MERG files, preferably zipped
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ os.chdir((MERGdirname+'/'))
+ imgFilename = ''
+
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ #for files in glob.glob("*-pixel"):
+ for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+ #TODO: must be something more pythonic!
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+ #generate the lats4D command for GrADS
+ lats4D = 'lats4d -v -q -lat ' + LATMIN +' ' + LATMAX + ' ' + '-lon ' + LONMIN + ' ' + LONMAX + ' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -lbc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+
+ #----- Generate GrADS images--------------
+ imgFilename = hr+'Z'+day+mth+yy+'.gif'
+ tempMaskedImages(imgFilename)
+ #-------End Generate GrADS images ---------
+
+ #remove the raw data file to save some resources
+ rmCmd= "rm " + fname
+ subprocess.call(rmCmd, shell=True)
+
+ #when all the files have been converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ #mv all images
+ subprocess.call('mkdir mergImgs', shell=True)
+ subprocess.call('mv *.gif mergImgs', shell=True)
+
+ return
+
+#******************************************************************
+def tempMaskedImages(imgFilename):
+ '''
+ Purpose:: To generate temperature-masked images for a first pass verification
+
+ Input::
+ filename for the gif file
+
+ Output::
+ None - gif images for each file of IR temperature < T_BB_MAX are generated in folder called mergImgs
+
+ Assumptions::
+ Same as for preprocessingMERG
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ subprocess.call('rm tempMaskedImages.gs', shell=True)
+ subprocess.call('touch tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'open merg.ctl''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set mpdset hires''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set lat ' +LATMIN + ' ' +LATMAX +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set lon ' +LONMIN + ' ' +LONMAX +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set cint 10''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set cmax '+T_BB_MAX +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'set cmin '+T_BB_MIN +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'d ch4+75''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'draw title Masked Temp @ '+imgFilename +'\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800''\'" >> tempMaskedImages.gs', shell=True)
+ subprocess.call('echo "''\'quit''\'" >> tempMaskedImages.gs', shell=True)
+ gradscmd = 'grads -blc ' + '\'run tempMaskedImages.gs''\''
+ subprocess.call(gradscmd, shell=True)
+ return
+
+#******************************************************************
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/preprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/preprocessingMERG.py b/mccsearch/code/preprocessingMERG.py
new file mode 100755
index 0000000..04964a8
--- /dev/null
+++ b/mccsearch/code/preprocessingMERG.py
@@ -0,0 +1,125 @@
+'''
+#preprocessing for MERG data
+# unzips files and generates the NETCDF files using GrADS lats4D
+# http://opengrads.org/doc/scripts/lats4d/
+# This script requires the merg_template.ctl resides with the folder
+# TODO: generate this file
+# Kim Whitehall
+# Last updated: June 4th 2013
+'''
+import subprocess
+import fileinput
+import json
+import os
+import glob
+import re
+import sys
+
+
+def preprocessing_merg():
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format
+ NOTE: VERY RAW AND DIRTY
+ Input::
+ none
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+ os.chdir("/Users/whitehal/Documents/kimSum2013/mergData1/")
+ searchExpDset = "DSET "
+ searchExpTdef = "TDEF "
+ #for files in glob.glob("*-pixel"):
+ for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+
+
+ # #cp template merg.ctl
+ # subprocess.call('cp -f merg_template.ctl merg.ctl', shell=True)
+
+ # #edit merg.ctl dset line to contain filename by running file
+ # #http://stackoverflow.com/questions/39086/search-and-replace-a-line-in-a-file-in-python
+ # replaceExpDset = 'DSET ' + fname
+ # replaceExpTdef = 'TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn'
+ # for line in fileinput.input("merg.ctl", inplace=True):
+ # if searchExpDset in line:
+ # line = line.replace(searchExpDset, replaceExpDset)
+ # if searchExpTdef in line:
+ # line = line.replace(searchExpTdef, replaceExpTdef)
+ # sys.stdout.write(line)
+
+ #lats4d cmd
+ lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+
+ #when all the files have benn converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ return
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/process.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.py b/mccsearch/code/process.py
new file mode 100644
index 0000000..5a362ec
--- /dev/null
+++ b/mccsearch/code/process.py
@@ -0,0 +1,1284 @@
+"""Collection of functions that process numpy arrays of varying shapes.
+Functions can range from subsetting to regridding"""
+
+# Python Standard Libary Imports
+import math
+import datetime
+import re
+import string
+import sys
+import time
+
+# 3rd Party Imports
+import Nio
+import numpy as np
+import numpy.ma as ma
+from scipy.ndimage import map_coordinates
+
+
+def extract_subregion_from_data_array(data, lats, lons, latmin, latmax, lonmin, lonmax):
+ '''
+ Extract a sub-region from a data array.
+
+ Example::
+ **e.g.** the user may load a global model file, but only want to examine data over North America
+ This function extracts a sub-domain from the original data.
+ The defined sub-region must be a regular lat/lon bounding box,
+ but the model data may be on a non-regular grid (e.g. rotated, or Guassian grid layout).
+ Data are kept on the original model grid and a rectangular (in model-space) region
+ is extracted which contains the rectangular (in lat/lon space) user supplied region.
+
+ INPUT::
+ data - 3d masked data array
+ lats - 2d array of latitudes corresponding to data array
+ lons - 2d array of longitudes corresponding to data array
+ latmin, latmax, lonmin, lonmax - bounding box of required region to extract
+
+ OUTPUT::
+ data2 - subset of original data array containing only data from required subregion
+ lats2 - subset of original latitude data array
+ lons2 - subset of original longitude data array
+
+ '''
+
+
+
+ # Mask model data array to find grid points inside the supplied bounding box
+ whlat = (lats > latmin) & (lats < latmax)
+ whlon = (lons > lonmin) & (lons < lonmax)
+ wh = whlat & whlon
+
+ # Find required matrix indices describing the limits of the regular lat/lon bounding box
+ jmax = np.where(wh == True)[0].max()
+ jmin = np.where(wh == True)[0].min()
+ imax = np.where(wh == True)[1].max()
+ imin = np.where(wh == True)[1].min()
+
+ # Cut out the sub-region from the data array
+ data2 = ma.zeros((data.shape[0], jmax - jmin, imax - imin))
+ data2 = data[:, jmin:jmax, imin:imax]
+
+ # Cut out sub-region from lats,lons arrays
+ lats2 = lats[jmin:jmax, imin:imax]
+ lons2 = lons[jmin:jmax, imin:imax]
+
+ return data2, lats2, lons2
+
+def calc_area_mean(data, lats, lons, mymask='not set'):
+ '''
+ Calculate Area Average of data in a masked array
+
+ INPUT::
+ data: a masked array of data (NB. only data from one time expected to be passed at once)
+ lats: 2d array of regularly gridded latitudes
+ lons: 2d array of regularly gridded longitudes
+ mymask: (optional) defines spatial region to do averaging over
+
+ OUTPUT::
+ area_mean: a value for the mean inside the area
+
+ '''
+
+
+
+ # If mask not passed in, then set maks to cover whole data domain
+ if(mymask == 'not set'):
+ mymask = np.empty(data.shape)
+ mymask[:] = False # NB. mask means (don't show), so False everywhere means use everything.
+
+ # Dimension check on lats, lons
+ # Sometimes these arrays are 3d, sometimes 2d, sometimes 1d
+ # This bit of code just converts to the required 2d array shape
+ if(len(lats.shape) == 3):
+ lats = lats[0, :, :]
+
+ if(len(lons.shape) == 3):
+ lons = lons[0, :, :]
+
+ if(np.logical_and(len(lats.shape) == 1, len(lons.shape) == 1)):
+ lons, lats = np.meshgrid(lons, lats)
+
+ # Calculate grid length (assuming regular lat/lon grid)
+ dlat = lats[1, 0] - lats[0, 0]
+ dlon = lons[0, 1] - lons[0, 0]
+
+ # Calculates weights for each grid box
+ myweights = calc_area_in_grid_box(lats, dlon, dlat)
+
+ # Create a new masked array covering just user selected area (defined in mymask)
+ # NB. this preserves missing data points in the observations data
+ subdata = ma.masked_array(data, mask=mymask)
+
+ if(myweights.shape != subdata.shape):
+ myweights.resize(subdata.shape)
+ myweights[1:, :] = myweights[0, :]
+
+ # Calculate weighted mean using ma.average (which takes weights)
+ area_mean = ma.average(subdata, weights=myweights)
+
+ return area_mean
+
+def calc_area_in_grid_box(latitude, dlat, dlon):
+ '''
+ Calculate area of regular lat-lon grid box
+
+ INPUT::
+ latitude: latitude of grid box (degrees)
+ dlat: grid length in latitude direction (degrees)
+ dlon: grid length in longitude direction (degrees)
+
+ OUTPUT::
+ A: area of the grid box
+
+ '''
+
+ R = 6371000 # radius of Earth in metres
+ C = 2 * math.pi * R
+
+ latitude = np.radians(latitude)
+
+ A = (dlon * (C / 360.)) * (dlat * (C / 360.) * np.cos(latitude))
+
+ return A
+
+def do_regrid(q, lat, lon, lat2, lon2, order=1, mdi= -999999999):
+ '''
+ Perform regridding from one set of lat,lon values onto a new set (lat2,lon2)
+
+ Input::
+ q - the variable to be regridded
+ lat,lon - original co-ordinates corresponding to q values
+ lat2,lon2 - new set of latitudes and longitudes that you want to regrid q onto
+ order - (optional) interpolation order 1=bi-linear, 3=cubic spline
+ mdi - (optional) fill value for missing data (used in creation of masked array)
+
+ Output::
+ q2 - q regridded onto the new set of lat2,lon2
+
+ '''
+
+ nlat = q.shape[0]
+ nlon = q.shape[1]
+
+ nlat2 = lat2.shape[0]
+ nlon2 = lon2.shape[1]
+
+ # To make our lives easier down the road, let's
+ # turn these into arrays of x & y coords
+ loni = lon2.ravel()
+ lati = lat2.ravel()
+
+ loni = loni.copy() # NB. it won't run unless you do this...
+ lati = lati.copy()
+
+ # Now, we'll set points outside the boundaries to lie along an edge
+ loni[loni > lon.max()] = lon.max()
+ loni[loni < lon.min()] = lon.min()
+
+ # To deal with the "hard" break, we'll have to treat y differently,
+ # so we're just setting the min here...
+ lati[lati > lat.max()] = lat.max()
+ lati[lati < lat.min()] = lat.min()
+
+
+ # We need to convert these to (float) indicies
+ # (xi should range from 0 to (nx - 1), etc)
+ loni = (nlon - 1) * (loni - lon.min()) / (lon.max() - lon.min())
+
+ # Deal with the "hard" break in the y-direction
+ lati = (nlat - 1) * (lati - lat.min()) / (lat.max() - lat.min())
+
+ # Notes on dealing with MDI when regridding data.
+ # Method adopted here:
+ # Use bilinear interpolation of data by default (but user can specify other order using order=... in call)
+ # Perform bilinear interpolation of data, and of mask.
+ # To be conservative, new grid point which contained some missing data on the old grid is set to missing data.
+ # -this is achieved by looking for any non-zero interpolated mask values.
+ # To avoid issues with bilinear interpolation producing strong gradients leading into the MDI,
+ # set values at MDI points to mean data value so little gradient visible = not ideal, but acceptable for now.
+
+ # Set values in MDI so that similar to surroundings so don't produce large gradients when interpolating
+ # Preserve MDI mask, by only changing data part of masked array object.
+ for shift in (-1, 1):
+ for axis in (0, 1):
+ q_shifted = np.roll(q, shift=shift, axis=axis)
+ idx = ~q_shifted.mask * q.mask
+ q.data[idx] = q_shifted[idx]
+
+ # Now we actually interpolate
+ # map_coordinates does cubic interpolation by default,
+ # use "order=1" to preform bilinear interpolation instead...
+ q2 = map_coordinates(q, [lati, loni], order=order)
+ q2 = q2.reshape([nlat2, nlon2])
+
+ # Set values to missing data outside of original domain
+ q2 = ma.masked_array(q2, mask=np.logical_or(np.logical_or(lat2 >= lat.max(),
+ lat2 <= lat.min()),
+ np.logical_or(lon2 <= lon.min(),
+ lon2 >= lon.max())))
+
+ # Make second map using nearest neighbour interpolation -use this to determine locations with MDI and mask these
+ qmdi = np.zeros_like(q)
+ qmdi[q.mask == True] = 1.
+ qmdi[q.mask == False] = 0.
+ qmdi_r = map_coordinates(qmdi, [lati, loni], order=order)
+ qmdi_r = qmdi_r.reshape([nlat2, nlon2])
+ mdimask = (qmdi_r != 0.0)
+
+ # Combine missing data mask, with outside domain mask define above.
+ q2.mask = np.logical_or(mdimask, q2.mask)
+
+ return q2
+
+def create_mask_using_threshold(masked_array, threshold=0.5):
+ '''
+ Routine to create a mask, depending on the proportion of times with missing data.
+ For each pixel, calculate proportion of times that are missing data,
+ **if** the proportion is above a specified threshold value,
+ then mark the pixel as missing data.
+
+ Input::
+ masked_array - a numpy masked array of data (assumes time on axis 0, and space on axes 1 and 2).
+ threshold - (optional) threshold proportion above which a pixel is marked as 'missing data'.
+ NB. default threshold = 50%
+
+ Output::
+ mymask - a numpy array describing the mask. NB. not a masked array, just the mask itself.
+
+ '''
+
+
+ # try, except used as some model files don't have a full mask, but a single bool
+ # the except catches this situation and deals with it appropriately.
+ try:
+ nT = masked_array.mask.shape[0]
+
+ # For each pixel, count how many times are masked.
+ nMasked = masked_array.mask[:, :, :].sum(axis=0)
+
+ # Define new mask as when a pixel has over a defined threshold ratio of masked data
+ # e.g. if the threshold is 75%, and there are 10 times,
+ # then a pixel will be masked if more than 5 times are masked.
+ mymask = nMasked > (nT * threshold)
+
+ except:
+ mymask = np.zeros_like(masked_array.data[0, :, :])
+
+ return mymask
+
+def calc_average_on_new_time_unit(data, dateList, unit='monthly'):
+ '''
+ Routine to calculate averages on longer time units than the data exists on.
+
+ Example::
+ If the data is 6-hourly, calculate daily, or monthly means.
+
+ Input::
+ data - data values
+ dateList - list of python datetime structures corresponding to data values
+ unit - string describing time unit to average onto.
+ *Valid values are: 'monthly', 'daily', 'pentad','annual','decadal'*
+
+ Output:
+ meanstorem - numpy masked array of data values meaned over required time period
+ newTimesList - a list of python datetime objects representing the data in the new averagin period
+ *NB.* currently set to beginning of averaging period,
+ **i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.**
+ '''
+
+ # First catch unknown values of time unit passed in by user
+ acceptable = (unit == 'full') | (unit == 'annual') | (unit == 'monthly') | (unit == 'daily') | (unit == 'pentad')
+
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging'
+ print ' Please check your code.'
+ return
+
+ # Calculate arrays of years (2007,2007),
+ # yearsmonths (200701,200702),
+ # or yearmonthdays (20070101,20070102)
+ # -depending on user selected averaging period.
+
+ # Year list
+ if unit == 'annual':
+ print 'Calculating annual mean data'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+
+ timeunits.append(str(dateList[i].year))
+
+ timeunits = np.array(timeunits, dtype=int)
+
+ # YearMonth format list
+ if unit == 'monthly':
+ print 'Calculating monthly mean data'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+
+ timeunits = np.array(timeunits, dtype=int)
+
+ # YearMonthDay format list
+ if unit == 'daily':
+ print 'Calculating daily mean data'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + \
+ str("%02d" % dateList[i].day))
+
+ timeunits = np.array(timeunits, dtype=int)
+
+
+ # TODO: add pentad setting using Julian days?
+
+
+ # Full list: a special case
+ if unit == 'full':
+ print 'Calculating means data over full time range'
+ timeunits = []
+
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
+
+ timeunits = np.array(timeunits, dtype=int)
+
+
+
+ # empty list to store new times
+ newTimesList = []
+
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and
+ # calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits) == (len(np.unique(timeunits))):
+ processing_required = False
+
+ # 1D data arrays, i.e. time series
+ if data.ndim == 1:
+ # Create array to store the resulting data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+
+ # Calculate the means across each unique time unit
+ i = 0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ datam = ma.masked_array(data, timeunits != myunit)
+ meanstore[i] = ma.average(datam)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+ i = i + 1
+
+ # 3D data arrays
+ if data.ndim == 3:
+
+ #datamask = create_mask_using_threshold(data,threshold=0.75)
+
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)), data.shape[1], data.shape[2]])
+
+ # Calculate the means across each unique time unit
+ i = 0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
+
+ mask = np.zeros_like(data)
+ mask[timeunits != myunit, :, :] = 1.0
+
+ # Calculate missing data mask within each time unit...
+ datamask_at_this_timeunit = np.zeros_like(data)
+ datamask_at_this_timeunit[:] = create_mask_using_threshold(data[timeunits == myunit, :, :], threshold=0.75)
+ # Store results for masking later
+ datamask_store.append(datamask_at_this_timeunit[0])
+
+ # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
+ datam = ma.masked_array(data, np.logical_or(mask, datamask_at_this_timeunit))
+ meanstore[i, :, :] = ma.average(datam, axis=0)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+
+ if processing_required:
+ # Create masked array (using missing data mask defined above)
+ datamask_store = np.array(datamask_store)
+ meanstorem = ma.masked_array(meanstore, datamask_store)
+
+ return meanstorem, newTimesList
+
+def calc_running_accum_from_period_accum(data):
+ '''
+ Routine to calculate running total accumulations from individual period accumulations.
+ ::
+
+ e.g. 0,0,1,0,0,2,2,1,0,0
+ -> 0,0,1,1,1,3,5,6,6,6
+
+ Input::
+ data: numpy array with time in the first axis
+
+ Output::
+ running_acc: running accumulations
+
+ '''
+
+
+ running_acc = np.zeros_like(data)
+
+ if(len(data.shape) == 1):
+ running_acc[0] = data[0]
+
+ if(len(data.shape) > 1):
+ running_acc[0, :] = data[0, :]
+
+ for i in np.arange(data.shape[0] - 1):
+ if(len(data.shape) == 1):
+ running_acc[i + 1] = running_acc[i] + data[i + 1]
+
+ if(len(data.shape) > 1):
+ running_acc[i + 1, :] = running_acc[i, :] + data[i + 1, :]
+
+ return running_acc
+
+def ignore_boundaries(data, rim=10):
+ '''
+ Routine to mask the lateral boundary regions of model data to ignore them from calculations.
+
+ Input::
+ data - a masked array of model data
+ rim - (optional) number of grid points to ignore
+
+ Output::
+ data - data array with boundary region masked
+
+ '''
+
+ nx = data.shape[1]
+ ny = data.shape[0]
+
+ rimmask = np.zeros_like(data)
+ for j in np.arange(rim):
+ rimmask[j, 0:nx - 1] = 1.0
+
+ for j in ny - 1 - np.arange(rim):
+ rimmask[j, 0:nx - 1] = 1.0
+
+ for i in np.arange(rim):
+ rimmask[0:ny - 1, i] = 1.0
+
+ for i in nx - 1 - np.arange(rim):
+ rimmask[0:ny - 1, i] = 1.0
+
+ data = ma.masked_array(data, mask=rimmask)
+
+ return data
+
+def normalizeDatetimes(datetimes, timestep):
+ """
+ Input::
+ datetimes - list of datetime objects that need to be normalized
+ timestep - string of value ('daily' | 'monthly')
+ Output::
+ normalDatetimes - list of datetime objects that have been normalized
+
+ Normalization Rules::
+ Daily data will be forced to an hour value of 00:00:00
+ Monthly data will be forced to the first of the month at midnight
+ """
+ normalDatetimes = []
+ if timestep.lower() == 'monthly':
+ for inputDatetime in datetimes:
+ if inputDatetime.day != 1:
+ # Clean the inputDatetime
+ inputDatetimeString = inputDatetime.strftime('%Y%m%d')
+ normalInputDatetimeString = inputDatetimeString[:6] + '01'
+ inputDatetime = datetime.datetime.strptime(normalInputDatetimeString, '%Y%m%d')
+
+ normalDatetimes.append(inputDatetime)
+
+ elif timestep.lower() == 'daily':
+ for inputDatetime in datetimes:
+ if inputDatetime.hour != 0 or inputDatetime.minute != 0 or inputDatetime.second != 0:
+ datetimeString = inputDatetime.strftime('%Y%m%d%H%M%S')
+ normalDatetimeString = datetimeString[:8] + '000000'
+ inputDatetime = datetime.datetime.strptime(normalDatetimeString, '%Y%m%d%H%M%S')
+
+ normalDatetimes.append(inputDatetime)
+
+ return normalDatetimes
+
+def getModelTimes(modelFile, timeVarName):
+ '''
+ TODO: Do a better job handling dates here
+ Routine to convert from model times ('hours since 1900...', 'days since ...')
+ into a python datetime structure
+
+ Input::
+ modelFile - path to the model tile you want to extract the times list and modelTimeStep from
+ timeVarName - name of the time variable in the model file
+
+ Output::
+ times - list of python datetime objects describing model data times
+ modelTimeStep - 'hourly','daily','monthly','annual'
+ '''
+
+ f = Nio.open_file(modelFile)
+ xtimes = f.variables[timeVarName]
+ timeFormat = xtimes.attributes['units']
+
+ # search to check if 'since' appears in units
+ try:
+ sinceLoc = re.search('since', timeFormat).end()
+
+
+ #KDW the below block generates and error. But the print statement, indicates that sinceLoc found something
+ except AttributeError:
+ print 'Error decoding model times: time variable attributes do not contain "since"'
+ raise
+
+ units = ''
+ TIME_UNITS = ('minutes', 'hours', 'days', 'months', 'years')
+ # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
+ for unit in TIME_UNITS:
+ if re.search(unit, timeFormat):
+ units = unit
+ break
+
+ # cut out base time (the bit following 'since')
+ base_time_string = string.lstrip(timeFormat[sinceLoc:])
+
+ # decode base time
+ base_time = decodeTimeFromString(base_time_string)
+
+ times = []
+
+
+
+ for xtime in xtimes[:]:
+
+ # Cast time as an int ***KDW remove this so fractional xtime can be read from MERG
+ xtime = int(xtime)
+
+ if units == 'minutes':
+ dt = datetime.timedelta(minutes=xtime)
+ new_time = base_time + dt
+ elif units == 'hours':
+ dt = datetime.timedelta(hours=xtime)
+ new_time = base_time + dt
+ elif units == 'days':
+ dt = datetime.timedelta(days=xtime)
+ new_time = base_time + dt
+ elif units == 'months':
+ # NB. adding months in python is complicated as month length varies and hence ambiguous.
+ # Perform date arithmatic manually
+ # Assumption: the base_date will usually be the first of the month
+ # NB. this method will fail if the base time is on the 29th or higher day of month
+ # -as can't have, e.g. Feb 31st.
+ new_month = int(base_time.month + xtime % 12)
+ new_year = int(math.floor(base_time.year + xtime / 12.))
+ new_time = datetime.datetime(new_year, new_month, base_time.day, base_time.hour, base_time.second, 0)
+ elif units == 'years':
+ dt = datetime.timedelta(years=xtime)
+ new_time = base_time + dt
+
+ #print "xtime is:", xtime, "dt is: ", dt
+
+
+
+ times.append(new_time)
+
+ try:
+ timeStepLength = int(xtimes[1] - xtimes[0] + 1.e-12)
+ modelTimeStep = getModelTimeStep(units, timeStepLength)
+
+ #KDW if timeStepLength is zero do not normalize times as this would create an empty list
+ if timeStepLength != 0:
+ times = normalizeDatetimes(times, modelTimeStep)
+ except:
+ raise
+
+ return times, modelTimeStep
+
+def getModelTimeStep(units, stepSize):
+ # Time units are now determined. Determine the time intervals of input data (mdlTimeStep)
+
+ if units == 'minutes':
+ if stepSize == 60:
+ modelTimeStep = 'hourly'
+ elif stepSize == 1440:
+ modelTimeStep = 'daily'
+ # 28 days through 31 days
+ elif 40320 <= stepSize <= 44640:
+ modelTimeStep = 'monthly'
+ # 365 days through 366 days
+ elif 525600 <= stepSize <= 527040:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'hours':
+ #need a check for fractional hrs and only one hr i.e. stepSize=0
+ if stepSize == 0 or stepSize == 1:
+ modelTimeStep = 'hourly'
+ elif stepSize == 24:
+ modelTimeStep = 'daily'
+ elif 672 <= stepSize <= 744:
+ modelTimeStep = 'monthly'
+ elif 8760 <= stepSize <= 8784:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'days':
+ if stepSize == 1:
+ modelTimeStep = 'daily'
+ elif 28 <= stepSize <= 31:
+ modelTimeStep = 'monthly'
+ elif 365 <= stepSize <= 366:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'months':
+ if stepSize == 1:
+ modelTimeStep = 'monthly'
+ elif stepSize == 12:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ elif units == 'years':
+ if stepSize == 1:
+ modelTimeStep = 'annual'
+ else:
+ raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
+
+ else:
+ errorMessage = 'the time unit ', units, ' is not currently handled in this version.'
+ raise Exception(errorMessage)
+
+ return modelTimeStep
+
+def decodeTimeFromString(time_string):
+ '''
+ Decodes string into a python datetime object
+ *Method:* tries a bunch of different time format possibilities and hopefully one of them will hit.
+ ::
+
+ **Input:** time_string - a string that represents a date/time
+
+ **Output:** mytime - a python datetime object
+ '''
+
+ # This will deal with times that use decimal seconds
+ if '.' in time_string:
+ time_string = time_string.split('.')[0] + '0'
+ else:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y/%m/%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y%m%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y:%m:%d %H:%M:%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y%m%d%H%M%S')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d %H:%M')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+#KDW added
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+ try:
+ mytime = time.strptime(time_string, '%Y-%m-%d %H')
+ mytime = datetime.datetime(*mytime[0:6])
+ return mytime
+
+ except ValueError:
+ pass
+
+
+ print 'Error decoding time string: string does not match a predefined time format'
+ return 0
+
+def regrid_wrapper(regrid_choice, obsdata, obslats, obslons, wrfdata, wrflats, wrflons):
+ '''
+ Wrapper routine for regridding.
+
+ Either regrids model to obs grid, or obs to model grid, depending on user choice
+
+ Inputs::
+ regrid_choice - [0] = Regrid obs data onto model grid or
+ [1] = Regrid model data onto obs grid
+
+ obsdata,wrfdata - data arrays
+ obslats,obslons - observation lat,lon arrays
+ wrflats,wrflons - model lat,lon arrays
+
+ Output::
+ rdata,rdata2 - regridded data
+ lats,lons - latitudes and longitudes of regridded data
+
+ '''
+
+ # Regrid obs data onto model grid
+ if(regrid_choice == '0'):
+
+ ndims = len(obsdata.shape)
+ if(ndims == 3):
+ newshape = [obsdata.shape[0], wrfdata.shape[1], wrfdata.shape[2]]
+ nT = obsdata.shape[0]
+
+ if(ndims == 2):
+ newshape = [wrfdata.shape[0], wrfdata.shape[1]]
+ nT = 0
+
+ rdata = wrfdata
+ lats, lons = wrflats, wrflons
+
+ # Create a new masked array with the required dimensions
+ tmp = np.zeros(newshape)
+ rdata2 = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
+
+ rdata2[:] = 0.0
+
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata2[t, :, :] = do_regrid(obsdata[t, :, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+
+ if(nT == 0):
+ rdata2[:, :] = do_regrid(obsdata[:, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+
+
+ # Regrid model data onto obs grid
+ if(regrid_choice == '1'):
+ ndims = len(wrfdata.shape)
+ if(ndims == 3):
+ newshape = [wrfdata.shape[0], obsdata.shape[1], obsdata.shape[2]]
+ nT = obsdata.shape[0]
+
+ if(ndims == 2):
+ newshape = [obsdata.shape[0], obsdata.shape[1]]
+ nT = 0
+
+ rdata2 = obsdata
+ lats, lons = obslats, obslons
+
+ tmp = np.zeros(newshape)
+ rdata = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
+
+ rdata[:] = 0.0
+
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata[t, :, :] = do_regrid(wrfdata[t, :, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+
+ if(nT == 0):
+ rdata[:, :] = do_regrid(wrfdata[:, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+
+
+ return rdata, rdata2, lats, lons
+
+def extract_sub_time_selection(allTimes, subTimes, data):
+ '''
+ Routine to extract a sub-selection of times from a data array.
+
+ Example::
+ Suppose a data array has 30 time values for daily data for a whole month,
+ but you only want the data from the 5th-15th of the month.
+
+ Input::
+ allTimes - list of datetimes describing what times the data in the data array correspond to
+ subTimes - the times you want to extract data from
+ data - the data array
+
+ Output::
+ subdata - subselection of data array
+
+ '''
+ # Create new array to store the subselection
+ subdata = np.zeros([len(subTimes), data.shape[1], data.shape[2]])
+
+ # Loop over all Times and when it is a member of the required subselection, copy the data across
+ i = 0 # counter for allTimes
+ subi = 0 # counter for subTimes
+ for t in allTimes:
+ if(np.setmember1d(allTimes, subTimes)):
+ subdata[subi, :, :] = data[i, :, :]
+ subi += 1
+ i += 1
+
+ return subdata
+
+def calc_average_on_new_time_unit_K(data, dateList, unit):
+ '''
+ Routine to calculate averages on longer time units than the data exists on.
+ e.g. if the data is 6-hourly, calculate daily, or monthly means.
+
+ Input:
+ data - data values
+ dateList - list of python datetime structures corresponding to data values
+ unit - string describing time unit to average onto
+ e.g. 'monthly', 'daily', 'pentad','annual','decadal'
+ Output:
+ meanstorem - numpy masked array of data values meaned over required time period
+ newTimesList - a list of python datetime objects representing the data in the new averagin period
+ NB. currently set to beginning of averaging period,
+ i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
+ '''
+
+ # Check if the user-selected temporal grid is valid. If not, EXIT
+ acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging: EXIT'
+ return -1,-1,-1,-1
+
+ # Calculate arrays of: annual timeseries: year (2007,2007),
+ # monthly time series: year-month (200701,200702),
+ # daily timeseries: year-month-day (20070101,20070102)
+ # depending on user-selected averaging period.
+
+ # Year list
+ if unit=='annual':
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year))
+ timeunits = np.array(timeunits, dtype=int)
+
+ # YearMonth format list
+ if unit=='monthly':
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+ timeunits = np.array(timeunits,dtype=int)
+
+ # YearMonthDay format list
+ if unit=='daily':
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
+ timeunits = np.array(timeunits,dtype=int)
+
+ # TODO: add pentad setting using Julian days?
+
+ # Full list: a special case
+ if unit == 'full':
+ comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
+ timeunits = np.array(timeunits, dtype=int)
+
+ # empty list to store new times
+ newTimesList = []
+
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and
+ # calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits)==(len(np.unique(timeunits))):
+ processing_required = False
+
+ # 1D data arrays, i.e. time series
+ if data.ndim==1:
+ # Create array to store the resulting data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+
+ # Calculate the means across each unique time unit
+ i=0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ datam=ma.masked_array(data,timeunits!=myunit)
+ meanstore[i] = ma.average(datam)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i = i+1
+
+ # 3D data arrays
+ if data.ndim==3:
+ # datamask = create_mask_using_threshold(data,threshold=0.75)
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
+
+ # Calculate the means across each unique time unit
+ i=0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ mask = np.zeros_like(data)
+ mask[timeunits!=myunit,:,:] = 1.0
+ # Calculate missing data mask within each time unit...
+ datamask_at_this_timeunit = np.zeros_like(data)
+ datamask_at_this_timeunit[:]= create_mask_using_threshold(data[timeunits==myunit,:,:],threshold=0.75)
+ # Store results for masking later
+ datamask_store.append(datamask_at_this_timeunit[0])
+ # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
+ datam = ma.masked_array(data,np.logical_or(mask,datamask_at_this_timeunit))
+ meanstore[i,:,:] = ma.average(datam,axis=0)
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+
+ if processing_required:
+ # Create masked array (using missing data mask defined above)
+ datamask_store = np.array(datamask_store)
+ meanstorem = ma.masked_array(meanstore, datamask_store)
+
+ return meanstorem,newTimesList
+
+def decode_model_timesK(ifile,timeVarName,file_type):
+ #################################################################################################
+ # Convert model times ('hours since 1900...', 'days since ...') into a python datetime structure
+ # Input:
+ # filelist - list of model files
+ # timeVarName - name of the time variable in the model files
+ # Output:
+ # times - list of python datetime objects describing model data times
+ # Peter Lean February 2011
+ #################################################################################################
+ f = Nio.open_file(ifile,mode='r',options=None,format=file_type)
+ xtimes = f.variables[timeVarName]
+ timeFormat = xtimes.attributes['units']
+ #timeFormat = "days since 1979-01-01 00:00:00"
+ # search to check if 'since' appears in units
+ try:
+ sinceLoc = re.search('since',timeFormat).end()
+ except:
+ print 'Error decoding model times: time var attributes do not contain "since": Exit'
+ sys.exit()
+ # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
+ # TODO: Swap out this section for a set of if...elseif statements
+ units = ''
+ try:
+ _ = re.search('minutes',timeFormat).end()
+ units = 'minutes'
+ except:
+ pass
+ try:
+ _ = re.search('hours',timeFormat).end()
+ units = 'hours'
+ except:
+ pass
+ try:
+ _ = re.search('days',timeFormat).end()
+ units = 'days'
+ except:
+ pass
+ try:
+ _ = re.search('months',timeFormat).end()
+ units = 'months'
+ except:
+ pass
+ try:
+ _ = re.search('years',timeFormat).end()
+ units = 'years'
+ except:
+ pass
+ # cut out base time (the bit following 'since')
+ base_time_string = string.lstrip(timeFormat[sinceLoc:])
+ # decode base time
+ # 9/25/2012: datetime.timedelta has problem with the argument because xtimes is NioVariable.
+ # Soln (J. Kim): use a temp variable ttmp in the for loop, then convert it into an integer xtime.
+ base_time = decodeTimeFromString(base_time_string)
+ times=[]
+ for ttmp in xtimes[:]:
+ xtime = int(ttmp)
+ if(units=='minutes'):
+ dt = datetime.timedelta(minutes=xtime); new_time = base_time + dt
+ if(units=='hours'):
+ dt = datetime.timedelta(hours=xtime); new_time = base_time + dt
+ if(units=='days'):
+ dt = datetime.timedelta(days=xtime); new_time = base_time + dt
+ if(units=='months'): # NB. adding months in python is complicated as month length varies and hence ambigous.
+ # Perform date arithmatic manually
+ # Assumption: the base_date will usually be the first of the month
+ # NB. this method will fail if the base time is on the 29th or higher day of month
+ # -as can't have, e.g. Feb 31st.
+ new_month = int(base_time.month + xtime % 12)
+ new_year = int(math.floor(base_time.year + xtime / 12.))
+ #print type(new_year),type(new_month),type(base_time.day),type(base_time.hour),type(base_time.second)
+ new_time = datetime.datetime(new_year,new_month,base_time.day,base_time.hour,base_time.second,0)
+ if(units=='years'):
+ dt = datetime.timedelta(years=xtime); new_time = base_time + dt
+ times.append(new_time)
+ return times
+
+
+def regrid_in_time(data,dateList,unit):
+ #################################################################################################
+ # Routine to calculate averages on longer time units than the data exists on.
+ # e.g. if the data is 6-hourly, calculate daily, or monthly means.
+ # Input:
+ # data - data values
+ # dateList - list of python datetime structures corresponding to data values
+ # unit - string describing time unit to average onto
+ # e.g. 'monthly', 'daily', 'pentad','annual','decadal'
+ # Output:
+ # meanstorem - numpy masked array of data values meaned over required time period
+ # newTimesList - a list of python datetime objects representing the data in the new averagin period
+ # NB. currently set to beginning of averaging period,
+ # i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
+ # ..............................
+ # Jinwon Kim, Sept 30, 2012
+ # Created from calc_average_on_new_time_unit_K, Peter's original, with the modification below:
+ # Instead of masked array used by Peter, use wh to defined data within the averaging range.
+ #################################################################################################
+
+ print '*** Begin calc_average_on_new_time_unit_KK ***'
+ # Check if the user-selected temporal grid is valid. If not, EXIT
+ acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging: EXIT'; return -1,-1,-1,-1
+
+ # Calculate time arrays of: annual (yyyy, [2007]), monthly (yyyymm, [200701,200702]), daily (yyyymmdd, [20070101,20070102])
+ # "%02d" is similar to i2.2 in Fortran
+ if unit=='annual': # YYYY
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year))
+ elif unit=='monthly': # YYYYMM
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+ elif unit=='daily': # YYYYMMDD
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
+ elif unit=='full': # Full list: a special case
+ comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
+ timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
+ timeunits = np.array(timeunits,dtype=int)
+ print 'timeRegridOption= ',unit#'; output timeunits= ',timeunits
+ #print 'timeRegridOption= ',unit,'; output timeunits= ',timeunits
+
+ # empty list to store time levels after temporal regridding
+ newTimesList = []
+
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits)==(len(np.unique(timeunits))):
+ processing_required = False
+ print 'processing_required= ',processing_required,': input time steps= ',len(timeunits),': regridded output time steps = ',len(np.unique(timeunits))
+ #print 'np.unique(timeunits): ',np.unique(timeunits)
+ print 'data.ndim= ',data.ndim
+
+ if data.ndim==1: # 1D data arrays, i.e. 1D time series
+ # Create array to store the temporally regridded data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+ # Calculate the means across each unique time unit
+ i=0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ wh = timeunits==myunit
+ datam = data[wh]
+ meanstore[i] = ma.average(datam)
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i = i+1
+
+ elif data.ndim==3: # 3D data arrays, i.e. 2D time series
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
+ # Calculate the means across each unique time unit
+ i=0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ wh = timeunits==myunit
+ datam = data[wh,:,:]
+ meanstore[i,:,:] = ma.average(datam,axis=0)
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit)==4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit)==6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit)==8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit)==3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1]-dateList[0]
+ halfway = dateList[0]+(dt/2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+ newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+ elif processing_required:
+ meanstorem = meanstore
+
+ print '*** End calc_average_on_new_time_unit_KK ***'
+ return meanstorem,newTimesList
[11/50] [abbrv] git commit: mccserach - some linting
Posted by le...@apache.org.
mccserach - some linting
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/0b6b8a27
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/0b6b8a27
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/0b6b8a27
Branch: refs/heads/master
Commit: 0b6b8a279b98210f96ecdb88020e2c1d046b25c5
Parents: 0747070
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 09:09:25 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Sun Apr 13 09:09:25 2014 -0400
----------------------------------------------------------------------
mccsearch/code/mainProg.py | 84 +++++++++++------------------------------
1 file changed, 22 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/0b6b8a27/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 90fb76d..9be65b5 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -1,14 +1,10 @@
'''
# running the program
-# Kim Whitehall Nov 2012
-# Last updated: 15 May 2013
'''
-#import rcmes
import sys
import networkx as nx
import mccSearch
-#import cloudclusters
import numpy as np
import numpy.ma as ma
import files
@@ -24,43 +20,43 @@ def main():
MCSMCCNodesList =[]
allMCSsList =[]
allCETRMMList =[]
+
+ #for GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy1"
- #mainDirStr= "/home/whitehal/mccsearch/caseStudy2"
- #directories for the original data
- #TRMMdirName ="/home/whitehal/summer2006TRMM"
- #CEoriDirName = "/home/whitehal/17Jul30Sep/mergNETCDF"
- TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
- CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
- #allMCSsDict ={{}}
+ mainDirStr= "/directory/to/where/to/store/outputs"
+ TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+ #for first time working with the raw MERG zipped files
# mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # sys.exit()
+ # ---------------------------------------------------------------------------------
- # # mccSearch.plotAccTRMM(TRMMCEdirName)
- # sys.exit()
#create main directory and file structure for storing intel
mccSearch.createMainDirectory(mainDirStr)
TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
CEdirName = mainDirStr+'/MERGnetcdfCEs'
- # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ # mccSearch.postProcessingNetCDF(3,CEoriDirName)
# mccSearch.postProcessingNetCDF(2)
- # sys.exit()
+ # -------------------------------------------------------------------------------------------------
- #using merg data ***********
+ #let's go!
print "\n -------------- Read MERG Data ----------"
mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
print ("-"*80)
+ print 'in main', len(mergImgs)
+ #print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
print "\n -------------- TESTING findCloudElements ----------"
CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName isnt entered for what ever reason
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
# CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
# allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
- #sys.exit()
+ # ----------------------------------------------------------------------------------------------
print ("-"*80)
print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
print ("-"*80)
@@ -74,32 +70,15 @@ def main():
MCCList,MCSList = mccSearch.findMCC(prunedGraph)
print ("-"*80)
print "MCC List has been acquired ", len(MCCList)
- # for eachList in MCCList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
print "MCS List has been acquired ", len(MCSList)
- # for eachList in MCSList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING TRMM ----------"
#now ready to perform various calculations/metrics
print "\n -------------- TESTING METRICS ----------"
- #MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
- print ("-"*80)
- #print "MCCTimes is: ", MCCTimes
- #mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- # mccSearch.displaySize(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistogram(MCCList)
- print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
- print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ #some calculations/metrics that work that work
+ # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
# print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
# print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
# print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
@@ -107,33 +86,14 @@ def main():
# print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
# print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
- # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
- # print "Histogram is ", hist, bin_edges
- # plt.bar(bin_edges[:-1], hist, width = 10)
- # plt.xlim(min(bin_edges), max(bin_edges))
- # plt.show()
-
#some plots that work
# mccSearch.plotAccTRMM(MCCList)
# mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
# mccSearch.displaySize(MCCList)
# mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistograms(MCCList)
- #sys.exit()
+ # mccSearch.plotHistogram(MCCList)
#
print ("-"*80)
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print "********** findPrecipRate found"
- # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print ("-"*80)
- # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
- # print ("-"*80)
- # # sys.exit()
main()
\ No newline at end of file
[19/50] [abbrv] git commit: fixed bug regarding ability to access a
subset of files in a folder
Posted by le...@apache.org.
fixed bug regarding ability to access a subset of files in a folder
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/21c1c04a
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/21c1c04a
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/21c1c04a
Branch: refs/heads/master
Commit: 21c1c04a2579f80a41d1b55022f2ee2ebee7df38
Parents: 0d1f3d0
Author: georgette <k_...@yahoo.com>
Authored: Tue Jun 3 09:39:06 2014 -0400
Committer: georgette <k_...@yahoo.com>
Committed: Tue Jun 3 09:39:06 2014 -0400
----------------------------------------------------------------------
.DS_Store | Bin 0 -> 6148 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/21c1c04a/.DS_Store
----------------------------------------------------------------------
diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000..73f6381
Binary files /dev/null and b/.DS_Store differ
[16/50] [abbrv] git commit: mccSearchUI.py
Posted by le...@apache.org.
mccSearchUI.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/f21e24d6
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/f21e24d6
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/f21e24d6
Branch: refs/heads/master
Commit: f21e24d6636480792fb44eb3df3c34e12495efb8
Parents: 63e7d5a
Author: kwhitehall <k_...@yahoo.com>
Authored: Thu May 15 15:04:28 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Thu May 15 15:04:28 2014 -0400
----------------------------------------------------------------------
mccsearch/GrADsScripts/TRMMCE.gs | 15 -----
mccsearch/GrADsScripts/cs2.gs | 104 --------------------------------
mccsearch/GrADsScripts/cs4.gs | 27 ---------
mccsearch/GrADsScripts/infrared.gs | 88 ---------------------------
mccsearch/GrADsScripts/mergeCE.gs | 88 ---------------------------
5 files changed, 322 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/f21e24d6/mccsearch/GrADsScripts/TRMMCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/TRMMCE.gs b/mccsearch/GrADsScripts/TRMMCE.gs
deleted file mode 100644
index cd1dbd6..0000000
--- a/mccsearch/GrADsScripts/TRMMCE.gs
+++ /dev/null
@@ -1,15 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/20090831000000F1CE1.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-'set datawarn off'
-'set gxout shaded on'
-'set csmooth on'
-
-'d precipAcc'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/20090831000000F1CE1.gif x800 y600 white'
-'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/f21e24d6/mccsearch/GrADsScripts/cs2.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs2.gs b/mccsearch/GrADsScripts/cs2.gs
deleted file mode 100644
index 9edcf5a..0000000
--- a/mccsearch/GrADsScripts/cs2.gs
+++ /dev/null
@@ -1,104 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE1.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE2.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE3.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE4.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE5.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE6.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE7.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE8.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE9.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-*'set lat 12.0234 16.8594'
-*'set lon -7.98438 7.91406'
-
-*set colors taken from colormap.gs
-*Blues
- 'set rgb 16 255 255 255'
- 'set rgb 17 108 20 156'
- 'set rgb 18 77 50 183'
- 'set rgb 19 48 83 213'
- 'set rgb 20 22 107 236'
- 'set rgb 21 0 193 254'
- 'set rgb 22 42 166 255'
- 'set rgb 23 66 197 249'
- 'set rgb 24 92 226 255'
- 'set rgb 25 124 255 249'
-
-*Greens
- 'set rgb 26 132 252 204'
- 'set rgb 27 135 252 145'
- 'set rgb 28 151 255 130'
- 'set rgb 29 209 255 128'
-
-*Yellows/Reds
- 'set rgb 30 255 246 117'
- 'set rgb 31 255 189 58'
- 'set rgb 32 249 136 6'
- 'set rgb 33 241 110 0'
- 'set rgb 34 212 93 1'
- 'set rgb 35 208 68 0'
- 'set rgb 36 182 48 10'
- 'set rgb 37 163 29 2'
- 'set rgb 38 138 15 0'
- 'set rgb 39 255 255 255'
-
-
- set_plot(198,312,5)
- say '-------------------------------'
- say 'Color Scale set to Color Infrared 1.'
- say '-------------------------------'
-
-*'set gxout shaded on'
-
-*'run cbarn'
-
-'d ch4.9'
-'d ch4.8'
-'d ch4.7'
-'d ch4.6'
-'d ch4.5'
-'d ch4.4'
-'d ch4.3'
-'d ch4.2'
-'d ch4.1'
-
-
-
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-08-31_23:00:00F7.gif x800 y600 white'
-'quit'
-
-
-
-
-
-function set_plot(min,max,int)
-
- value = min
- cval=16
- c_levs = ''
- c_cols = ''
-
- while( value <= max )
- c_levs = c_levs ' ' value
- c_cols = c_cols ' ' cval
- value = value + int
- cval=cval+1
- endwhile
- c_cols=c_cols' 'cval-1
-
- say '-------------------------------'
- say 'Contour levels set to: 'c_levs
- say 'Color Values set to: 'c_cols
- say '-------------------------------'
-
- 'set clevs 'c_levs
- 'set ccols 'c_cols
-
-return
-
http://git-wip-us.apache.org/repos/asf/climate/blob/f21e24d6/mccsearch/GrADsScripts/cs4.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs4.gs b/mccsearch/GrADsScripts/cs4.gs
deleted file mode 100644
index e6396f7..0000000
--- a/mccsearch/GrADsScripts/cs4.gs
+++ /dev/null
@@ -1,27 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE1.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE2.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE3.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE4.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE5.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE6.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE7.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-'set datawarn off'
-'set gxout shaded on'
-'set csmooth on'
-
-'d precipAcc.7'
-'d precipAcc.6'
-'d precipAcc.5'
-'d precipAcc.4'
-'d precipAcc.3'
-'d precipAcc.2'
-'d precipAcc.1'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/TRMM2009-09-01_23:00:00F48.gif x800 y600 white'
-'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/f21e24d6/mccsearch/GrADsScripts/infrared.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/infrared.gs b/mccsearch/GrADsScripts/infrared.gs
deleted file mode 100644
index 5c22b96..0000000
--- a/mccsearch/GrADsScripts/infrared.gs
+++ /dev/null
@@ -1,88 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF/ctlFiles/merg_2009090123_4km-pixel.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-*'set lat 12.0234 16.8594'
-*'set lon -7.98438 7.91406'
-
-*set colors taken from colormap.gs
-*Blues
- 'set rgb 16 255 255 255'
- 'set rgb 17 108 20 156'
- 'set rgb 18 77 50 183'
- 'set rgb 19 48 83 213'
- 'set rgb 20 22 107 236'
- 'set rgb 21 0 193 254'
- 'set rgb 22 42 166 255'
- 'set rgb 23 66 197 249'
- 'set rgb 24 92 226 255'
- 'set rgb 25 124 255 249'
-
-*Greens
- 'set rgb 26 132 252 204'
- 'set rgb 27 135 252 145'
- 'set rgb 28 151 255 130'
- 'set rgb 29 209 255 128'
-
-*Yellows/Reds
- 'set rgb 30 255 246 117'
- 'set rgb 31 255 189 58'
- 'set rgb 32 249 136 6'
- 'set rgb 33 241 110 0'
- 'set rgb 34 212 93 1'
- 'set rgb 35 208 68 0'
- 'set rgb 36 182 48 10'
- 'set rgb 37 163 29 2'
- 'set rgb 38 138 15 0'
- 'set rgb 39 255 255 255'
-
-
- set_plot(198,312,5)
- say '-------------------------------'
- say 'Color Scale set to Color Infrared 1.'
- say '-------------------------------'
-
-*'set gxout shaded on'
-
-*'run cbarn'
-
-
-
-
-
-'d ch4'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/merg_2009090123_4km-pixel.gif x800 y600 white'
-'quit'
-
-
-
-
-function set_plot(min,max,int)
-
- value = min
- cval=16
- c_levs = ''
- c_cols = ''
-
- while( value <= max )
- c_levs = c_levs ' ' value
- c_cols = c_cols ' ' cval
- value = value + int
- cval=cval+1
- endwhile
- c_cols=c_cols' 'cval-1
-
- say '-------------------------------'
- say 'Contour levels set to: 'c_levs
- say 'Color Values set to: 'c_cols
- say '-------------------------------'
-
- 'set clevs 'c_levs
- 'set ccols 'c_cols
-
-return
-
http://git-wip-us.apache.org/repos/asf/climate/blob/f21e24d6/mccsearch/GrADsScripts/mergeCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/mergeCE.gs b/mccsearch/GrADsScripts/mergeCE.gs
deleted file mode 100644
index b89945c..0000000
--- a/mccsearch/GrADsScripts/mergeCE.gs
+++ /dev/null
@@ -1,88 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-09-01_00:00:00F8CE5.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-*'set lat 12.0234 16.8594'
-*'set lon -7.98438 7.91406'
-
-*set colors taken from colormap.gs
-*Blues
- 'set rgb 16 255 255 255'
- 'set rgb 17 108 20 156'
- 'set rgb 18 77 50 183'
- 'set rgb 19 48 83 213'
- 'set rgb 20 22 107 236'
- 'set rgb 21 0 193 254'
- 'set rgb 22 42 166 255'
- 'set rgb 23 66 197 249'
- 'set rgb 24 92 226 255'
- 'set rgb 25 124 255 249'
-
-*Greens
- 'set rgb 26 132 252 204'
- 'set rgb 27 135 252 145'
- 'set rgb 28 151 255 130'
- 'set rgb 29 209 255 128'
-
-*Yellows/Reds
- 'set rgb 30 255 246 117'
- 'set rgb 31 255 189 58'
- 'set rgb 32 249 136 6'
- 'set rgb 33 241 110 0'
- 'set rgb 34 212 93 1'
- 'set rgb 35 208 68 0'
- 'set rgb 36 182 48 10'
- 'set rgb 37 163 29 2'
- 'set rgb 38 138 15 0'
- 'set rgb 39 255 255 255'
-
-
- set_plot(198,312,5)
- say '-------------------------------'
- say 'Color Scale set to Color Infrared 1.'
- say '-------------------------------'
-
-*'set gxout shaded on'
-
-*'run cbarn'
-
-'d ch4'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-09-01_00:00:00F8CE5.gif x800 y600 white'
-'quit'
-
-
-
-
-
-
-
-
-function set_plot(min,max,int)
-
- value = min
- cval=16
- c_levs = ''
- c_cols = ''
-
- while( value <= max )
- c_levs = c_levs ' ' value
- c_cols = c_cols ' ' cval
- value = value + int
- cval=cval+1
- endwhile
- c_cols=c_cols' 'cval-1
-
- say '-------------------------------'
- say 'Contour levels set to: 'c_levs
- say 'Color Values set to: 'c_cols
- say '-------------------------------'
-
- 'set clevs 'c_levs
- 'set ccols 'c_cols
-
-return
-
[23/50] [abbrv] Fix formatting, remove unused imports,
update files.py and process.py to most recent OCW code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/code/process.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.py b/mccsearch/code/process.py
index 5a362ec..7367ee7 100644
--- a/mccsearch/code/process.py
+++ b/mccsearch/code/process.py
@@ -1,4 +1,20 @@
-"""Collection of functions that process numpy arrays of varying shapes.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Collection of functions that process numpy arrays of varying shapes.
Functions can range from subsetting to regridding"""
# Python Standard Libary Imports
@@ -10,38 +26,38 @@ import sys
import time
# 3rd Party Imports
-import Nio
+import netCDF4
import numpy as np
import numpy.ma as ma
-from scipy.ndimage import map_coordinates
+
def extract_subregion_from_data_array(data, lats, lons, latmin, latmax, lonmin, lonmax):
'''
Extract a sub-region from a data array.
-
+
Example::
**e.g.** the user may load a global model file, but only want to examine data over North America
This function extracts a sub-domain from the original data.
The defined sub-region must be a regular lat/lon bounding box,
but the model data may be on a non-regular grid (e.g. rotated, or Guassian grid layout).
- Data are kept on the original model grid and a rectangular (in model-space) region
+ Data are kept on the original model grid and a rectangular (in model-space) region
is extracted which contains the rectangular (in lat/lon space) user supplied region.
-
+
INPUT::
data - 3d masked data array
lats - 2d array of latitudes corresponding to data array
lons - 2d array of longitudes corresponding to data array
latmin, latmax, lonmin, lonmax - bounding box of required region to extract
-
+
OUTPUT::
data2 - subset of original data array containing only data from required subregion
lats2 - subset of original latitude data array
lons2 - subset of original longitude data array
'''
-
-
+
+
# Mask model data array to find grid points inside the supplied bounding box
whlat = (lats > latmin) & (lats < latmax)
@@ -67,37 +83,37 @@ def extract_subregion_from_data_array(data, lats, lons, latmin, latmax, lonmin,
def calc_area_mean(data, lats, lons, mymask='not set'):
'''
Calculate Area Average of data in a masked array
-
+
INPUT::
data: a masked array of data (NB. only data from one time expected to be passed at once)
lats: 2d array of regularly gridded latitudes
lons: 2d array of regularly gridded longitudes
mymask: (optional) defines spatial region to do averaging over
-
+
OUTPUT::
area_mean: a value for the mean inside the area
'''
-
-
+
+
# If mask not passed in, then set maks to cover whole data domain
if(mymask == 'not set'):
- mymask = np.empty(data.shape)
- mymask[:] = False # NB. mask means (don't show), so False everywhere means use everything.
-
+ mymask = np.empty(data.shape)
+ mymask[:] = False # NB. mask means (don't show), so False everywhere means use everything.
+
# Dimension check on lats, lons
# Sometimes these arrays are 3d, sometimes 2d, sometimes 1d
# This bit of code just converts to the required 2d array shape
if(len(lats.shape) == 3):
- lats = lats[0, :, :]
+ lats = lats[0, :, :]
if(len(lons.shape) == 3):
- lons = lons[0, :, :]
+ lons = lons[0, :, :]
if(np.logical_and(len(lats.shape) == 1, len(lons.shape) == 1)):
- lons, lats = np.meshgrid(lons, lats)
-
+ lons, lats = np.meshgrid(lons, lats)
+
# Calculate grid length (assuming regular lat/lon grid)
dlat = lats[1, 0] - lats[0, 0]
dlon = lons[0, 1] - lons[0, 0]
@@ -108,28 +124,28 @@ def calc_area_mean(data, lats, lons, mymask='not set'):
# Create a new masked array covering just user selected area (defined in mymask)
# NB. this preserves missing data points in the observations data
subdata = ma.masked_array(data, mask=mymask)
-
+
if(myweights.shape != subdata.shape):
- myweights.resize(subdata.shape)
- myweights[1:, :] = myweights[0, :]
+ myweights.resize(subdata.shape)
+ myweights[1:, :] = myweights[0, :]
# Calculate weighted mean using ma.average (which takes weights)
area_mean = ma.average(subdata, weights=myweights)
-
+
return area_mean
def calc_area_in_grid_box(latitude, dlat, dlon):
'''
Calculate area of regular lat-lon grid box
-
+
INPUT::
latitude: latitude of grid box (degrees)
dlat: grid length in latitude direction (degrees)
dlon: grid length in longitude direction (degrees)
-
+
OUTPUT::
A: area of the grid box
-
+
'''
R = 6371000 # radius of Earth in metres
@@ -141,422 +157,286 @@ def calc_area_in_grid_box(latitude, dlat, dlon):
return A
-def do_regrid(q, lat, lon, lat2, lon2, order=1, mdi= -999999999):
+def do_regrid(q, lat, lon, lat2, lon2, order=1, mdi=-999999999):
+ """
+ This function has been moved to the ocw/dataset_processor module
+ """
+ from ocw import dataset_processor
+ q2 = dataset_processor._rcmes_spatial_regrid(q, lat, lon, lat2, lon2, order=1)
+
+ return q2
+
+def create_mask_using_threshold(masked_array, threshold=0.5):
+ """
+ .. deprecated:: 0.3-incubating
+ Use :func:`ocw.dataset_processor._rcmes_create_mask_using_threshold` instead.
+ """
+ from ocw import dataset_processor
+ mask = dataset_processor._rcmes_create_mask_using_threshold(masked_array, threshold)
+
+ return mask
+
+def calc_average_on_new_time_unit(data, dateList, unit='monthly'):
'''
- Perform regridding from one set of lat,lon values onto a new set (lat2,lon2)
-
+ Routine to calculate averages on longer time units than the data exists on.
+
+ Example::
+ If the data is 6-hourly, calculate daily, or monthly means.
+
Input::
- q - the variable to be regridded
- lat,lon - original co-ordinates corresponding to q values
- lat2,lon2 - new set of latitudes and longitudes that you want to regrid q onto
- order - (optional) interpolation order 1=bi-linear, 3=cubic spline
- mdi - (optional) fill value for missing data (used in creation of masked array)
-
- Output::
- q2 - q regridded onto the new set of lat2,lon2
-
+ data - data values
+ dateList - list of python datetime structures corresponding to data values
+ unit - string describing time unit to average onto.
+ *Valid values are: 'monthly', 'daily', 'pentad','annual','decadal'*
+
+ Output:
+ meanstorem - numpy masked array of data values meaned over required time period
+ newTimesList - a list of python datetime objects representing the data in the new averagin period
+ *NB.* currently set to beginning of averaging period,
+ **i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.**
'''
- nlat = q.shape[0]
- nlon = q.shape[1]
-
- nlat2 = lat2.shape[0]
- nlon2 = lon2.shape[1]
-
- # To make our lives easier down the road, let's
- # turn these into arrays of x & y coords
- loni = lon2.ravel()
- lati = lat2.ravel()
-
- loni = loni.copy() # NB. it won't run unless you do this...
- lati = lati.copy()
-
- # Now, we'll set points outside the boundaries to lie along an edge
- loni[loni > lon.max()] = lon.max()
- loni[loni < lon.min()] = lon.min()
-
- # To deal with the "hard" break, we'll have to treat y differently,
- # so we're just setting the min here...
- lati[lati > lat.max()] = lat.max()
- lati[lati < lat.min()] = lat.min()
-
-
- # We need to convert these to (float) indicies
- # (xi should range from 0 to (nx - 1), etc)
- loni = (nlon - 1) * (loni - lon.min()) / (lon.max() - lon.min())
-
- # Deal with the "hard" break in the y-direction
- lati = (nlat - 1) * (lati - lat.min()) / (lat.max() - lat.min())
-
- # Notes on dealing with MDI when regridding data.
- # Method adopted here:
- # Use bilinear interpolation of data by default (but user can specify other order using order=... in call)
- # Perform bilinear interpolation of data, and of mask.
- # To be conservative, new grid point which contained some missing data on the old grid is set to missing data.
- # -this is achieved by looking for any non-zero interpolated mask values.
- # To avoid issues with bilinear interpolation producing strong gradients leading into the MDI,
- # set values at MDI points to mean data value so little gradient visible = not ideal, but acceptable for now.
-
- # Set values in MDI so that similar to surroundings so don't produce large gradients when interpolating
- # Preserve MDI mask, by only changing data part of masked array object.
- for shift in (-1, 1):
- for axis in (0, 1):
- q_shifted = np.roll(q, shift=shift, axis=axis)
- idx = ~q_shifted.mask * q.mask
- q.data[idx] = q_shifted[idx]
-
- # Now we actually interpolate
- # map_coordinates does cubic interpolation by default,
- # use "order=1" to preform bilinear interpolation instead...
- q2 = map_coordinates(q, [lati, loni], order=order)
- q2 = q2.reshape([nlat2, nlon2])
-
- # Set values to missing data outside of original domain
- q2 = ma.masked_array(q2, mask=np.logical_or(np.logical_or(lat2 >= lat.max(),
- lat2 <= lat.min()),
- np.logical_or(lon2 <= lon.min(),
- lon2 >= lon.max())))
-
- # Make second map using nearest neighbour interpolation -use this to determine locations with MDI and mask these
- qmdi = np.zeros_like(q)
- qmdi[q.mask == True] = 1.
- qmdi[q.mask == False] = 0.
- qmdi_r = map_coordinates(qmdi, [lati, loni], order=order)
- qmdi_r = qmdi_r.reshape([nlat2, nlon2])
- mdimask = (qmdi_r != 0.0)
-
- # Combine missing data mask, with outside domain mask define above.
- q2.mask = np.logical_or(mdimask, q2.mask)
+ # First catch unknown values of time unit passed in by user
+ acceptable = (unit == 'full') | (unit == 'annual') | (unit == 'monthly') | (unit == 'daily') | (unit == 'pentad')
- return q2
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging'
+ print ' Please check your code.'
+ return
-def create_mask_using_threshold(masked_array, threshold=0.5):
- '''
- Routine to create a mask, depending on the proportion of times with missing data.
- For each pixel, calculate proportion of times that are missing data,
- **if** the proportion is above a specified threshold value,
- then mark the pixel as missing data.
-
- Input::
- masked_array - a numpy masked array of data (assumes time on axis 0, and space on axes 1 and 2).
- threshold - (optional) threshold proportion above which a pixel is marked as 'missing data'.
- NB. default threshold = 50%
-
- Output::
- mymask - a numpy array describing the mask. NB. not a masked array, just the mask itself.
+ # Calculate arrays of years (2007,2007),
+ # yearsmonths (200701,200702),
+ # or yearmonthdays (20070101,20070102)
+ # -depending on user selected averaging period.
- '''
-
+ # Year list
+ if unit == 'annual':
+ print 'Calculating annual mean data'
+ timeunits = []
- # try, except used as some model files don't have a full mask, but a single bool
- # the except catches this situation and deals with it appropriately.
- try:
- nT = masked_array.mask.shape[0]
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year))
- # For each pixel, count how many times are masked.
- nMasked = masked_array.mask[:, :, :].sum(axis=0)
+ timeunits = np.array(timeunits, dtype=int)
- # Define new mask as when a pixel has over a defined threshold ratio of masked data
- # e.g. if the threshold is 75%, and there are 10 times,
- # then a pixel will be masked if more than 5 times are masked.
- mymask = nMasked > (nT * threshold)
+ # YearMonth format list
+ if unit == 'monthly':
+ print 'Calculating monthly mean data'
+ timeunits = []
- except:
- mymask = np.zeros_like(masked_array.data[0, :, :])
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
- return mymask
+ timeunits = np.array(timeunits, dtype=int)
-def calc_average_on_new_time_unit(data, dateList, unit='monthly'):
- '''
- Routine to calculate averages on longer time units than the data exists on.
-
- Example::
- If the data is 6-hourly, calculate daily, or monthly means.
-
- Input::
- data - data values
- dateList - list of python datetime structures corresponding to data values
- unit - string describing time unit to average onto.
- *Valid values are: 'monthly', 'daily', 'pentad','annual','decadal'*
-
- Output:
- meanstorem - numpy masked array of data values meaned over required time period
- newTimesList - a list of python datetime objects representing the data in the new averagin period
- *NB.* currently set to beginning of averaging period,
- **i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.**
- '''
+ # YearMonthDay format list
+ if unit == 'daily':
+ print 'Calculating daily mean data'
+ timeunits = []
- # First catch unknown values of time unit passed in by user
- acceptable = (unit == 'full') | (unit == 'annual') | (unit == 'monthly') | (unit == 'daily') | (unit == 'pentad')
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + \
+ str("%02d" % dateList[i].day))
- if not acceptable:
- print 'Error: unknown unit type selected for time averaging'
- print ' Please check your code.'
- return
+ timeunits = np.array(timeunits, dtype=int)
- # Calculate arrays of years (2007,2007),
- # yearsmonths (200701,200702),
- # or yearmonthdays (20070101,20070102)
- # -depending on user selected averaging period.
- # Year list
- if unit == 'annual':
- print 'Calculating annual mean data'
- timeunits = []
+ # TODO: add pentad setting using Julian days?
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year))
+ # Full list: a special case
+ if unit == 'full':
+ print 'Calculating means data over full time range'
+ timeunits = []
- timeunits = np.array(timeunits, dtype=int)
-
- # YearMonth format list
- if unit == 'monthly':
- print 'Calculating monthly mean data'
- timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+ timeunits = np.array(timeunits, dtype=int)
- timeunits = np.array(timeunits, dtype=int)
- # YearMonthDay format list
- if unit == 'daily':
- print 'Calculating daily mean data'
- timeunits = []
-
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + \
- str("%02d" % dateList[i].day))
- timeunits = np.array(timeunits, dtype=int)
+ # empty list to store new times
+ newTimesList = []
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and
+ # calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits) == (len(np.unique(timeunits))):
+ processing_required = False
- # TODO: add pentad setting using Julian days?
+ # 1D data arrays, i.e. time series
+ if data.ndim == 1:
+ # Create array to store the resulting data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+ # Calculate the means across each unique time unit
+ i = 0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ datam = ma.masked_array(data, timeunits != myunit)
+ meanstore[i] = ma.average(datam)
- # Full list: a special case
- if unit == 'full':
- print 'Calculating means data over full time range'
- timeunits = []
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
- for i in np.arange(len(dateList)):
- timeunits.append(999) # i.e. we just want the same value for all times.
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+ i = i + 1
- timeunits = np.array(timeunits, dtype=int)
+ # 3D data arrays
+ if data.ndim == 3:
+ #datamask = create_mask_using_threshold(data,threshold=0.75)
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)), data.shape[1], data.shape[2]])
- # empty list to store new times
- newTimesList = []
+ # Calculate the means across each unique time unit
+ i = 0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
- # Decide whether or not you need to do any time averaging.
- # i.e. if data are already on required time unit then just pass data through and
- # calculate and return representative datetimes.
- processing_required = True
- if len(timeunits) == (len(np.unique(timeunits))):
- processing_required = False
+ mask = np.zeros_like(data)
+ mask[timeunits != myunit, :, :] = 1.0
- # 1D data arrays, i.e. time series
- if data.ndim == 1:
- # Create array to store the resulting data
- meanstore = np.zeros(len(np.unique(timeunits)))
-
- # Calculate the means across each unique time unit
- i = 0
- for myunit in np.unique(timeunits):
- if processing_required:
- datam = ma.masked_array(data, timeunits != myunit)
- meanstore[i] = ma.average(datam)
-
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit) == 4: # YYYY
- yyyy = int(myunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit) == 6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit) == 8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit) == 3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1] - dateList[0]
- halfway = dateList[0] + (dt / 2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
-
- newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
- i = i + 1
-
- # 3D data arrays
- if data.ndim == 3:
-
- #datamask = create_mask_using_threshold(data,threshold=0.75)
-
- # Create array to store the resulting data
- meanstore = np.zeros([len(np.unique(timeunits)), data.shape[1], data.shape[2]])
-
- # Calculate the means across each unique time unit
- i = 0
- datamask_store = []
- for myunit in np.unique(timeunits):
- if processing_required:
-
- mask = np.zeros_like(data)
- mask[timeunits != myunit, :, :] = 1.0
-
- # Calculate missing data mask within each time unit...
- datamask_at_this_timeunit = np.zeros_like(data)
- datamask_at_this_timeunit[:] = create_mask_using_threshold(data[timeunits == myunit, :, :], threshold=0.75)
- # Store results for masking later
- datamask_store.append(datamask_at_this_timeunit[0])
-
- # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
- datam = ma.masked_array(data, np.logical_or(mask, datamask_at_this_timeunit))
- meanstore[i, :, :] = ma.average(datam, axis=0)
-
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit) == 4: # YYYY
- yyyy = int(smyunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit) == 6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit) == 8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit) == 3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1] - dateList[0]
- halfway = dateList[0] + (dt / 2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
-
- newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
-
- i += 1
-
- if not processing_required:
- meanstorem = data
-
- if processing_required:
- # Create masked array (using missing data mask defined above)
- datamask_store = np.array(datamask_store)
- meanstorem = ma.masked_array(meanstore, datamask_store)
-
- return meanstorem, newTimesList
+ # Calculate missing data mask within each time unit...
+ datamask_at_this_timeunit = np.zeros_like(data)
+ datamask_at_this_timeunit[:] = create_mask_using_threshold(data[timeunits == myunit, :, :], threshold=0.75)
+ # Store results for masking later
+ datamask_store.append(datamask_at_this_timeunit[0])
+
+ # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
+ datam = ma.masked_array(data, np.logical_or(mask, datamask_at_this_timeunit))
+ meanstore[i, :, :] = ma.average(datam, axis=0)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+
+ if processing_required:
+ # Create masked array (using missing data mask defined above)
+ datamask_store = np.array(datamask_store)
+ meanstorem = ma.masked_array(meanstore, datamask_store)
+
+ return meanstorem, newTimesList
def calc_running_accum_from_period_accum(data):
'''
Routine to calculate running total accumulations from individual period accumulations.
::
-
+
e.g. 0,0,1,0,0,2,2,1,0,0
-> 0,0,1,1,1,3,5,6,6,6
-
+
Input::
data: numpy array with time in the first axis
-
+
Output::
running_acc: running accumulations
-
+
'''
-
-
+
+
running_acc = np.zeros_like(data)
-
+
if(len(data.shape) == 1):
running_acc[0] = data[0]
-
+
if(len(data.shape) > 1):
running_acc[0, :] = data[0, :]
-
+
for i in np.arange(data.shape[0] - 1):
if(len(data.shape) == 1):
running_acc[i + 1] = running_acc[i] + data[i + 1]
-
+
if(len(data.shape) > 1):
running_acc[i + 1, :] = running_acc[i, :] + data[i + 1, :]
-
+
return running_acc
def ignore_boundaries(data, rim=10):
'''
Routine to mask the lateral boundary regions of model data to ignore them from calculations.
-
+
Input::
data - a masked array of model data
rim - (optional) number of grid points to ignore
-
+
Output::
data - data array with boundary region masked
-
+
'''
-
+
nx = data.shape[1]
ny = data.shape[0]
-
+
rimmask = np.zeros_like(data)
for j in np.arange(rim):
rimmask[j, 0:nx - 1] = 1.0
-
+
for j in ny - 1 - np.arange(rim):
rimmask[j, 0:nx - 1] = 1.0
-
+
for i in np.arange(rim):
rimmask[0:ny - 1, i] = 1.0
-
+
for i in nx - 1 - np.arange(rim):
rimmask[0:ny - 1, i] = 1.0
-
+
data = ma.masked_array(data, mask=rimmask)
-
+
return data
def normalizeDatetimes(datetimes, timestep):
- """
- Input::
- datetimes - list of datetime objects that need to be normalized
- timestep - string of value ('daily' | 'monthly')
- Output::
- normalDatetimes - list of datetime objects that have been normalized
-
- Normalization Rules::
- Daily data will be forced to an hour value of 00:00:00
- Monthly data will be forced to the first of the month at midnight
- """
- normalDatetimes = []
- if timestep.lower() == 'monthly':
- for inputDatetime in datetimes:
- if inputDatetime.day != 1:
- # Clean the inputDatetime
- inputDatetimeString = inputDatetime.strftime('%Y%m%d')
- normalInputDatetimeString = inputDatetimeString[:6] + '01'
- inputDatetime = datetime.datetime.strptime(normalInputDatetimeString, '%Y%m%d')
-
- normalDatetimes.append(inputDatetime)
-
- elif timestep.lower() == 'daily':
- for inputDatetime in datetimes:
- if inputDatetime.hour != 0 or inputDatetime.minute != 0 or inputDatetime.second != 0:
- datetimeString = inputDatetime.strftime('%Y%m%d%H%M%S')
- normalDatetimeString = datetimeString[:8] + '000000'
- inputDatetime = datetime.datetime.strptime(normalDatetimeString, '%Y%m%d%H%M%S')
-
- normalDatetimes.append(inputDatetime)
-
- return normalDatetimes
+ """ This function has been moved to the ocw/dataset_processor module """
+ from ocw import dataset_processor as dsp
+ return dsp._rcmes_normalize_datetimes(datetimes, timestep)
def getModelTimes(modelFile, timeVarName):
'''
@@ -573,21 +453,19 @@ def getModelTimes(modelFile, timeVarName):
modelTimeStep - 'hourly','daily','monthly','annual'
'''
- f = Nio.open_file(modelFile)
+ f = netCDF4.Dataset(modelFile, mode='r')
xtimes = f.variables[timeVarName]
- timeFormat = xtimes.attributes['units']
-
+ timeFormat = xtimes.units.encode()
+
# search to check if 'since' appears in units
try:
- sinceLoc = re.search('since', timeFormat).end()
-
+ sinceLoc = re.search('since', timeFormat).end()
- #KDW the below block generates and error. But the print statement, indicates that sinceLoc found something
except AttributeError:
- print 'Error decoding model times: time variable attributes do not contain "since"'
- raise
+ print 'Error decoding model times: time variable attributes do not contain "since"'
+ raise
- units = ''
+ units = None
TIME_UNITS = ('minutes', 'hours', 'days', 'months', 'years')
# search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
for unit in TIME_UNITS:
@@ -597,17 +475,14 @@ def getModelTimes(modelFile, timeVarName):
# cut out base time (the bit following 'since')
base_time_string = string.lstrip(timeFormat[sinceLoc:])
-
# decode base time
base_time = decodeTimeFromString(base_time_string)
-
times = []
-
-
for xtime in xtimes[:]:
-
- # Cast time as an int ***KDW remove this so fractional xtime can be read from MERG
+
+ # Cast time as an int
+ #TODO: KDW this may cause problems for data that is hourly with more than one timestep in it
xtime = int(xtime)
if units == 'minutes':
@@ -631,23 +506,24 @@ def getModelTimes(modelFile, timeVarName):
elif units == 'years':
dt = datetime.timedelta(years=xtime)
new_time = base_time + dt
-
- #print "xtime is:", xtime, "dt is: ", dt
-
-
times.append(new_time)
-
+
try:
- timeStepLength = int(xtimes[1] - xtimes[0] + 1.e-12)
+ if len(xtimes) == 1:
+ timeStepLength = 0
+ else:
+ timeStepLength = int(xtimes[1] - xtimes[0] + 1.e-12)
+
modelTimeStep = getModelTimeStep(units, timeStepLength)
-
- #KDW if timeStepLength is zero do not normalize times as this would create an empty list
+
+ #if timeStepLength is zero do not normalize times as this would create an empty list for MERG (hourly) data
if timeStepLength != 0:
- times = normalizeDatetimes(times, modelTimeStep)
+ times = normalizeDatetimes(times, modelTimeStep)
except:
raise
+
return times, modelTimeStep
def getModelTimeStep(units, stepSize):
@@ -661,22 +537,22 @@ def getModelTimeStep(units, stepSize):
# 28 days through 31 days
elif 40320 <= stepSize <= 44640:
modelTimeStep = 'monthly'
- # 365 days through 366 days
+ # 365 days through 366 days
elif 525600 <= stepSize <= 527040:
- modelTimeStep = 'annual'
+ modelTimeStep = 'annual'
else:
raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
elif units == 'hours':
- #need a check for fractional hrs and only one hr i.e. stepSize=0
+ #need a check for fractional hrs and only one hr i.e. stepSize=0 e.g. with MERG data
if stepSize == 0 or stepSize == 1:
modelTimeStep = 'hourly'
elif stepSize == 24:
modelTimeStep = 'daily'
elif 672 <= stepSize <= 744:
- modelTimeStep = 'monthly'
+ modelTimeStep = 'monthly'
elif 8760 <= stepSize <= 8784:
- modelTimeStep = 'annual'
+ modelTimeStep = 'annual'
else:
raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
@@ -717,7 +593,7 @@ def decodeTimeFromString(time_string):
::
**Input:** time_string - a string that represents a date/time
-
+
**Output:** mytime - a python datetime object
'''
@@ -774,9 +650,9 @@ def decodeTimeFromString(time_string):
except ValueError:
pass
-#KDW added
+
try:
- mytime = time.strptime(time_string, '%Y-%m-%d')
+ mytime = time.strptime(time_string, '%Y-%m-%d %H')
mytime = datetime.datetime(*mytime[0:6])
return mytime
@@ -784,210 +660,286 @@ def decodeTimeFromString(time_string):
pass
try:
- mytime = time.strptime(time_string, '%Y-%m-%d %H')
+ mytime = time.strptime(time_string, '%Y-%m-%d')
mytime = datetime.datetime(*mytime[0:6])
return mytime
except ValueError:
pass
-
print 'Error decoding time string: string does not match a predefined time format'
return 0
def regrid_wrapper(regrid_choice, obsdata, obslats, obslons, wrfdata, wrflats, wrflons):
- '''
- Wrapper routine for regridding.
-
- Either regrids model to obs grid, or obs to model grid, depending on user choice
-
- Inputs::
- regrid_choice - [0] = Regrid obs data onto model grid or
- [1] = Regrid model data onto obs grid
-
- obsdata,wrfdata - data arrays
- obslats,obslons - observation lat,lon arrays
- wrflats,wrflons - model lat,lon arrays
-
- Output::
- rdata,rdata2 - regridded data
- lats,lons - latitudes and longitudes of regridded data
-
- '''
+ '''
+ Wrapper routine for regridding.
+
+ Either regrids model to obs grid, or obs to model grid, depending on user choice
+
+ Inputs::
+ regrid_choice - [0] = Regrid obs data onto model grid or
+ [1] = Regrid model data onto obs grid
+
+ obsdata,wrfdata - data arrays
+ obslats,obslons - observation lat,lon arrays
+ wrflats,wrflons - model lat,lon arrays
+
+ Output::
+ rdata,rdata2 - regridded data
+ lats,lons - latitudes and longitudes of regridded data
- # Regrid obs data onto model grid
- if(regrid_choice == '0'):
+ '''
+
+ # Regrid obs data onto model grid
+ if(regrid_choice == '0'):
+
+ ndims = len(obsdata.shape)
+ if(ndims == 3):
+ newshape = [obsdata.shape[0], wrfdata.shape[1], wrfdata.shape[2]]
+ nT = obsdata.shape[0]
- ndims = len(obsdata.shape)
- if(ndims == 3):
- newshape = [obsdata.shape[0], wrfdata.shape[1], wrfdata.shape[2]]
- nT = obsdata.shape[0]
+ if(ndims == 2):
+ newshape = [wrfdata.shape[0], wrfdata.shape[1]]
+ nT = 0
- if(ndims == 2):
- newshape = [wrfdata.shape[0], wrfdata.shape[1]]
- nT = 0
+ rdata = wrfdata
+ lats, lons = wrflats, wrflons
- rdata = wrfdata
- lats, lons = wrflats, wrflons
+ # Create a new masked array with the required dimensions
+ tmp = np.zeros(newshape)
+ rdata2 = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
- # Create a new masked array with the required dimensions
- tmp = np.zeros(newshape)
- rdata2 = ma.MaskedArray(tmp, mask=(tmp == 0))
- tmp = 0 # free up some memory
+ rdata2[:] = 0.0
- rdata2[:] = 0.0
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata2[t, :, :] = do_regrid(obsdata[t, :, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
- if(nT > 0):
- for t in np.arange(nT):
- rdata2[t, :, :] = do_regrid(obsdata[t, :, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+ if(nT == 0):
+ rdata2[:, :] = do_regrid(obsdata[:, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
- if(nT == 0):
- rdata2[:, :] = do_regrid(obsdata[:, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+ # Regrid model data onto obs grid
+ if(regrid_choice == '1'):
+ ndims = len(wrfdata.shape)
+ if(ndims == 3):
+ newshape = [wrfdata.shape[0], obsdata.shape[1], obsdata.shape[2]]
+ nT = obsdata.shape[0]
- # Regrid model data onto obs grid
- if(regrid_choice == '1'):
- ndims = len(wrfdata.shape)
- if(ndims == 3):
- newshape = [wrfdata.shape[0], obsdata.shape[1], obsdata.shape[2]]
- nT = obsdata.shape[0]
+ if(ndims == 2):
+ newshape = [obsdata.shape[0], obsdata.shape[1]]
+ nT = 0
- if(ndims == 2):
- newshape = [obsdata.shape[0], obsdata.shape[1]]
- nT = 0
+ rdata2 = obsdata
+ lats, lons = obslats, obslons
- rdata2 = obsdata
- lats, lons = obslats, obslons
+ tmp = np.zeros(newshape)
+ rdata = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
- tmp = np.zeros(newshape)
- rdata = ma.MaskedArray(tmp, mask=(tmp == 0))
- tmp = 0 # free up some memory
+ rdata[:] = 0.0
- rdata[:] = 0.0
-
- if(nT > 0):
- for t in np.arange(nT):
- rdata[t, :, :] = do_regrid(wrfdata[t, :, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata[t, :, :] = do_regrid(wrfdata[t, :, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
- if(nT == 0):
- rdata[:, :] = do_regrid(wrfdata[:, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+ if(nT == 0):
+ rdata[:, :] = do_regrid(wrfdata[:, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
-
- return rdata, rdata2, lats, lons
+
+ return rdata, rdata2, lats, lons
def extract_sub_time_selection(allTimes, subTimes, data):
'''
Routine to extract a sub-selection of times from a data array.
Example::
- Suppose a data array has 30 time values for daily data for a whole month,
+ Suppose a data array has 30 time values for daily data for a whole month,
but you only want the data from the 5th-15th of the month.
-
+
Input::
allTimes - list of datetimes describing what times the data in the data array correspond to
subTimes - the times you want to extract data from
data - the data array
-
+
Output::
subdata - subselection of data array
-
+
'''
# Create new array to store the subselection
subdata = np.zeros([len(subTimes), data.shape[1], data.shape[2]])
-
+
# Loop over all Times and when it is a member of the required subselection, copy the data across
i = 0 # counter for allTimes
subi = 0 # counter for subTimes
for t in allTimes:
- if(np.setmember1d(allTimes, subTimes)):
+ if(np.in1d(allTimes, subTimes)):
subdata[subi, :, :] = data[i, :, :]
subi += 1
i += 1
-
+
return subdata
def calc_average_on_new_time_unit_K(data, dateList, unit):
- '''
- Routine to calculate averages on longer time units than the data exists on.
- e.g. if the data is 6-hourly, calculate daily, or monthly means.
-
- Input:
- data - data values
- dateList - list of python datetime structures corresponding to data values
- unit - string describing time unit to average onto
- e.g. 'monthly', 'daily', 'pentad','annual','decadal'
- Output:
- meanstorem - numpy masked array of data values meaned over required time period
- newTimesList - a list of python datetime objects representing the data in the new averagin period
- NB. currently set to beginning of averaging period,
- i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
- '''
+ """
+ This function has been moved to the ocw/dataset_processor module
+ """
+ from ocw import dataset_processor
+ temporally_regridded_data = dataset_processor._rcmes_calc_average_on_new_time_unit_K(data, dateList, unit)
+ return temporally_regridded_data
+
+def decode_model_timesK(ifile,timeVarName,file_type):
+ #################################################################################################
+ # Convert model times ('hours since 1900...', 'days since ...') into a python datetime structure
+ # Input:
+ # filelist - list of model files
+ # timeVarName - name of the time variable in the model files
+ # Output:
+ # times - list of python datetime objects describing model data times
+ # Peter Lean February 2011
+ #################################################################################################
+ f = netCDF4.Dataset(ifile,mode='r',format=file_type)
+ xtimes = f.variables[timeVarName]
+ timeFormat = xtimes.units.encode()
+ #timeFormat = "days since 1979-01-01 00:00:00"
+ # search to check if 'since' appears in units
+ try:
+ sinceLoc = re.search('since',timeFormat).end()
+ except:
+ print 'Error decoding model times: time var attributes do not contain "since": Exit'
+ sys.exit()
+ # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
+ # TODO: Swap out this section for a set of if...elseif statements
+ units = ''
+ try:
+ _ = re.search('minutes',timeFormat).end()
+ units = 'minutes'
+ except:
+ pass
+ try:
+ _ = re.search('hours',timeFormat).end()
+ units = 'hours'
+ except:
+ pass
+ try:
+ _ = re.search('days',timeFormat).end()
+ units = 'days'
+ except:
+ pass
+ try:
+ _ = re.search('months',timeFormat).end()
+ units = 'months'
+ except:
+ pass
+ try:
+ _ = re.search('years',timeFormat).end()
+ units = 'years'
+ except:
+ pass
+ # cut out base time (the bit following 'since')
+ base_time_string = string.lstrip(timeFormat[sinceLoc:])
+ # decode base time
+ # 9/25/2012: datetime.timedelta has problem with the argument because xtimes is NioVariable.
+ # Soln (J. Kim): use a temp variable ttmp in the for loop, then convert it into an integer xtime.
+ base_time = decodeTimeFromString(base_time_string)
+ times=[]
+ for ttmp in xtimes[:]:
+ xtime = int(ttmp)
+ if(units=='minutes'):
+ dt = datetime.timedelta(minutes=xtime); new_time = base_time + dt
+ if(units=='hours'):
+ dt = datetime.timedelta(hours=xtime); new_time = base_time + dt
+ if(units=='days'):
+ dt = datetime.timedelta(days=xtime); new_time = base_time + dt
+ if(units=='months'): # NB. adding months in python is complicated as month length varies and hence ambigous.
+ # Perform date arithmatic manually
+ # Assumption: the base_date will usually be the first of the month
+ # NB. this method will fail if the base time is on the 29th or higher day of month
+ # -as can't have, e.g. Feb 31st.
+ new_month = int(base_time.month + xtime % 12)
+ new_year = int(math.floor(base_time.year + xtime / 12.))
+ #print type(new_year),type(new_month),type(base_time.day),type(base_time.hour),type(base_time.second)
+ new_time = datetime.datetime(new_year,new_month,base_time.day,base_time.hour,base_time.second,0)
+ if(units=='years'):
+ dt = datetime.timedelta(years=xtime); new_time = base_time + dt
+ times.append(new_time)
+ return times
+
+
+def regrid_in_time(data,dateList,unit):
+ #################################################################################################
+ # Routine to calculate averages on longer time units than the data exists on.
+ # e.g. if the data is 6-hourly, calculate daily, or monthly means.
+ # Input:
+ # data - data values
+ # dateList - list of python datetime structures corresponding to data values
+ # unit - string describing time unit to average onto
+ # e.g. 'monthly', 'daily', 'pentad','annual','decadal'
+ # Output:
+ # meanstorem - numpy masked array of data values meaned over required time period
+ # newTimesList - a list of python datetime objects representing the data in the new averagin period
+ # NB. currently set to beginning of averaging period,
+ # i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
+ # ..............................
+ # Jinwon Kim, Sept 30, 2012
+ # Created from calc_average_on_new_time_unit_K, Peter's original, with the modification below:
+ # Instead of masked array used by Peter, use wh to defined data within the averaging range.
+ #################################################################################################
+
+ print '*** Begin calc_average_on_new_time_unit_KK ***'
# Check if the user-selected temporal grid is valid. If not, EXIT
acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
if not acceptable:
- print 'Error: unknown unit type selected for time averaging: EXIT'
- return -1,-1,-1,-1
-
- # Calculate arrays of: annual timeseries: year (2007,2007),
- # monthly time series: year-month (200701,200702),
- # daily timeseries: year-month-day (20070101,20070102)
- # depending on user-selected averaging period.
+ print 'Error: unknown unit type selected for time averaging: EXIT'; return -1,-1,-1,-1
- # Year list
- if unit=='annual':
+ # Calculate time arrays of: annual (yyyy, [2007]), monthly (yyyymm, [200701,200702]), daily (yyyymmdd, [20070101,20070102])
+ # "%02d" is similar to i2.2 in Fortran
+ if unit=='annual': # YYYY
timeunits = []
for i in np.arange(len(dateList)):
timeunits.append(str(dateList[i].year))
- timeunits = np.array(timeunits, dtype=int)
-
- # YearMonth format list
- if unit=='monthly':
+ elif unit=='monthly': # YYYYMM
timeunits = []
for i in np.arange(len(dateList)):
timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
- timeunits = np.array(timeunits,dtype=int)
-
- # YearMonthDay format list
- if unit=='daily':
+ elif unit=='daily': # YYYYMMDD
timeunits = []
for i in np.arange(len(dateList)):
timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
- timeunits = np.array(timeunits,dtype=int)
-
- # TODO: add pentad setting using Julian days?
-
- # Full list: a special case
- if unit == 'full':
+ elif unit=='full': # Full list: a special case
comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
timeunits = []
for i in np.arange(len(dateList)):
- timeunits.append(999) # i.e. we just want the same value for all times.
- timeunits = np.array(timeunits, dtype=int)
+ timeunits.append(999) # i.e. we just want the same value for all times.
+ timeunits = np.array(timeunits,dtype=int)
+ print 'timeRegridOption= ',unit#'; output timeunits= ',timeunits
+ #print 'timeRegridOption= ',unit,'; output timeunits= ',timeunits
- # empty list to store new times
+ # empty list to store time levels after temporal regridding
newTimesList = []
# Decide whether or not you need to do any time averaging.
- # i.e. if data are already on required time unit then just pass data through and
- # calculate and return representative datetimes.
+ # i.e. if data are already on required time unit then just pass data through and calculate and return representative datetimes.
processing_required = True
if len(timeunits)==(len(np.unique(timeunits))):
processing_required = False
+ print 'processing_required= ',processing_required,': input time steps= ',len(timeunits),': regridded output time steps = ',len(np.unique(timeunits))
+ #print 'np.unique(timeunits): ',np.unique(timeunits)
+ print 'data.ndim= ',data.ndim
- # 1D data arrays, i.e. time series
- if data.ndim==1:
- # Create array to store the resulting data
+ if data.ndim==1: # 1D data arrays, i.e. 1D time series
+ # Create array to store the temporally regridded data
meanstore = np.zeros(len(np.unique(timeunits)))
-
# Calculate the means across each unique time unit
i=0
for myunit in np.unique(timeunits):
if processing_required:
- datam=ma.masked_array(data,timeunits!=myunit)
+ wh = timeunits==myunit
+ datam = data[wh]
meanstore[i] = ma.average(datam)
-
- # construct new times list
+ # construct new times list
smyunit = str(myunit)
if len(smyunit)==4: # YYYY
yyyy = int(myunit[0:4])
@@ -1008,32 +960,21 @@ def calc_average_on_new_time_unit_K(data, dateList, unit):
yyyy = int(halfway.year)
mm = int(halfway.month)
dd = int(halfway.day)
-
newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
i = i+1
- # 3D data arrays
- if data.ndim==3:
- # datamask = create_mask_using_threshold(data,threshold=0.75)
+ elif data.ndim==3: # 3D data arrays, i.e. 2D time series
# Create array to store the resulting data
meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
-
# Calculate the means across each unique time unit
i=0
datamask_store = []
for myunit in np.unique(timeunits):
if processing_required:
- mask = np.zeros_like(data)
- mask[timeunits!=myunit,:,:] = 1.0
- # Calculate missing data mask within each time unit...
- datamask_at_this_timeunit = np.zeros_like(data)
- datamask_at_this_timeunit[:]= create_mask_using_threshold(data[timeunits==myunit,:,:],threshold=0.75)
- # Store results for masking later
- datamask_store.append(datamask_at_this_timeunit[0])
- # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
- datam = ma.masked_array(data,np.logical_or(mask,datamask_at_this_timeunit))
+ wh = timeunits==myunit
+ datam = data[wh,:,:]
meanstore[i,:,:] = ma.average(datam,axis=0)
- # construct new times list
+ # construct new times list
smyunit = str(myunit)
if len(smyunit)==4: # YYYY
yyyy = int(smyunit[0:4])
@@ -1059,226 +1000,8 @@ def calc_average_on_new_time_unit_K(data, dateList, unit):
if not processing_required:
meanstorem = data
+ elif processing_required:
+ meanstorem = meanstore
- if processing_required:
- # Create masked array (using missing data mask defined above)
- datamask_store = np.array(datamask_store)
- meanstorem = ma.masked_array(meanstore, datamask_store)
-
+ print '*** End calc_average_on_new_time_unit_KK ***'
return meanstorem,newTimesList
-
-def decode_model_timesK(ifile,timeVarName,file_type):
- #################################################################################################
- # Convert model times ('hours since 1900...', 'days since ...') into a python datetime structure
- # Input:
- # filelist - list of model files
- # timeVarName - name of the time variable in the model files
- # Output:
- # times - list of python datetime objects describing model data times
- # Peter Lean February 2011
- #################################################################################################
- f = Nio.open_file(ifile,mode='r',options=None,format=file_type)
- xtimes = f.variables[timeVarName]
- timeFormat = xtimes.attributes['units']
- #timeFormat = "days since 1979-01-01 00:00:00"
- # search to check if 'since' appears in units
- try:
- sinceLoc = re.search('since',timeFormat).end()
- except:
- print 'Error decoding model times: time var attributes do not contain "since": Exit'
- sys.exit()
- # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
- # TODO: Swap out this section for a set of if...elseif statements
- units = ''
- try:
- _ = re.search('minutes',timeFormat).end()
- units = 'minutes'
- except:
- pass
- try:
- _ = re.search('hours',timeFormat).end()
- units = 'hours'
- except:
- pass
- try:
- _ = re.search('days',timeFormat).end()
- units = 'days'
- except:
- pass
- try:
- _ = re.search('months',timeFormat).end()
- units = 'months'
- except:
- pass
- try:
- _ = re.search('years',timeFormat).end()
- units = 'years'
- except:
- pass
- # cut out base time (the bit following 'since')
- base_time_string = string.lstrip(timeFormat[sinceLoc:])
- # decode base time
- # 9/25/2012: datetime.timedelta has problem with the argument because xtimes is NioVariable.
- # Soln (J. Kim): use a temp variable ttmp in the for loop, then convert it into an integer xtime.
- base_time = decodeTimeFromString(base_time_string)
- times=[]
- for ttmp in xtimes[:]:
- xtime = int(ttmp)
- if(units=='minutes'):
- dt = datetime.timedelta(minutes=xtime); new_time = base_time + dt
- if(units=='hours'):
- dt = datetime.timedelta(hours=xtime); new_time = base_time + dt
- if(units=='days'):
- dt = datetime.timedelta(days=xtime); new_time = base_time + dt
- if(units=='months'): # NB. adding months in python is complicated as month length varies and hence ambigous.
- # Perform date arithmatic manually
- # Assumption: the base_date will usually be the first of the month
- # NB. this method will fail if the base time is on the 29th or higher day of month
- # -as can't have, e.g. Feb 31st.
- new_month = int(base_time.month + xtime % 12)
- new_year = int(math.floor(base_time.year + xtime / 12.))
- #print type(new_year),type(new_month),type(base_time.day),type(base_time.hour),type(base_time.second)
- new_time = datetime.datetime(new_year,new_month,base_time.day,base_time.hour,base_time.second,0)
- if(units=='years'):
- dt = datetime.timedelta(years=xtime); new_time = base_time + dt
- times.append(new_time)
- return times
-
-
-def regrid_in_time(data,dateList,unit):
- #################################################################################################
- # Routine to calculate averages on longer time units than the data exists on.
- # e.g. if the data is 6-hourly, calculate daily, or monthly means.
- # Input:
- # data - data values
- # dateList - list of python datetime structures corresponding to data values
- # unit - string describing time unit to average onto
- # e.g. 'monthly', 'daily', 'pentad','annual','decadal'
- # Output:
- # meanstorem - numpy masked array of data values meaned over required time period
- # newTimesList - a list of python datetime objects representing the data in the new averagin period
- # NB. currently set to beginning of averaging period,
- # i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
- # ..............................
- # Jinwon Kim, Sept 30, 2012
- # Created from calc_average_on_new_time_unit_K, Peter's original, with the modification below:
- # Instead of masked array used by Peter, use wh to defined data within the averaging range.
- #################################################################################################
-
- print '*** Begin calc_average_on_new_time_unit_KK ***'
- # Check if the user-selected temporal grid is valid. If not, EXIT
- acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
- if not acceptable:
- print 'Error: unknown unit type selected for time averaging: EXIT'; return -1,-1,-1,-1
-
- # Calculate time arrays of: annual (yyyy, [2007]), monthly (yyyymm, [200701,200702]), daily (yyyymmdd, [20070101,20070102])
- # "%02d" is similar to i2.2 in Fortran
- if unit=='annual': # YYYY
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year))
- elif unit=='monthly': # YYYYMM
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
- elif unit=='daily': # YYYYMMDD
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
- elif unit=='full': # Full list: a special case
- comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(999) # i.e. we just want the same value for all times.
- timeunits = np.array(timeunits,dtype=int)
- print 'timeRegridOption= ',unit#'; output timeunits= ',timeunits
- #print 'timeRegridOption= ',unit,'; output timeunits= ',timeunits
-
- # empty list to store time levels after temporal regridding
- newTimesList = []
-
- # Decide whether or not you need to do any time averaging.
- # i.e. if data are already on required time unit then just pass data through and calculate and return representative datetimes.
- processing_required = True
- if len(timeunits)==(len(np.unique(timeunits))):
- processing_required = False
- print 'processing_required= ',processing_required,': input time steps= ',len(timeunits),': regridded output time steps = ',len(np.unique(timeunits))
- #print 'np.unique(timeunits): ',np.unique(timeunits)
- print 'data.ndim= ',data.ndim
-
- if data.ndim==1: # 1D data arrays, i.e. 1D time series
- # Create array to store the temporally regridded data
- meanstore = np.zeros(len(np.unique(timeunits)))
- # Calculate the means across each unique time unit
- i=0
- for myunit in np.unique(timeunits):
- if processing_required:
- wh = timeunits==myunit
- datam = data[wh]
- meanstore[i] = ma.average(datam)
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit)==4: # YYYY
- yyyy = int(myunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit)==6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit)==8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit)==3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1]-dateList[0]
- halfway = dateList[0]+(dt/2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
- newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
- i = i+1
-
- elif data.ndim==3: # 3D data arrays, i.e. 2D time series
- # Create array to store the resulting data
- meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
- # Calculate the means across each unique time unit
- i=0
- datamask_store = []
- for myunit in np.unique(timeunits):
- if processing_required:
- wh = timeunits==myunit
- datam = data[wh,:,:]
- meanstore[i,:,:] = ma.average(datam,axis=0)
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit)==4: # YYYY
- yyyy = int(smyunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit)==6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit)==8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit)==3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1]-dateList[0]
- halfway = dateList[0]+(dt/2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
- newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
- i += 1
-
- if not processing_required:
- meanstorem = data
- elif processing_required:
- meanstorem = meanstore
-
- print '*** End calc_average_on_new_time_unit_KK ***'
- return meanstorem,newTimesList
http://git-wip-us.apache.org/repos/asf/climate/blob/4d64d214/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
index 466e027..6b3ab1a 100644
--- a/mccsearch/docs/mccsearch.md
+++ b/mccsearch/docs/mccsearch.md
@@ -9,14 +9,14 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
##What is needed?
* Python 2.7.4 (We used Anaconda 1.5.1 64 bit libraries, which installed most of the dependencies.) Other module dependences:
- * Nio
- * netCDF
- * sciPy
- * NumPy
- * Networkx
- * matplotlib
- * GrADS (We used OpenGrADS grads2 Version 2.0.1.oga.1)
- * LATS4D
+ * Nio - https://www.pyngl.ucar.edu/Nio.shtml
+ * netCDF - http://www.unidata.ucar.edu/software/netcdf/
+ * sciPy - http://www.scipy.org/scipylib/index.html
+ * NumPy - http://www.scipy.org/scipylib/download.html
+ * Networkx - https://networkx.github.io/download.html
+ * matplotlib - http://matplotlib.org/downloads.html
+ * GrADS (We used OpenGrADS grads2 Version 2.0.1.oga.1) - http://sourceforge.net/projects/opengrads/files/
+ * LATS4D - http://sourceforge.net/projects/opengrads/files/
##Download the source code and store in a folder
* mccSearch.py contains all the function needed
@@ -24,7 +24,7 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
* process.py contains some needed functions (from older version of Apache OCW)
* file.py contains some needed functions (from older version of Apache OCW)
* mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
- * mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be store
+ * mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be stored
* TRMMdirName : this is the directory where the original TRMM data in netCDF format is stored
* CEoriDirName : this is the directory where the original MERG data in netCDF format is stored
* Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
[35/50] [abbrv] git commit: initial mccsearch code
Posted by le...@apache.org.
initial mccsearch code
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/96aad4bf
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/96aad4bf
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/96aad4bf
Branch: refs/heads/master
Commit: 96aad4bf06e2931551ff96a18560343804e8d347
Parents: 7a0a7e2
Author: kwhitehall <k_...@yahoo.com>
Authored: Mon Apr 7 18:14:54 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:04 2014 -0700
----------------------------------------------------------------------
mccsearch/GrADsScripts/TRMMCE.gs | 15 +
mccsearch/GrADsScripts/accuTRMM.gs | 11 +
mccsearch/GrADsScripts/c3.gs | 11 +
.../GrADsScripts/cloudElementVars_template | 12 +
.../GrADsScripts/cloudElementVars_template2 | 11 +
mccsearch/GrADsScripts/cs1.gs | 83 +
mccsearch/GrADsScripts/cs2.gs | 104 +
mccsearch/GrADsScripts/cs3.gs | 10 +
mccsearch/GrADsScripts/cs4.gs | 27 +
mccsearch/GrADsScripts/infrared.gs | 88 +
mccsearch/GrADsScripts/mergeCE.gs | 88 +
mccsearch/code/files.py | 696 +++
mccsearch/code/mainProg.py | 142 +
mccsearch/code/mainProgGoyens.py | 126 +
mccsearch/code/mainProgWorks.py | 124 +
mccsearch/code/mccSearch.py | 4340 ++++++++++++++++++
mccsearch/code/mccSearchGoyens.py | 3606 +++++++++++++++
mccsearch/code/newPreprocessingMERG.py | 175 +
mccsearch/code/preprocessingMERG.py | 125 +
mccsearch/code/process.py | 1284 ++++++
20 files changed, 11078 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/TRMMCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/TRMMCE.gs b/mccsearch/GrADsScripts/TRMMCE.gs
new file mode 100644
index 0000000..cd1dbd6
--- /dev/null
+++ b/mccsearch/GrADsScripts/TRMMCE.gs
@@ -0,0 +1,15 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/20090831000000F1CE1.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
+'d precipAcc'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/20090831000000F1CE1.gif x800 y600 white'
+'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/accuTRMM.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/accuTRMM.gs b/mccsearch/GrADsScripts/accuTRMM.gs
new file mode 100644
index 0000000..01eda71
--- /dev/null
+++ b/mccsearch/GrADsScripts/accuTRMM.gs
@@ -0,0 +1,11 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+'set datawarn off'
+'draw title TRMM Accumulated Precipitation [mm]'
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/c3.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/c3.gs b/mccsearch/GrADsScripts/c3.gs
new file mode 100644
index 0000000..f19e5e5
--- /dev/null
+++ b/mccsearch/GrADsScripts/c3.gs
@@ -0,0 +1,11 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/cloudElementVars_template
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cloudElementVars_template b/mccsearch/GrADsScripts/cloudElementVars_template
new file mode 100644
index 0000000..bb572f6
--- /dev/null
+++ b/mccsearch/GrADsScripts/cloudElementVars_template
@@ -0,0 +1,12 @@
+DSET
+dtype netcdf
+undef 0
+TITLE MCC search Output Grid: Time lat lon
+XDEF 413 LINEAR -9.984375 0.036378335
+YDEF 412 LINEAR 5.03515625 0.036378335
+tdef 99999 linear 31aug2009 1hr
+zdef 01 LEVELS 1
+vars 1
+brightnesstemp=>ch4 1 t,y,x brightnesstemperature
+endvars
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/cloudElementVars_template2
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cloudElementVars_template2 b/mccsearch/GrADsScripts/cloudElementVars_template2
new file mode 100644
index 0000000..42746f5
--- /dev/null
+++ b/mccsearch/GrADsScripts/cloudElementVars_template2
@@ -0,0 +1,11 @@
+DSET
+undef 0
+TITLE MCC search Output Grid: Time lat lon
+XDEF 413 LINEAR -9.984375 0.036378335
+YDEF 412 LINEAR 5.03515625 0.036378335
+tdef 99999 linear 31aug2009 1hr
+zdef 01 LEVELS 1
+vars 1
+precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu
+endvars
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/cs1.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs1.gs b/mccsearch/GrADsScripts/cs1.gs
new file mode 100644
index 0000000..8a790ed
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs1.gs
@@ -0,0 +1,83 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+
+
+
+
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/cs2.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs2.gs b/mccsearch/GrADsScripts/cs2.gs
new file mode 100644
index 0000000..9edcf5a
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs2.gs
@@ -0,0 +1,104 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE1.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE2.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE3.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE4.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE5.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE6.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE7.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE8.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE9.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+'d ch4.9'
+'d ch4.8'
+'d ch4.7'
+'d ch4.6'
+'d ch4.5'
+'d ch4.4'
+'d ch4.3'
+'d ch4.2'
+'d ch4.1'
+
+
+
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-08-31_23:00:00F7.gif x800 y600 white'
+'quit'
+
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/cs3.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs3.gs b/mccsearch/GrADsScripts/cs3.gs
new file mode 100644
index 0000000..1c23d96
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs3.gs
@@ -0,0 +1,10 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/cs4.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs4.gs b/mccsearch/GrADsScripts/cs4.gs
new file mode 100644
index 0000000..e6396f7
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs4.gs
@@ -0,0 +1,27 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE1.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE2.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE3.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE4.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE5.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE6.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE7.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
+'d precipAcc.7'
+'d precipAcc.6'
+'d precipAcc.5'
+'d precipAcc.4'
+'d precipAcc.3'
+'d precipAcc.2'
+'d precipAcc.1'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/TRMM2009-09-01_23:00:00F48.gif x800 y600 white'
+'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/infrared.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/infrared.gs b/mccsearch/GrADsScripts/infrared.gs
new file mode 100644
index 0000000..5c22b96
--- /dev/null
+++ b/mccsearch/GrADsScripts/infrared.gs
@@ -0,0 +1,88 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF/ctlFiles/merg_2009090123_4km-pixel.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+
+
+
+
+'d ch4'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/merg_2009090123_4km-pixel.gif x800 y600 white'
+'quit'
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/GrADsScripts/mergeCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/mergeCE.gs b/mccsearch/GrADsScripts/mergeCE.gs
new file mode 100644
index 0000000..b89945c
--- /dev/null
+++ b/mccsearch/GrADsScripts/mergeCE.gs
@@ -0,0 +1,88 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-09-01_00:00:00F8CE5.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+'d ch4'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-09-01_00:00:00F8CE5.gif x800 y600 white'
+'quit'
+
+
+
+
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/files.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.py b/mccsearch/code/files.py
new file mode 100644
index 0000000..82b508a
--- /dev/null
+++ b/mccsearch/code/files.py
@@ -0,0 +1,696 @@
+"""
+Module for handling data input files. Requires PyNIO and Numpy be
+installed.
+
+This module can easily open NetCDF, HDF and Grib files. Search the PyNIO
+documentation for a complete list of supported formats.
+"""
+
+from os import path
+
+try:
+ import Nio
+except ImportError:
+ import nio as Nio
+
+import numpy as np
+import numpy.ma as ma
+import sys
+import process #KDW added
+#from toolkit import process
+#from utils import fortranfile
+#from utils import misc
+
+
+VARIABLE_NAMES = {'time': ['time', 'times', 'date', 'dates', 'julian'],
+ 'latitude': ['latitude', 'lat', 'lats', 'latitudes'],
+ 'longitude': ['longitude', 'lon', 'lons', 'longitudes']
+ }
+
+
+def findunique(seq):
+ keys = {}
+ for e in seq:
+ keys[e] = 1
+ return keys.keys()
+
+def getVariableByType(filename, variableType):
+ """
+ Function that will try to return the variable from a file based on a provided
+ parameter type.
+
+ Input::
+ filename - the file to inspect
+ variableType - time | latitude | longitude
+
+ Output::
+ variable name OR list of all variables in the file if a single variable
+ name match cannot be found.
+ """
+ try:
+ f = Nio.open_file(filename)
+ except:
+ #print 'PyNio had an issue opening the filename (%s) you provided' % filename
+ print "NIOError:", sys.exc_info()[0]
+ raise
+
+ variableKeys = f.variables.keys()
+ f.close()
+ variableKeys = [variable.lower() for variable in variableKeys]
+ variableMatch = VARIABLE_NAMES[variableType]
+
+ commonVariables = list(set(variableKeys).intersection(variableMatch))
+
+ if len(commonVariables) == 1:
+ return str(commonVariables[0])
+
+ else:
+ return variableKeys
+
+def getVariableRange(filename, variableName):
+ """
+ Function to return the min and max values of the given variable in
+ the supplied filename.
+
+ Input::
+ filename - absolute path to a file
+ variableName - variable whose min and max values should be returned
+
+ Output::
+ variableRange - tuple of order (variableMin, variableMax)
+ """
+ try:
+ f = Nio.open_file(filename)
+ except:
+ #print 'PyNio had an issue opening the filename (%s) you provided' % filename
+ print "NIOError:", sys.exc_info()[0]
+ raise
+
+ varArray = f.variables[variableName][:]
+ return (varArray.min(), varArray.max())
+
+
+def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarName):
+ '''
+ Read in data from a list of model files into a single data structure
+
+ Input:
+ filelist - list of filenames (including path)
+ myvar - string containing name of variable to load in (as it appears in file)
+ Output:
+ lat, lon - 2D array of latitude and longitude values
+ timestore - list of times
+ t2store - numpy array containing data from all files
+
+ NB. originally written specific for WRF netCDF output files
+ modified to make more general (Feb 2011)
+
+ Peter Lean July 2010
+ '''
+
+ filelist.sort()
+ filename = filelist[0]
+ # Crash nicely if 'filelist' is zero length
+ """TODO: Throw Error instead via try Except"""
+ if len(filelist) == 0:
+ print 'Error: no files have been passed to read_data_from_file_list()'
+ sys.exit()
+
+ # Open the first file in the list to:
+ # i) read in lats, lons
+ # ii) find out how many timesteps in the file
+ # (assume same ntimes in each file in list)
+ # -allows you to create an empty array to store variable data for all times
+ tmp = Nio.open_file(filename)
+ latsraw = tmp.variables[latVarName][:]
+ lonsraw = tmp.variables[lonVarName][:]
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ """TODO: Guard against case where latsraw and lonsraw are not the same dim?"""
+
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
+
+ timesraw = tmp.variables[timeVarName]
+ ntimes = len(timesraw)
+
+ print 'Lats and lons read in for first file in filelist', ntimes
+
+ # Create a single empty masked array to store model data from all files
+ t2store = ma.zeros((ntimes * len(filelist), len(lat[:, 0]), len(lon[0, :])))
+ timestore = ma.zeros((ntimes * len(filelist)))
+
+ # Now load in the data for real
+ # NB. no need to reload in the latitudes and longitudes -assume invariant
+ i = 0
+ timesaccu = 0 # a counter for number of times stored so far in t2store
+ # NB. this method allows for missing times in data files
+ # as no assumption made that same number of times in each file...
+
+
+ for ifile in filelist:
+
+ #print 'Loading data from file: ',filelist[i]
+ f = Nio.open_file(ifile)
+ t2raw = f.variables[myvar][:]
+ timesraw = f.variables[timeVarName]
+ time = timesraw[:]
+ ntimes = len(time)
+ print 'file= ', i, 'ntimes= ', ntimes, filelist[i]
+ print '\nt2raw shape: ', t2raw.shape
+
+ # Flatten dimensions which needn't exist, i.e. level
+ # e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
+ # Code requires data to have dimensions, (time,lat,lon)
+ # i.e. remove level dimensions
+ # Remove 1d axis from the t2raw array
+ # Example: t2raw.shape == (365, 180, 360 1) <maps to (time, lat, lon, height)>
+ # After the squeeze you will be left with (365, 180, 360) instead
+ t2tmp = t2raw.squeeze()
+ # Nb. if this happens to be data for a single time only, then we just flattened it by accident
+ # lets put it back...
+ if t2tmp.ndim == 2:
+ t2tmp = np.expand_dims(t2tmp, 0)
+
+ print "\n timesaccu is: ", timesaccu, time,timesaccu + np.arange(ntimes)
+
+ t2store[timesaccu + np.arange(ntimes), :, :] = t2tmp[:, :, :]
+ timestore[timesaccu + np.arange(ntimes)] = time
+ timesaccu = timesaccu + ntimes
+ print "\n ***timesaccu is: ", timesaccu
+ f.close()
+ i += 1
+
+ print '\nData read in successfully with dimensions: ', t2store.shape, timesaccu
+
+ # TODO: search for duplicated entries (same time) and remove duplicates.
+ # Check to see if number of unique times == number of times, if so then no problem
+
+ if(len(np.unique(timestore)) != len(np.where(timestore != 0)[0].view())):
+ print '\nWARNING: Possible duplicated times'
+
+ # Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
+ timestore, _ = process.getModelTimes(filename, timeVarName)
+
+ data_dict = {}
+ data_dict['lats'] = lat
+ data_dict['lons'] = lon
+ data_dict['times'] = timestore
+ data_dict['data'] = t2store
+ #return lat, lon, timestore, t2store
+ return data_dict
+
+def select_var_from_file(myfile, fmt='not set'):
+ '''
+ Routine to act as user interface to allow users to select variable of interest from a file.
+
+ Input:
+ myfile - filename
+ fmt - (optional) specify fileformat for PyNIO if filename suffix is non-standard
+
+ Output:
+ myvar - variable name in file
+
+ Peter Lean September 2010
+ '''
+
+ print fmt
+
+ if fmt == 'not set':
+ f = Nio.open_file(myfile)
+
+ if fmt != 'not set':
+ f = Nio.open_file(myfile, format=fmt)
+
+ keylist = f.variables.keys()
+
+ i = 0
+ for v in keylist:
+ print '[', i, '] ', f.variables[v].long_name, ' (', v, ')'
+ i += 1
+
+ user_selection = raw_input('Please select variable : [0 -' + str(i - 1) + '] ')
+
+ myvar = keylist[int(user_selection)]
+
+ return myvar
+
+def select_var_from_wrf_file(myfile):
+ '''
+ Routine to act as user interface to allow users to select variable of interest from a wrf netCDF file.
+
+ Input:
+ myfile - filename
+
+ Output:
+ mywrfvar - variable name in wrf file
+
+ Peter Lean September 2010
+ '''
+
+ f = Nio.open_file(myfile, format='nc')
+
+ keylist = f.variables.keys()
+
+ i = 0
+ for v in keylist:
+ try:
+ print '[', i, '] ', f.variables[v].description, ' (', v, ')'
+ except:
+ print ''
+
+ i += 1
+
+ user_selection = raw_input('Please select WRF variable : [0 -' + str(i - 1) + '] ')
+
+ mywrfvar = keylist[int(user_selection)]
+
+ return mywrfvar
+
+def read_lolaT_from_file(filename, latVarName, lonVarName, timeVarName, file_type):
+ """
+ Function that will return lat, lon, and time arrays
+
+ Input::
+ filename - the file to inspect
+ latVarName - name of the Latitude Variable
+ lonVarName - name of the Longitude Variable
+ timeVarName - name of the Time Variable
+ fileType = type of file we are trying to parse
+
+ Output::
+ lat - MESH GRID of Latitude values with shape (nx, ny)
+ lon - MESH GRID of Longitude values with shape (nx, ny)
+ timestore - Python list of Datetime objects
+
+ MESHGRID docs: http://docs.scipy.org/doc/numpy/reference/generated/numpy.meshgrid.html
+
+ """
+
+ tmp = Nio.open_file(filename, format=file_type)
+ lonsraw = tmp.variables[lonVarName][:]
+ latsraw = tmp.variables[latVarName][:]
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
+ timestore, _ = process.getModelTimes(filename, timeVarName)
+ print ' read_lolaT_from_file: Lats, lons and times read in for the model domain'
+ return lat, lon, timestore
+
+def read_data_from_one_file(ifile, myvar, timeVarName, lat, file_type):
+ ##################################################################################
+ # Read in data from one file at a time
+ # Input: filelist - list of filenames (including path)
+ # myvar - string containing name of variable to load in (as it appears in file)
+ # Output: lat, lon - 2D array of latitude and longitude values
+ # times - list of times
+ # t2store - numpy array containing data from all files
+ # Modified from read_data_from_file_list to read data from multiple models into a 4-D array
+ # 1. The code now processes model data that completely covers the 20-yr period. Thus,
+ # all model data must have the same time levels (ntimes). Unlike in the oroginal, ntimes
+ # is fixed here.
+ # 2. Because one of the model data exceeds 240 mos (243 mos), the model data must be
+ # truncated to the 240 mons using the ntimes determined from the first file.
+ ##################################################################################
+ f = Nio.open_file(ifile)
+ try:
+ varUnit = f.variables[myvar].units.upper()
+ except:
+ varUnit = raw_input('Enter the model variable unit: \n> ').upper()
+ t2raw = f.variables[myvar][:]
+ t2tmp = t2raw.squeeze()
+ if t2tmp.ndim == 2:
+ t2tmp = np.expand_dims(t2tmp, 0)
+ t2tmp = t2tmp
+ f.close()
+ print ' success read_data_from_one_file: VarName=', myvar, ' Shape(Full)= ', t2tmp.shape, ' Unit= ', varUnit
+ timestore = process.decode_model_timesK(ifile, timeVarName, file_type)
+ return timestore, t2tmp, varUnit
+
+def findTimeVariable(filename):
+ """
+ Function to find what the time variable is called in a model file.
+ Input::
+ filename - file to crack open and check for a time variable
+ Output::
+ timeName - name of the input file's time variable
+ variableNameList - list of variable names from the input filename
+ """
+ try:
+ f = Nio.open_file(filename, mode='r')
+ except:
+ print("Unable to open '%s' to try and read the Time variable" % filename)
+ raise
+
+ variableNameList = f.variables.keys()
+ # convert all variable names into lower case
+ varNameListLowerCase = [x.lower() for x in variableNameList]
+
+ # Use "set" types for finding common variable name from in the file and from the list of possibilities
+ possibleTimeNames = set(['time', 'times', 'date', 'dates', 'julian'])
+
+ # Use the sets to find the intersection where variable names are in possibleNames
+ timeNameSet = possibleTimeNames.intersection(varNameListLowerCase)
+
+ if len(timeNameSet) == 0:
+ print("Unable to autodetect the Time Variable Name in the '%s'" % filename)
+ timeName = misc.askUserForVariableName(variableNameList, targetName ="Time")
+
+ else:
+ timeName = timeNameSet.pop()
+
+ return timeName, variableNameList
+
+
+def findLatLonVarFromFile(filename):
+ """
+ Function to find what the latitude and longitude variables are called in a model file.
+
+ Input::
+ -filename
+ Output::
+ -latVarName
+ -lonVarName
+ -latMin
+ -latMax
+ -lonMin
+ -lonMax
+ """
+ try:
+ f = Nio.open_file(filename, mode='r')
+ except:
+ print("Unable to open '%s' to try and read the Latitude and Longitude variables" % filename)
+ raise
+
+ variableNameList = f.variables.keys()
+ # convert all variable names into lower case
+ varNameListLowerCase = [x.lower() for x in variableNameList]
+
+ # Use "set" types for finding common variable name from in the file and from the list of possibilities
+ possibleLatNames = set(['latitude', 'lat', 'lats', 'latitudes'])
+ possibleLonNames = set(['longitude', 'lon', 'lons', 'longitudes'])
+
+ # Use the sets to find the intersection where variable names are in possibleNames
+ latNameSet = possibleLatNames.intersection(varNameListLowerCase)
+ lonNameSet = possibleLonNames.intersection(varNameListLowerCase)
+
+ if len(latNameSet) == 0 or len(lonNameSet) == 0:
+ print("Unable to autodetect Latitude and/or Longitude values in the file")
+ latName = misc.askUserForVariableName(variableNameList, targetName ="Latitude")
+ lonName = misc.askUserForVariableName(variableNameList, targetName ="Longitude")
+
+ else:
+ latName = latNameSet.pop()
+ lonName = lonNameSet.pop()
+
+ lats = np.array(f.variables[latName][:])
+ lons = np.array(f.variables[lonName][:])
+
+ latMin = lats.min()
+ latMax = lats.max()
+
+ # Convert the lons from 0:360 into -180:180
+ lons[lons > 180] = lons[lons > 180] - 360.
+ lonMin = lons.min()
+ lonMax = lons.max()
+
+ return latName, lonName, latMin, latMax, lonMin, lonMax
+
+
+def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarName, file_type):
+ ##################################################################################
+ # Read in data from a list of model files into a single data structure
+ # Input: filelist - list of filenames (including path)
+ # myvar - string containing name of variable to load in (as it appears in file)
+ # Output: lat, lon - 2D array of latitude and longitude values
+ # times - list of times
+ # t2store - numpy array containing data from all files
+ # Modified from read_data_from_file_list to read data from multiple models into a 4-D array
+ # 1. The code now processes model data that completely covers the 20-yr period. Thus,
+ # all model data must have the same time levels (ntimes). Unlike in the oroginal, ntimes
+ # is fixed here.
+ # 2. Because one of the model data exceeds 240 mos (243 mos), the model data must be
+ # truncated to the 240 mons using the ntimes determined from the first file.
+ ##################################################################################
+ filelist.sort()
+ nfiles = len(filelist)
+ # Crash nicely if 'filelist' is zero length
+ if nfiles == 0:
+ print 'Error: no files have been passed to read_data_from_file_list(): Exit'
+ sys.exit()
+
+ # Open the first file in the list to:
+ # i) read in lats, lons
+ # ii) find out how many timesteps in the file (assume same ntimes in each file in list)
+ # -allows you to create an empty array to store variable data for all times
+ tmp = Nio.open_file(filelist[0], format=file_type)
+ latsraw = tmp.variables[latVarName][:]
+ lonsraw = tmp.variables[lonVarName][:]
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw; lat = latsraw
+
+ timesraw = tmp.variables[timeVarName]
+ ntimes = len(timesraw); nygrd = len(lat[:, 0]); nxgrd = len(lon[0, :])
+
+ print 'Lats and lons read in for first file in filelist'
+
+ # Create a single empty masked array to store model data from all files
+ #t2store = ma.zeros((ntimes*nfiles,nygrd,nxgrd))
+ t2store = ma.zeros((nfiles, ntimes, nygrd, nxgrd))
+ #timestore=ma.zeros((ntimes*nfiles))
+
+ ## Now load in the data for real
+ ## NB. no need to reload in the latitudes and longitudes -assume invariant
+ #timesaccu=0 # a counter for number of times stored so far in t2store
+ # NB. this method allows for missing times in data files
+ # as no assumption made that same number of times in each file...
+
+ i = 0
+ for ifile in filelist:
+ #print 'Loading data from file: ',filelist[i]
+ f = Nio.open_file(ifile)
+ t2raw = f.variables[myvar][:]
+ timesraw = f.variables[timeVarName]
+ #time = timesraw[0:ntimes]
+ ntimes = len(timesraw)
+ #ntimes=len(time)
+ print 'file= ',i,' ntimes= ',ntimes,filelist[i]
+ ## Flatten dimensions which needn't exist, i.e. level
+ ## e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
+ ## Code requires data to have dimensions, (time,lat,lon)
+ ## i.e. remove level dimensions
+ t2tmp = t2raw.squeeze()
+ ## Nb. if data happen to be for a single time, we flattened it by accident; lets put it back...
+ if t2tmp.ndim == 2:
+ t2tmp = np.expand_dims(t2tmp, 0)
+ #t2store[timesaccu+np.arange(ntimes),:,:]=t2tmp[0:ntimes,:,:]
+ t2store[i, 0:(ntimes-1), :, :] = t2tmp[0:(ntimes-1), :, :]
+ #timestore[timesaccu+np.arange(ntimes)]=time
+ #timesaccu=timesaccu+ntimes
+ f.close()
+ i += 1
+
+ print 'Data read in successfully with dimensions: ', t2store.shape
+
+ # Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
+ ifile = filelist[0]
+ timestore, _ = process.getModelTimes(ifile, timeVarName)
+
+ return lat, lon, timestore, t2store
+
+def find_latlon_ranges(filelist, lat_var_name, lon_var_name):
+ # Function to return the latitude and longitude ranges of the data in a file,
+ # given the identifying variable names.
+ #
+ # Input:
+ # filelist - list of filenames (data is read in from first file only)
+ # lat_var_name - variable name of the 'latitude' variable
+ # lon_var_name - variable name of the 'longitude' variable
+ #
+ # Output:
+ # latMin, latMax, lonMin, lonMax - self explanatory
+ #
+ # Peter Lean March 2011
+
+ filename = filelist[0]
+
+ try:
+ f = Nio.open_file(filename)
+
+ lats = f.variables[lat_var_name][:]
+ latMin = lats.min()
+ latMax = lats.max()
+
+ lons = f.variables[lon_var_name][:]
+ lons[lons > 180] = lons[lons > 180] - 360.
+ lonMin = lons.min()
+ lonMax = lons.max()
+
+ return latMin, latMax, lonMin, lonMax
+
+ except:
+ print 'Error: there was a problem with finding the latitude and longitude ranges in the file'
+ print ' Please check that you specified the filename, and variable names correctly.'
+
+ sys.exit()
+
+def writeBN_lola(fileName, lons, lats):
+ # write a binary data file that include longitude (1-d) and latitude (1-d) values
+
+ F = fortranfile.FortranFile(fileName, mode='w')
+ ngrdY = lons.shape[0]; ngrdX = lons.shape[1]
+ tmpDat = ma.zeros(ngrdX); tmpDat[:] = lons[0, :]; F.writeReals(tmpDat)
+ tmpDat = ma.zeros(ngrdY); tmpDat[:] = lats[:, 0]; F.writeReals(tmpDat)
+ # release temporary arrays
+ tmpDat = 0
+ F.close()
+
+def writeBNdata(fileName, numOBSs, numMDLs, nT, ngrdX, ngrdY, numSubRgn, obsData, mdlData, obsRgnAvg, mdlRgnAvg):
+ #(fileName,maskOption,numOBSs,numMDLs,nT,ngrdX,ngrdY,numSubRgn,obsData,mdlData,obsRgnAvg,mdlRgnAvg):
+ # write spatially- and regionally regridded data into a binary data file
+ missing = -1.e26
+ F = fortranfile.FortranFile(fileName, mode='w')
+ # construct a data array to replace mask flag with a missing value (missing=-1.e12) for printing
+ data = ma.zeros((nT, ngrdY, ngrdX))
+ for m in np.arange(numOBSs):
+ data[:, :, :] = obsData[m, :, :, :]; msk = data.mask
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ for i in np.arange(ngrdX):
+ if msk[n, j, i]: data[n, j, i] = missing
+
+ # write observed data. allowed to write only one row at a time
+ tmpDat = ma.zeros(ngrdX)
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ tmpDat[:] = data[n, j, :]
+ F.writeReals(tmpDat)
+
+ # write model data (dep. on the number of models).
+ for m in np.arange(numMDLs):
+ data[:, :, :] = mdlData[m, :, :, :]
+ msk = data.mask
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ for i in np.arange(ngrdX):
+ if msk[n, j, i]:
+ data[n, j, i] = missing
+
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ tmpDat[:] = data[n, j, :]
+ F.writeReals(tmpDat)
+
+ data = 0 # release the array allocated for data
+ # write data in subregions
+ if numSubRgn > 0:
+ print 'Also included are the time series of the means over ', numSubRgn, ' areas from obs and model data'
+ tmpDat = ma.zeros(nT); print numSubRgn
+ for m in np.arange(numOBSs):
+ for n in np.arange(numSubRgn):
+ tmpDat[:] = obsRgnAvg[m, n, :]
+ F.writeReals(tmpDat)
+ for m in np.arange(numMDLs):
+ for n in np.arange(numSubRgn):
+ tmpDat[:] = mdlRgnAvg[m, n, :]
+ F.writeReals(tmpDat)
+ tmpDat = 0 # release the array allocated for tmpDat
+ F.close()
+
+def writeNCfile(fileName, numSubRgn, lons, lats, obsData, mdlData, obsRgnAvg, mdlRgnAvg, obsList, mdlList, subRegions):
+ # write an output file of variables up to 3 dimensions
+ # fileName: the name of the output data file
+ # numSubRgn : the number of subregions
+ # lons[ngrdX]: longitude
+ # lats[ngrdY]: latitudes
+ # obsData[nT,ngrdY,ngrdX]: the obs time series of the entire model domain
+ # mdlData[numMDLs,nT,ngrdY,ngrdX]: the mdltime series of the entire model domain
+ # obsRgnAvg[numSubRgn,nT]: the obs time series for the all subregions
+ # mdlRgnAvg[numMDLs,numSubRgn,nT]: the mdl time series for the all subregions
+ dimO = obsData.shape[0] # the number of obs data
+ dimM = mdlData.shape[0] # the number of mdl data
+ dimT = mdlData.shape[1] # the number of time levels
+ dimY = mdlData.shape[2] # y-dimension
+ dimX = mdlData.shape[3] # x-dimension
+ dimR = obsRgnAvg.shape[1] # the number of subregions
+ f = Nio.open_file(fileName, mode='w', format='nc')
+ print mdlRgnAvg.shape, dimM, dimR, dimT
+ #create global attributes
+ f.globalAttName = ''
+ # create dimensions
+ print 'Creating Dimensions within the NetCDF Object...'
+ f.create_dimension('unity', 1)
+ f.create_dimension('time', dimT)
+ f.create_dimension('west_east', dimX)
+ f.create_dimension('south_north', dimY)
+ f.create_dimension('obs', dimO)
+ f.create_dimension('models', dimM)
+
+ # create the variable (real*4) to be written in the file
+ print 'Creating Variables...'
+ f.create_variable('lon', 'd', ('south_north', 'west_east'))
+ f.create_variable('lat', 'd', ('south_north', 'west_east'))
+ f.create_variable('oDat', 'd', ('obs', 'time', 'south_north', 'west_east'))
+ f.create_variable('mDat', 'd', ('models', 'time', 'south_north', 'west_east'))
+
+ if subRegions:
+ f.create_dimension('regions', dimR)
+ f.create_variable('oRgn', 'd', ('obs', 'regions', 'time'))
+ f.create_variable('mRgn', 'd', ('models', 'regions', 'time'))
+ f.variables['oRgn'].varAttName = 'Observation time series: Subregions'
+ f.variables['mRgn'].varAttName = 'Model time series: Subregions'
+
+ loadDataIntoNetCDF(f, obsList, obsData, 'Observation')
+ print 'Loaded the Observations into the NetCDF'
+
+ loadDataIntoNetCDF(f, mdlList, mdlData, 'Model')
+
+ # create attributes and units for the variable
+ print 'Creating Attributes and Units...'
+ f.variables['lon'].varAttName = 'Longitudes'
+ f.variables['lon'].varUnit = 'degrees East'
+ f.variables['lat'].varAttName = 'Latitudes'
+ f.variables['lat'].varUnit = 'degrees North'
+ f.variables['oDat'].varAttName = 'Observation time series: entire domain'
+ f.variables['mDat'].varAttName = 'Model time series: entire domain'
+
+ # assign the values to the variable and write it
+ f.variables['lon'][:, :] = lons
+ f.variables['lat'][:, :] = lats
+ if subRegions:
+ f.variables['oRgn'][:, :, :] = obsRgnAvg
+ f.variables['mRgn'][:, :, :] = mdlRgnAvg
+
+ f.close()
+
+def loadDataIntoNetCDF(fileObject, datasets, dataArray, dataType):
+ """
+ Input::
+ fileObject - PyNIO file object data will be loaded into
+ datasets - List of dataset names
+ dataArray - Multi-dimensional array of data to be loaded into the NetCDF file
+ dataType - String with value of either 'Model' or 'Observation'
+ Output::
+ No return value. PyNIO file object is updated in place
+ """
+ datasetCount = 0
+ for dataset in datasets:
+ if dataType.lower() == 'observation':
+ datasetName = dataset.replace(' ','')
+ elif dataType.lower() == 'model':
+ datasetName = path.splitext(path.basename(dataset))[0]
+ print "Creating variable %s" % datasetName
+ fileObject.create_variable(datasetName, 'd', ('time', 'south_north', 'west_east'))
+ fileObject.variables[datasetName].varAttName = 'Obseration time series: entire domain'
+ print 'Loading values into %s' % datasetName
+ fileObject.variables[datasetName].assign_value(dataArray[datasetCount,:,:,:])
+ datasetCount += 1
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
new file mode 100644
index 0000000..37e1cc8
--- /dev/null
+++ b/mccsearch/code/mainProg.py
@@ -0,0 +1,142 @@
+'''
+# running the program
+# Kim Whitehall Nov 2012
+# Last updated: 15 May 2013
+'''
+
+#import rcmes
+import sys
+import networkx as nx
+import mccSearch
+#import cloudclusters
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+import subprocess
+
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy"
+ #mainDirStr= "/home/whitehal/mccsearch/caseStudy2"
+ #directories for the original data
+ #TRMMdirName ="/home/whitehal/summer2006TRMM"
+ #CEoriDirName = "/home/whitehal/17Jul30Sep/mergNETCDF"
+ TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
+ CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
+ #allMCSsDict ={{}}
+
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # sys.exit()
+
+ # # mccSearch.plotAccTRMM(TRMMCEdirName)
+ # sys.exit()
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # sys.exit()
+
+ #using merg data ***********
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName isnt entered for what ever reason
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
+ # sys.exit()
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ # for eachList in MCCList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print "MCS List has been acquired ", len(MCSList)
+ # for eachList in MCSList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING TRMM ----------"
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+ #MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ print ("-"*80)
+ #print "MCCTimes is: ", MCCTimes
+ #mccSearch.plotAccTRMM(MCCList)
+ mccSearch.displayPrecip(MCCList)
+ mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ mccSearch.displaySize(MCCList)
+ mccSearch.displayPrecip(MCCList)
+ mccSearch.plotHistograms(MCCList)
+ print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+
+ # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
+ # print "Histogram is ", hist, bin_edges
+ # plt.bar(bin_edges[:-1], hist, width = 10)
+ # plt.xlim(min(bin_edges), max(bin_edges))
+ # plt.show()
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistograms(MCCList)
+ #sys.exit()
+ #
+ print ("-"*80)
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print "********** findPrecipRate found"
+ # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print ("-"*80)
+ # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
+ # print ("-"*80)
+ # # sys.exit()
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/mainProgGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgGoyens.py b/mccsearch/code/mainProgGoyens.py
new file mode 100644
index 0000000..ad54ab1
--- /dev/null
+++ b/mccsearch/code/mainProgGoyens.py
@@ -0,0 +1,126 @@
+'''
+# running the program
+# Kim Whitehall Nov 2012
+# Last updated: 15 May 2013
+'''
+
+#import rcmes
+import sys
+import networkx as nx
+import mccSearchGoyens
+#import cloudclusters
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccSearchGoyens/caseStudy1"
+ #directories for the original data
+ TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
+ CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
+ #allMCSsDict ={{}}
+
+ # mccSearchGoyens.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # sys.exit()
+
+ # # mccSearchGoyens.plotAccTRMM(TRMMCEdirName)
+ # sys.exit()
+
+ #create main directory and file structure for storing intel
+ mccSearchGoyens.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # #mccSearchGoyens.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearchGoyens.postProcessingNetCDF(2)
+ # sys.exit()
+
+ #using merg data ***********
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearchGoyens.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName isnt entered for what ever reason
+ # CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearchGoyens.findPrecipRate(TRMMdirName)
+ # sys.exit()
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearchGoyens.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearchGoyens.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ # for eachList in MCCList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
+ print "MCS List has been acquired ", len(MCSList)
+ # for eachList in MCSList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING TRMM ----------"
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+ #MCCTimes, tdelta = mccSearchGoyens.temporalAndAreaInfoMetric(MCCList)
+ print ("-"*80)
+ #print "MCCTimes is: ", MCCTimes
+ print "creating the MCS userfile ", mccSearchGoyens.createTextFile(MCCList)
+
+ # print "number of MCCs is: ", mccSearchGoyens.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearchGoyens.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearchGoyens.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearchGoyens.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearchGoyens.averageFeatureSize(MCCList), "km^2"
+
+ # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
+ # hist, bin_edges = mccSearchGoyens.commonFeatureSize(allMCSsList)
+ # print "Histogram is ", hist, bin_edges
+ # plt.bar(bin_edges[:-1], hist, width = 10)
+ # plt.xlim(min(bin_edges), max(bin_edges))
+ # plt.show()
+
+ #some plots that work
+ # mccSearchGoyens.plotAccTRMM(MCCList)
+ # mccSearchGoyens.displayPrecip(MCCList)
+ # mccSearchGoyens.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearchGoyens.displaySize(MCCList)
+ # mccSearchGoyens.displayPrecip(MCCList)
+ # mccSearchGoyens.plotHistograms(MCCList)
+ #sys.exit()
+ #
+ print ("-"*80)
+
+ # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
+ # print "********** findPrecipRate found"
+ # allCETRMMList = mccSearchGoyens.findPrecipRate(CEdirName,TRMMdirName)
+ # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
+
+ # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
+ # print ("-"*80)
+ # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
+ # print ("-"*80)
+ # # sys.exit()
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/mainProgWorks.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgWorks.py b/mccsearch/code/mainProgWorks.py
new file mode 100644
index 0000000..f6eb55d
--- /dev/null
+++ b/mccsearch/code/mainProgWorks.py
@@ -0,0 +1,124 @@
+'''
+# running the program
+# Kim Whitehall Nov 2012
+# Last updated: 15 May 2013
+'''
+
+#import rcmes
+import sys
+import networkx as nx
+import mccSearch
+#import cloudclusters
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+ #directories for the original data
+ TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
+ CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
+ #allMCSsDict ={{}}
+
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # sys.exit()
+
+ # # mccSearch.plotAccTRMM(TRMMCEdirName)
+ # sys.exit()
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # sys.exit()
+
+ #using merg data ***********
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName isnt entered for what ever reason
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
+ # sys.exit()
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ # for eachList in MCCList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print "MCS List has been acquired ", len(MCSList)
+ # for eachList in MCSList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING TRMM ----------"
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+ MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ print ("-"*80)
+ print "MCCTimes is: ", MCCTimes
+ print ("-"*80)
+ print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
+ # print "Histogram is ", hist, bin_edges
+ # plt.bar(bin_edges[:-1], hist, width = 10)
+ # plt.xlim(min(bin_edges), max(bin_edges))
+ # plt.show()
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistograms(MCCList)
+ #sys.exit()
+ #
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print "********** findPrecipRate found"
+ # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print ("-"*80)
+ # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
+ # print ("-"*80)
+ # # sys.exit()
+
+main()
\ No newline at end of file
[09/50] [abbrv] git commit: 1 Updated doc strings,
removed print statements and updated documentation 2 Refactored
checkCriteriaB to be generic function checkCriteria 3 Also fixed
findPrecipRate
Posted by le...@apache.org.
1 Updated doc strings, removed print statements and updated
documentation
2 Refactored checkCriteriaB to be generic function checkCriteria
3 Also fixed findPrecipRate
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/5e1a3d1c
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/5e1a3d1c
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/5e1a3d1c
Branch: refs/heads/master
Commit: 5e1a3d1c3adad1df0d0db78a9711b9d0480932b2
Parents: aed34f0
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 09:01:33 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Sun Apr 13 09:01:33 2014 -0400
----------------------------------------------------------------------
mccsearch/code/files.pyc | Bin 0 -> 18453 bytes
mccsearch/code/mainProgTemp.py | 99 ++++++++++++++++++++++++++++++++++++
mccsearch/code/mccSearch.pyc | Bin 0 -> 98227 bytes
mccsearch/code/process.pyc | Bin 0 -> 29786 bytes
4 files changed, 99 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/5e1a3d1c/mccsearch/code/files.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.pyc b/mccsearch/code/files.pyc
new file mode 100644
index 0000000..f8a5577
Binary files /dev/null and b/mccsearch/code/files.pyc differ
http://git-wip-us.apache.org/repos/asf/climate/blob/5e1a3d1c/mccsearch/code/mainProgTemp.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemp.py b/mccsearch/code/mainProgTemp.py
new file mode 100644
index 0000000..9be65b5
--- /dev/null
+++ b/mccsearch/code/mainProgTemp.py
@@ -0,0 +1,99 @@
+'''
+# running the program
+'''
+
+import sys
+import networkx as nx
+import mccSearch
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+import subprocess
+
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+
+ #for GrADs
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ mainDirStr= "/directory/to/where/to/store/outputs"
+ TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+
+ #for first time working with the raw MERG zipped files
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # ---------------------------------------------------------------------------------
+
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ # mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # -------------------------------------------------------------------------------------------------
+
+ #let's go!
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ #print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
+ # ----------------------------------------------------------------------------------------------
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ print "MCS List has been acquired ", len(MCSList)
+ print ("-"*80)
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+
+ #some calculations/metrics that work that work
+ # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistogram(MCCList)
+ #
+ print ("-"*80)
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/5e1a3d1c/mccsearch/code/mccSearch.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.pyc b/mccsearch/code/mccSearch.pyc
new file mode 100644
index 0000000..dc462d1
Binary files /dev/null and b/mccsearch/code/mccSearch.pyc differ
http://git-wip-us.apache.org/repos/asf/climate/blob/5e1a3d1c/mccsearch/code/process.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.pyc b/mccsearch/code/process.pyc
new file mode 100644
index 0000000..d0133fc
Binary files /dev/null and b/mccsearch/code/process.pyc differ
[08/50] [abbrv] git commit: Some linting and refactoring
Posted by le...@apache.org.
Some linting and refactoring
1 Updated doc strings, removed print statements and updated
documentation
2 Refactored checkCriteriaB to be generic function checkCriteria
3 Also fixed findPrecipRate
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/aed34f0b
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/aed34f0b
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/aed34f0b
Branch: refs/heads/master
Commit: aed34f0b8135694d4cc3e97d8f5960b89343f1c2
Parents: 5f3a852
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 08:55:11 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Sun Apr 13 08:55:11 2014 -0400
----------------------------------------------------------------------
mccsearch/README.docx | Bin 0 -> 195234 bytes
mccsearch/code/mainProg.py | 19 +-
mccsearch/code/mccSearch.py | 1833 +++++++++++---------------------------
3 files changed, 531 insertions(+), 1321 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/aed34f0b/mccsearch/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/README.docx b/mccsearch/README.docx
new file mode 100644
index 0000000..a0a7d1e
Binary files /dev/null and b/mccsearch/README.docx differ
http://git-wip-us.apache.org/repos/asf/climate/blob/aed34f0b/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 37e1cc8..90fb76d 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -26,7 +26,7 @@ def main():
allCETRMMList =[]
subprocess.call('export DISPLAY=:0.0', shell=True)
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy"
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy1"
#mainDirStr= "/home/whitehal/mccsearch/caseStudy2"
#directories for the original data
#TRMMdirName ="/home/whitehal/summer2006TRMM"
@@ -55,15 +55,12 @@ def main():
mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
print ("-"*80)
- print 'in main', len(mergImgs)
- print 'timeList', timeList
- print 'TRMMdirName ', TRMMdirName
print "\n -------------- TESTING findCloudElements ----------"
CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
#if the TRMMdirName isnt entered for what ever reason
# CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
- # sys.exit()
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
+ #sys.exit()
print ("-"*80)
print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
print ("-"*80)
@@ -95,11 +92,11 @@ def main():
print ("-"*80)
#print "MCCTimes is: ", MCCTimes
#mccSearch.plotAccTRMM(MCCList)
- mccSearch.displayPrecip(MCCList)
- mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- mccSearch.displaySize(MCCList)
- mccSearch.displayPrecip(MCCList)
- mccSearch.plotHistograms(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistogram(MCCList)
print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
[49/50] [abbrv] git commit: Smarten up documentation for mccsearch
branch.
Posted by le...@apache.org.
Smarten up documentation for mccsearch branch.
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/f9ae9c2d
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/f9ae9c2d
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/f9ae9c2d
Branch: refs/heads/master
Commit: f9ae9c2d85adc15cff578f9a272198c6121cbcff
Parents: 232601c
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 11:00:14 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/README.md | 2 +-
mccsearch/docs/mccsearch.md | 18 +++++++++---------
2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/f9ae9c2d/mccsearch/README.md
----------------------------------------------------------------------
diff --git a/mccsearch/README.md b/mccsearch/README.md
index 6ee45e9..e337ce9 100644
--- a/mccsearch/README.md
+++ b/mccsearch/README.md
@@ -1,3 +1,3 @@
#Graph-based Search for the Identification and Chatacterization of Mesoscale Convective Complexes
-Please see /docs directory for all current documentation relating to this module.
\ No newline at end of file
+Please see /docs/mccsearch.md document for all current documentation relating to this module.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/f9ae9c2d/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
index 4948228..466e027 100644
--- a/mccsearch/docs/mccsearch.md
+++ b/mccsearch/docs/mccsearch.md
@@ -19,21 +19,21 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
* LATS4D
##Download the source code and store in a folder
- (1) mccSearch.py contains all the function needed
- (2) mccSearchUI.py contains a wizard type Q&A for running the program
- (3) process.py contains some needed functions (from older version of Apache OCW)
- (4) file.py contains some needed functions (from older version of Apache OCW)
- (5) mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
+ * mccSearch.py contains all the function needed
+ * mccSearchUI.py contains a wizard type Q&A for running the program
+ * process.py contains some needed functions (from older version of Apache OCW)
+ * file.py contains some needed functions (from older version of Apache OCW)
+ * mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
* mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be store
* TRMMdirName : this is the directory where the original TRMM data in netCDF format is stored
* CEoriDirName : this is the directory where the original MERG data in netCDF format is stored
- (6) Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
+ * Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
##Download data in one folder (with only the data files)
This is a very crude program. The following assumptions are made:
- (1) input data are in one folder. For MERG data this is CEoriDirName and for the TRMM data this is TRMMdirName in mainProg.py. These directories cannot be the same.
- (2) THERE IS NO FILE CHECKING. So please ensure ALL your files are there in netCDF format.
- (3) THERE IS NO FILE ERROR HANDLING. Please ensure that the MERG data and the TRMM data files are correlated temporally and spatially
+ * input data are in one folder. For MERG data this is CEoriDirName and for the TRMM data this is TRMMdirName in mainProg.py. These directories cannot be the same.
+ * THERE IS NO FILE CHECKING. So please ensure ALL your files are there in netCDF format.
+ * THERE IS NO FILE ERROR HANDLING. Please ensure that the MERG data and the TRMM data files are correlated temporally and spatially
##Run mccSearchUI.py
As a first try to determine the workflow, run the mccSearchUI.py wizard.
[41/50] [abbrv] git commit: Smarten up documentation for mccsearch
branch and move it into a docs directory
Posted by le...@apache.org.
Smarten up documentation for mccsearch branch and move it into a docs directory
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/232601c0
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/232601c0
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/232601c0
Branch: refs/heads/master
Commit: 232601c0010cebb6b9293984372fea96ee7834f5
Parents: 37d837f
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 10:57:52 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/README.docx | Bin 192118 -> 0 bytes
mccsearch/README.md | 3 +++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/232601c0/mccsearch/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/README.docx b/mccsearch/README.docx
deleted file mode 100644
index 648637e..0000000
Binary files a/mccsearch/README.docx and /dev/null differ
http://git-wip-us.apache.org/repos/asf/climate/blob/232601c0/mccsearch/README.md
----------------------------------------------------------------------
diff --git a/mccsearch/README.md b/mccsearch/README.md
index e69de29..6ee45e9 100644
--- a/mccsearch/README.md
+++ b/mccsearch/README.md
@@ -0,0 +1,3 @@
+#Graph-based Search for the Identification and Chatacterization of Mesoscale Convective Complexes
+
+Please see /docs directory for all current documentation relating to this module.
\ No newline at end of file
[45/50] [abbrv] git commit: Remove reference to Nio,
this has been updated to use netCDF4
Posted by le...@apache.org.
Remove reference to Nio, this has been updated to use netCDF4
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/e4fedfad
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/e4fedfad
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/e4fedfad
Branch: refs/heads/master
Commit: e4fedfad7907eab8e881cb7348e9eb9b050fef7e
Parents: dc09159
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 16:48:09 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/docs/mccsearch.md | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/e4fedfad/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
index 6b3ab1a..a6bb336 100644
--- a/mccsearch/docs/mccsearch.md
+++ b/mccsearch/docs/mccsearch.md
@@ -9,8 +9,7 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
##What is needed?
* Python 2.7.4 (We used Anaconda 1.5.1 64 bit libraries, which installed most of the dependencies.) Other module dependences:
- * Nio - https://www.pyngl.ucar.edu/Nio.shtml
- * netCDF - http://www.unidata.ucar.edu/software/netcdf/
+ * netCDF4 - http://www.unidata.ucar.edu/software/netcdf/
* sciPy - http://www.scipy.org/scipylib/index.html
* NumPy - http://www.scipy.org/scipylib/download.html
* Networkx - https://networkx.github.io/download.html
[46/50] [abbrv] git commit: fixed bug regarding ability to access a
subset of files in a folder
Posted by le...@apache.org.
fixed bug regarding ability to access a subset of files in a folder
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/ce04f722
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/ce04f722
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/ce04f722
Branch: refs/heads/master
Commit: ce04f722940b3a03c578ed99d55e8c8b63e3f837
Parents: 1f2948d
Author: georgette <k_...@yahoo.com>
Authored: Tue Jun 3 09:39:06 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
.DS_Store | Bin 0 -> 6148 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/ce04f722/.DS_Store
----------------------------------------------------------------------
diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000..73f6381
Binary files /dev/null and b/.DS_Store differ
[25/50] [abbrv] git commit: Remove reference to Nio,
this has been updated to use netCDF4
Posted by le...@apache.org.
Remove reference to Nio, this has been updated to use netCDF4
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/ebd2ef2e
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/ebd2ef2e
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/ebd2ef2e
Branch: refs/heads/master
Commit: ebd2ef2efc4e70474423800a1db4973a0ed8a758
Parents: 4d64d21
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 16:48:09 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Mon Aug 25 16:48:09 2014 -0700
----------------------------------------------------------------------
mccsearch/docs/mccsearch.md | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/ebd2ef2e/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
index 6b3ab1a..a6bb336 100644
--- a/mccsearch/docs/mccsearch.md
+++ b/mccsearch/docs/mccsearch.md
@@ -9,8 +9,7 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
##What is needed?
* Python 2.7.4 (We used Anaconda 1.5.1 64 bit libraries, which installed most of the dependencies.) Other module dependences:
- * Nio - https://www.pyngl.ucar.edu/Nio.shtml
- * netCDF - http://www.unidata.ucar.edu/software/netcdf/
+ * netCDF4 - http://www.unidata.ucar.edu/software/netcdf/
* sciPy - http://www.scipy.org/scipylib/index.html
* NumPy - http://www.scipy.org/scipylib/download.html
* Networkx - https://networkx.github.io/download.html
[02/50] [abbrv] git commit: Climate has become a TLP
Posted by le...@apache.org.
Climate has become a TLP
git-svn-id: https://svn.apache.org/repos/asf/climate/branches/0.1-incubating@1573437 13f79535-47bb-0310-9956-ffa450edef68
Former-commit-id: 307dd5c264b81aa4016e355d6a1d60bd28e095b7
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/8158c151
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/8158c151
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/8158c151
Branch: refs/heads/master
Commit: 8158c151a1307cffb0ea55cede3360579010c46e
Parents: d3cb950
Author: Joe Schaefer <jo...@apache.org>
Authored: Mon Mar 3 04:18:24 2014 +0000
Committer: Joe Schaefer <jo...@apache.org>
Committed: Mon Mar 3 04:18:24 2014 +0000
----------------------------------------------------------------------
----------------------------------------------------------------------
[50/50] [abbrv] git commit: Merge branch 'mccsearch' of
https://github.com/kwhitehall/climate into mccsearch
Posted by le...@apache.org.
Merge branch 'mccsearch' of https://github.com/kwhitehall/climate into mccsearch
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/26b6c910
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/26b6c910
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/26b6c910
Branch: refs/heads/master
Commit: 26b6c910c41c72eada1a9de6bbdfa3bc35ed8f01
Parents: e19fe29 8fcd2a2
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Wed Oct 15 15:19:22 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:19:22 2014 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[20/50] [abbrv] git commit: Move docsgit status
Posted by le...@apache.org.
Move docsgit status
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/73f82089
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/73f82089
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/73f82089
Branch: refs/heads/master
Commit: 73f82089e5345a6512cff3b243cff3aefdd92a73
Parents: 21c1c04
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 10:56:57 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Mon Aug 25 10:56:57 2014 -0700
----------------------------------------------------------------------
mccsearch/README.md | 0
mccsearch/docs/README.docx | Bin 0 -> 192118 bytes
mccsearch/docs/mccsearch.md | 58 +++++++++++++++++++++++++++++
mccsearch/docs/mccsearch_workflow.png | Bin 0 -> 132430 bytes
4 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/73f82089/mccsearch/README.md
----------------------------------------------------------------------
diff --git a/mccsearch/README.md b/mccsearch/README.md
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/climate/blob/73f82089/mccsearch/docs/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/docs/README.docx b/mccsearch/docs/README.docx
new file mode 100644
index 0000000..648637e
Binary files /dev/null and b/mccsearch/docs/README.docx differ
http://git-wip-us.apache.org/repos/asf/climate/blob/73f82089/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
new file mode 100644
index 0000000..4948228
--- /dev/null
+++ b/mccsearch/docs/mccsearch.md
@@ -0,0 +1,58 @@
+#Graph-based Search for the Identification and Chatacterization of Mesoscale Convective Complexes
+
+##Very Brief Introduction
+If you are in this file it means that you wish to use the mccSearch.py program, or learn more about it.
+
+##What is this project about?
+This program searches for a weather feature known as a Mesoscale Convective Complex (MCC) in gridded infrared and precipitation rate satellite data (MERG and TRMM datasets have been tested).
+The data is read from netCDF files into arrays with the dimensions time, latitude, longitude, value.
+
+##What is needed?
+ * Python 2.7.4 (We used Anaconda 1.5.1 64 bit libraries, which installed most of the dependencies.) Other module dependences:
+ * Nio
+ * netCDF
+ * sciPy
+ * NumPy
+ * Networkx
+ * matplotlib
+ * GrADS (We used OpenGrADS grads2 Version 2.0.1.oga.1)
+ * LATS4D
+
+##Download the source code and store in a folder
+ (1) mccSearch.py contains all the function needed
+ (2) mccSearchUI.py contains a wizard type Q&A for running the program
+ (3) process.py contains some needed functions (from older version of Apache OCW)
+ (4) file.py contains some needed functions (from older version of Apache OCW)
+ (5) mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
+ * mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be store
+ * TRMMdirName : this is the directory where the original TRMM data in netCDF format is stored
+ * CEoriDirName : this is the directory where the original MERG data in netCDF format is stored
+ (6) Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
+
+##Download data in one folder (with only the data files)
+This is a very crude program. The following assumptions are made:
+ (1) input data are in one folder. For MERG data this is CEoriDirName and for the TRMM data this is TRMMdirName in mainProg.py. These directories cannot be the same.
+ (2) THERE IS NO FILE CHECKING. So please ensure ALL your files are there in netCDF format.
+ (3) THERE IS NO FILE ERROR HANDLING. Please ensure that the MERG data and the TRMM data files are correlated temporally and spatially
+
+##Run mccSearchUI.py
+As a first try to determine the workflow, run the mccSearchUI.py wizard.
+
+##Configure mainProg.py
+Guidance on the workflow can be seen in the figure below. More details on individual functions in mccSearch.py can be found in the DocStrings.
+
+
+The general workflow of the program. The dashed lines indicate optional paths.
+
+##Run mainProg.py
+Keep your fingers & toes crossed!! Once everything went well, the directory you indicated where outputs should be stored will be generated, and four folders should appear in it.
+ * The image folder will store all images generated from plots.
+ * The textFiles folder will store all the text files generated during the run, e.g. cloudElementsUserFile.txt that contains information about each cloud element identified
+ * The MERGnetcdfCEs folder contains the infrared data in clipped netCDF files that have been generated for each cloud element identified
+ * The TRMMnetcdfCEs folder contains the precipitation data in clipped netCDF files that have been generated for each cloud element identified.
+
+##Anticipated future work
+ * Implement functionality for file checking to ensure all files are there
+ * Create a user interface based on mainProg.py.
+ * Plug the project into the Regional Climate Model Evaluation Database (RCMED) that is available at Apache OCW.
+With increased usage, it is anticipated that more metrics and visualizations will be provided.
http://git-wip-us.apache.org/repos/asf/climate/blob/73f82089/mccsearch/docs/mccsearch_workflow.png
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch_workflow.png b/mccsearch/docs/mccsearch_workflow.png
new file mode 100644
index 0000000..42a2680
Binary files /dev/null and b/mccsearch/docs/mccsearch_workflow.png differ
[31/50] [abbrv] git commit: mccserach - some linting
Posted by le...@apache.org.
mccserach - some linting
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/99f6e36c
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/99f6e36c
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/99f6e36c
Branch: refs/heads/master
Commit: 99f6e36c2945d3de544221794df675ac8e6045f3
Parents: 91eafd0
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 09:09:25 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:04 2014 -0700
----------------------------------------------------------------------
mccsearch/code/mainProg.py | 84 +++++++++++------------------------------
1 file changed, 22 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/99f6e36c/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
index 90fb76d..9be65b5 100644
--- a/mccsearch/code/mainProg.py
+++ b/mccsearch/code/mainProg.py
@@ -1,14 +1,10 @@
'''
# running the program
-# Kim Whitehall Nov 2012
-# Last updated: 15 May 2013
'''
-#import rcmes
import sys
import networkx as nx
import mccSearch
-#import cloudclusters
import numpy as np
import numpy.ma as ma
import files
@@ -24,43 +20,43 @@ def main():
MCSMCCNodesList =[]
allMCSsList =[]
allCETRMMList =[]
+
+ #for GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy1"
- #mainDirStr= "/home/whitehal/mccsearch/caseStudy2"
- #directories for the original data
- #TRMMdirName ="/home/whitehal/summer2006TRMM"
- #CEoriDirName = "/home/whitehal/17Jul30Sep/mergNETCDF"
- TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
- CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
- #allMCSsDict ={{}}
+ mainDirStr= "/directory/to/where/to/store/outputs"
+ TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+ #for first time working with the raw MERG zipped files
# mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # sys.exit()
+ # ---------------------------------------------------------------------------------
- # # mccSearch.plotAccTRMM(TRMMCEdirName)
- # sys.exit()
#create main directory and file structure for storing intel
mccSearch.createMainDirectory(mainDirStr)
TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
CEdirName = mainDirStr+'/MERGnetcdfCEs'
- # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ # mccSearch.postProcessingNetCDF(3,CEoriDirName)
# mccSearch.postProcessingNetCDF(2)
- # sys.exit()
+ # -------------------------------------------------------------------------------------------------
- #using merg data ***********
+ #let's go!
print "\n -------------- Read MERG Data ----------"
mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
print ("-"*80)
+ print 'in main', len(mergImgs)
+ #print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
print "\n -------------- TESTING findCloudElements ----------"
CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName isnt entered for what ever reason
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
# CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
# allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
- #sys.exit()
+ # ----------------------------------------------------------------------------------------------
print ("-"*80)
print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
print ("-"*80)
@@ -74,32 +70,15 @@ def main():
MCCList,MCSList = mccSearch.findMCC(prunedGraph)
print ("-"*80)
print "MCC List has been acquired ", len(MCCList)
- # for eachList in MCCList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
print "MCS List has been acquired ", len(MCSList)
- # for eachList in MCSList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING TRMM ----------"
#now ready to perform various calculations/metrics
print "\n -------------- TESTING METRICS ----------"
- #MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
- print ("-"*80)
- #print "MCCTimes is: ", MCCTimes
- #mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- # mccSearch.displaySize(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistogram(MCCList)
- print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
- print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ #some calculations/metrics that work that work
+ # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
# print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
# print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
# print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
@@ -107,33 +86,14 @@ def main():
# print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
# print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
- # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
- # print "Histogram is ", hist, bin_edges
- # plt.bar(bin_edges[:-1], hist, width = 10)
- # plt.xlim(min(bin_edges), max(bin_edges))
- # plt.show()
-
#some plots that work
# mccSearch.plotAccTRMM(MCCList)
# mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
# mccSearch.displaySize(MCCList)
# mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistograms(MCCList)
- #sys.exit()
+ # mccSearch.plotHistogram(MCCList)
#
print ("-"*80)
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print "********** findPrecipRate found"
- # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print ("-"*80)
- # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
- # print ("-"*80)
- # # sys.exit()
main()
\ No newline at end of file
[30/50] [abbrv] git commit: 1 Updated doc strings,
removed print statements and updated documentation 2 Refactored
checkCriteriaB to be generic function checkCriteria 3 Also fixed
findPrecipRate
Posted by le...@apache.org.
1 Updated doc strings, removed print statements and updated
documentation
2 Refactored checkCriteriaB to be generic function checkCriteria
3 Also fixed findPrecipRate
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/91eafd02
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/91eafd02
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/91eafd02
Branch: refs/heads/master
Commit: 91eafd0223115502caf83447e486921af60c69ea
Parents: 93fbf75
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 09:01:33 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:04 2014 -0700
----------------------------------------------------------------------
mccsearch/code/files.pyc | Bin 0 -> 18453 bytes
mccsearch/code/mainProgTemp.py | 99 ++++++++++++++++++++++++++++++++++++
mccsearch/code/mccSearch.pyc | Bin 0 -> 98227 bytes
mccsearch/code/process.pyc | Bin 0 -> 29786 bytes
4 files changed, 99 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/91eafd02/mccsearch/code/files.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.pyc b/mccsearch/code/files.pyc
new file mode 100644
index 0000000..f8a5577
Binary files /dev/null and b/mccsearch/code/files.pyc differ
http://git-wip-us.apache.org/repos/asf/climate/blob/91eafd02/mccsearch/code/mainProgTemp.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemp.py b/mccsearch/code/mainProgTemp.py
new file mode 100644
index 0000000..9be65b5
--- /dev/null
+++ b/mccsearch/code/mainProgTemp.py
@@ -0,0 +1,99 @@
+'''
+# running the program
+'''
+
+import sys
+import networkx as nx
+import mccSearch
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+import subprocess
+
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+
+ #for GrADs
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ mainDirStr= "/directory/to/where/to/store/outputs"
+ TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
+ CEoriDirName = "/directory/to/the/MERG/netCDF/files"
+
+ #for first time working with the raw MERG zipped files
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # ---------------------------------------------------------------------------------
+
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
+ # mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # -------------------------------------------------------------------------------------------------
+
+ #let's go!
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ #print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
+ # ----------------------------------------------------------------------------------------------
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ print "MCS List has been acquired ", len(MCSList)
+ print ("-"*80)
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+
+ #some calculations/metrics that work that work
+ # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+ # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistogram(MCCList)
+ #
+ print ("-"*80)
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/91eafd02/mccsearch/code/mccSearch.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.pyc b/mccsearch/code/mccSearch.pyc
new file mode 100644
index 0000000..dc462d1
Binary files /dev/null and b/mccsearch/code/mccSearch.pyc differ
http://git-wip-us.apache.org/repos/asf/climate/blob/91eafd02/mccsearch/code/process.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.pyc b/mccsearch/code/process.pyc
new file mode 100644
index 0000000..d0133fc
Binary files /dev/null and b/mccsearch/code/process.pyc differ
[13/50] [abbrv] remove unnecessary files
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/mccSearchGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchGoyens.py b/mccsearch/code/mccSearchGoyens.py
deleted file mode 100644
index 0f49a10..0000000
--- a/mccsearch/code/mccSearchGoyens.py
+++ /dev/null
@@ -1,3606 +0,0 @@
-'''
-# All the functions for the MCC search algorithm
-# RCMES data in format (t,lat,lon), value
-# Kim Whitehall Sep 2012
-# Last updated: 23Jan 2014, 6 Dec 2013, 21 May 2013, 4 Jun 2013
-# June 13 - called this merg.py and edited out comments. merg.py is where all "cleaned" functions are to go
-# June 27 - including the first part of graphing, according to the center_of_mass
-# July 1 - going back to brute force graph approach, but weigh the edges according to closeness, up to within 20_perc of the forecasted
-#
-
-Mimumum MCS is 3 hours
-'''
-
-import datetime
-from datetime import timedelta, datetime
-import calendar
-import fileinput
-import glob
-import itertools
-import json
-import math
-import Nio
-from netCDF4 import Dataset, num2date, date2num
-import numpy as np
-import numpy.ma as ma
-import os
-import pickle
-import re
-from scipy import ndimage
-import string
-import subprocess
-import sys
-import time
-
-import networkx as nx
-import matplotlib.pyplot as plt
-import matplotlib.dates as mdates
-from matplotlib.dates import DateFormatter,HourLocator
-from matplotlib import cm
-import matplotlib.cm as cm
-import matplotlib.colors as mcolors
-from matplotlib.ticker import FuncFormatter, FormatStrFormatter
-
-#existing modules in services
-import files
-import process
-#----------------------- GLOBAL VARIABLES --------------------------
-# --------------------- User defined variables ---------------------
-#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
-#the first point closest the the value (for the min) from the MERG data is used, etc.
-LATMIN = '5.0' #'5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
-LATMAX = '20.0' #'20.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
-LONMIN = '0.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
-LONMAX = '30.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
-XRES = 4.0 #x direction spatial resolution in km
-YRES = 4.0 #y direction spatial resolution in km
-TRES = 1 #temporal resolution in hrs
-T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
-T_BB_MIN = 218 #221 #cooler temp for the center of the system
-STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
- #have same rank of the matrix it is being compared against
-CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
-MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
-MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
-MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
-LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
-LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
-DIRECTION = 45.0 #general direction of the wind flow in degrees
-AREA_MIN = 3500.0 #2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
-MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
-
-#---Assuming using the MCC function, these will have to be changed
-ECCENTRICITY_THRESHOLD_MAX = 0.75 #tending to 1 is a circle e.g. hurricane,
-ECCENTRICITY_THRESHOLD_MIN = 0.25 #0.65 #tending to 0 is a linear e.g. squall line
-OUTER_CLOUD_SHIELD_AREA = 30000.0 #80000.0 #100000.0 #km^2
-INNER_CLOUD_SHIELD_AREA = 3000.0#30000.0 #50000.0 #km^2
-OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
-INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
-MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
-MAXIMUM_DURATION = 100 #24 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
-
-#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
-#------------------- End user defined Variables -------------------
-edgeWeight = [1,2,3] #weights for the graph edges
-#graph object fo the CEs meeting the criteria
-CLOUD_ELEMENT_GRAPH = nx.DiGraph()
-#graph meeting the CC criteria
-PRUNED_GRAPH = nx.DiGraph()
-#------------------------ End GLOBAL VARS -------------------------
-#************************ Begin Functions *************************
-#******************************************************************
-def readMergData(dirname):
- '''
- Purpose::
- Read MERG data into RCMES format
-
- Input::
- Directory to the MERG files in NETCDF format
-
- Output::
- A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
- criteria for each frame
- **remove below**
- A dictionary of all the MERG data from the files in the directory given.
- The dictionary contains, a string representing the time (a datetime string), a
- tuple (lat,lon,value) representing only the data that meets the temperature requirements
- i.e. T_BB_MAX
-
- Assumptions::
- The MERG data has been converted to NETCDF using LATS4D
- The data has the same lat/lon format
- '''
-
- global LAT
- global LON
-
- # these strings are specific to the MERG data
- mergVarName = 'ch4'
- mergTimeVarName = 'time'
- mergLatVarName = 'latitude'
- mergLonVarName = 'longitude'
-
- filelistInstructions = dirname + '/*'
- filelist = glob.glob(filelistInstructions)
-
-
- #sat_img is the array that will contain all the masked frames
- mergImgs = []
- #timelist of python time strings
- timelist = []
- time2store = None
- tempMaskedValueNp =[]
-
-
- filelist.sort()
- nfiles = len(filelist)
-
- # Crash nicely if there are no netcdf files
- if nfiles == 0:
- print 'Error: no files in this directory! Exiting elegantly'
- sys.exit()
- else:
- # Open the first file in the list to read in lats, lons and generate the grid for comparison
- tmp = Nio.open_file(filelist[0], format='nc')
- #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
-
- #clip the lat/lon grid according to user input
- #http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
- latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
- lonsraw = tmp.variables[mergLonVarName][mergLonVarName+"|"+LONMIN+":"+LONMAX].astype('f2')
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
-
- LON, LAT = np.meshgrid(lonsraw, latsraw)
- #clean up
- latsraw =[]
- lonsraw = []
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
- #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
- tmp.close
-
- for files in filelist:
- try:
- thisFile = Nio.open_file(files, format='nc')
- #TODO: thisFile = netCDF4.Dataset(files)
-
- #clip the dataset according to user lat,lon coordinates
- #mask the data and fill with zeros for later
- tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
- +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
-
- tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
-
- #get the actual values that the mask returned
- tempMaskedValue = ma.zeros((tempRaw.shape)).astype('int16')
- for index, value in maenumerate(tempMask):
- time_index, lat_index, lon_index = index
- tempMaskedValue[time_index,lat_index, lon_index]=value
-
- timesRaw = thisFile.variables[mergTimeVarName]
- #convert this time to a python datastring
- time2store, _ = process.getModelTimes(files, mergTimeVarName)
- #extend instead of append because getModelTimes returns a list already and we don't
- #want a list of list
- timelist.extend(time2store)
- #to get the actual masked data only
- #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
- mergImgs.extend(tempMaskedValue)
- thisFile.close
- thisFile = None
-
- except:
- print "bad file! ", file
-
- mergImgs = ma.array(mergImgs)
-
- return mergImgs, timelist
-#******************************************************************
-def findCloudElements(mergImgs,timelist,TRMMdirName=None):
- '''
- Purpose::
- Determines the contiguous boxes for a given time of the satellite imaages i.e. each frame
- It recursively calls another function to determine if the boxes are touching
-
- Input::
- 2 variables
- sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
- maximum acceptable temperature, T_BB_MAX
- timelist: a list of python datatimes
- TRMMdirName (optional): string representing the path where to find the TRMM datafiles
-
- Output::
- 1 variable
- cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
- The dictionary is cloudElements={"frame": datetime object,
- "cloudElementNum": integer,
- "CEuniqueID": string,
- "cloudElementLatLon": 2D array of lat_lon}
- "cloudElementCenter": 1D array [0] is lat, [1]is lon of the center_of_mass}
- "cloudElementEccentricity: float"
-
- Assumptions::
- Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
- required according to Vila et al. (2008) is 2400km^2
- therefore, 2400/16 = 150 contiguous squares
- '''
-
- frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
- CEcounter = 0
- frameCEcounter = 0
- frameNum = 0
- cloudElementEpsilon = 0.0
- cloudElementDict = {}
- cloudElementCenter = [] #list with two elements [lat,lon] for the center of a CE
- prevFrameCEs = [] #list for CEs in previous frame
- currFrameCEs = [] #list for CEs in current frame
- cloudElementLat = [] #list for a particular CE's lat values
- cloudElementLon = [] #list for a particular CE's lon values
- cloudElementLatLons = [] #list for a particular CE's (lat,lon) values
-
- prevLatValue = 0.0
- prevLonValue = 0.0
- TIR_min = 0.0
- TIR_max = 0.0
- temporalRes = 3 # TRMM data is 3 hourly
- precipTotal = 0.0
- CETRMMList =[]
- precip =[]
- TRMMCloudElementLatLons =[]
-
- #edgeWeight = [1,2]
- minCELatLimit = 0.0
- minCELonLimit = 0.0
- maxCELatLimit = 0.0
- maxCELonLimit = 0.0
-
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
-
- #openfile for storing ALL cloudElement information
- cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
-
-
- #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
- cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
- #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
-
- #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
- for t in xrange(mergImgs.shape[0]):
- #-------------------------------------------------
- # #textfile name for saving the data for arcgis
- # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt'
- # cloudElementsTextFile = open(thisFileName,'w')
- #-------------------------------------------------
-
- #determine contiguous locations with temeperature below the warmest temp i.e. cloudElements in each frame
- frame, CEcounter = ndimage.measurements.label(mergImgs[t,:,:], structure=STRUCTURING_ELEMENT)
- frameCEcounter=0
- frameNum += 1
-
- #for each of the areas identified, check to determine if it a valid CE via an area and T requirement
- for count in xrange(CEcounter):
- #[0] is time dimension. Determine the actual values from the data
- #loc is a masked array
- loc = ndimage.find_objects(frame==(count+1))[0]
- cloudElement = mergImgs[t,:,:][loc]
- labels, lcounter = ndimage.label(cloudElement)
-
- #determine the true lats and lons for this particular CE
- cloudElementLat = LAT[loc[0],0]
- cloudElementLon = LON[0,loc[1]]
-
- #determine number of boxes in this cloudelement
- numOfBoxes = np.count_nonzero(cloudElement)
- cloudElementArea = numOfBoxes*XRES*YRES
-
- #If the area is greater than the area required, then start considering as CE
- #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
- #if it does, still consider as CE
-
- if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
- #do a second pass on the identified area
- # if lcounter > 1:
- # #i.e.there is more than one item in the area identified
- # for lcount in xrange(lcounter):
- # print " in lcount for", lcount
- # loc = ndimage.find_objects(labels ==(lcount+1))[0]
- # lcloudElement = cloudElement[:,:][loc]
- # llabels, _ = ndimage.label(lcloudElement)
- # plt.imshow(llabels)
- # plt.show()
-
-
- #get some time information and labeling info
- frameTime = str(timelist[t])
- frameCEcounter +=1
- CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
-
- #-------------------------------------------------
- #textfile name for accesing CE data using MATLAB code
- # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
- # cloudElementsTextFile = open(thisFileName,'w')
- #-------------------------------------------------
-
- # ------ NETCDF File stuff for brightness temp stuff ------------------------------------
- thisFileName = MAINDIRECTORY +'/MERGnetcdfCEs/cloudElements'+ (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
- currNetCDFCEData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFCEData.description = 'Cloud Element '+CEuniqueID + ' temperature data'
- currNetCDFCEData.calendar = 'standard'
- currNetCDFCEData.conventions = 'COARDS'
- # dimensions
- currNetCDFCEData.createDimension('time', None)
- currNetCDFCEData.createDimension('lat', len(LAT[:,0]))
- currNetCDFCEData.createDimension('lon', len(LON[0,:]))
- # variables
- tempDims = ('time','lat', 'lon',)
- times = currNetCDFCEData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitudes = currNetCDFCEData.createVariable('latitude', 'f8', ('lat',))
- longitudes = currNetCDFCEData.createVariable('longitude', 'f8', ('lon',))
- brightnesstemp = currNetCDFCEData.createVariable('brightnesstemp', 'i16',tempDims )
- brightnesstemp.units = 'Kelvin'
- # NETCDF data
- dates=[timelist[t]+timedelta(hours=0)]
- times[:] = date2num(dates,units=times.units)
- longitudes[:] = LON[0,:]
- longitudes.units = "degrees_east"
- longitudes.long_name = "Longitude"
-
- latitudes[:] = LAT[:,0]
- latitudes.units = "degrees_north"
- latitudes.long_name ="Latitude"
-
- #generate array of zeros for brightness temperature
- brightnesstemp1 = ma.zeros((1,len(latitudes), len(longitudes))).astype('int16')
- #-----------End most of NETCDF file stuff ------------------------------------
-
- #if other dataset (TRMM) assumed to be a precipitation dataset was entered
- if TRMMdirName:
- #------------------TRMM stuff -------------------------------------------------
- fileDate = ((str(timelist[t])).replace(" ", "")[:-8]).replace("-","")
- fileHr1 = (str(timelist[t])).replace(" ", "")[-8:-6]
-
- if int(fileHr1) % temporalRes == 0:
- fileHr = fileHr1
- else:
- fileHr = (int(fileHr1)/temporalRes) * temporalRes
- if fileHr < 10:
- fileHr = '0'+str(fileHr)
- else:
- str(fileHr)
-
- #open TRMM file for the resolution info and to create the appropriate sized grid
- TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+str(fileHr)+".7A.nc"
-
- TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
- precipRate = TRMMData.variables['pcp'][:,:,:]
- latsrawTRMMData = TRMMData.variables['latitude'][:]
- lonsrawTRMMData = TRMMData.variables['longitude'][:]
- lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
- LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
-
- nygrdTRMM = len(LATTRMM[:,0]); nxgrdTRMM = len(LONTRMM[0,:])
- precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
- #---------regrid the TRMM data to the MERG dataset ----------------------------------
- #regrid using the do_regrid stuff from the Apache OCW
- regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
- regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
- #----------------------------------------------------------------------------------
-
- # #get the lat/lon info from cloudElement
- #get the lat/lon info from the file
- latCEStart = LAT[0][0]
- latCEEnd = LAT[-1][0]
- lonCEStart = LON[0][0]
- lonCEEnd = LON[0][-1]
-
- #get the lat/lon info for TRMM data (different resolution)
- latStartT = find_nearest(latsrawTRMMData, latCEStart)
- latEndT = find_nearest(latsrawTRMMData, latCEEnd)
- lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
- lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
- latStartIndex = np.where(latsrawTRMMData == latStartT)
- latEndIndex = np.where(latsrawTRMMData == latEndT)
- lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
- lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
-
- #get the relevant TRMM info
- CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
- TRMMData.close()
-
- # ------ NETCDF File info for writing TRMM CE rainfall ------------------------------------
- thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
- currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' precipitation data'
- currNetCDFTRMMData.calendar = 'standard'
- currNetCDFTRMMData.conventions = 'COARDS'
- # dimensions
- currNetCDFTRMMData.createDimension('time', None)
- currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
- currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
-
- # variables
- TRMMprecip = ('time','lat', 'lon',)
- times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
- longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
- rainFallacc.units = 'mm'
-
- longitude[:] = LON[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
-
- latitude[:] = LAT[:,0]
- latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
-
- finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
- #-----------End most of NETCDF file stuff ------------------------------------
-
- #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
- for index,value in np.ndenumerate(cloudElement):
- if value != 0 :
- lat_index,lon_index = index
- lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value)
-
- #generate the comma separated file for GIS
- cloudElementLatLons.append(lat_lon_tuple)
-
- #temp data for CE NETCDF file
- brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value
-
- if TRMMdirName:
- finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])]
- CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]]))
-
-
- brightnesstemp[:] = brightnesstemp1[:]
- currNetCDFCEData.close()
-
- if TRMMdirName:
-
- #print finalCETRMMvalues
- #calculate the total precip associated with the feature
- for index, value in np.ndenumerate(finalCETRMMvalues):
- precipTotal += value
- precip.append(value)
-
- rainFallacc[:] = finalCETRMMvalues[:]
- currNetCDFTRMMData.close()
- TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
- TRMMArea = TRMMnumOfBoxes*XRES*YRES
- try:
- maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
- minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
- except:
- pass
-
- #sort cloudElementLatLons by lats
- cloudElementLatLons.sort(key=lambda tup: tup[0])
-
- #determine if the cloud element the shape
- cloudElementEpsilon = eccentricity (cloudElement)
- cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
- cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
-
- #latCenter and lonCenter are given according to the particular array defining this CE
- #so you need to convert this value to the overall domain truth
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementCenter.append(latCenter)
- cloudElementCenter.append(lonCenter)
- cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
- cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
- cloudElementsUserFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
- #populate the dictionary
- if TRMMdirName:
- cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min, 'cloudElementPrecipTotal':precipTotal,'cloudElementLatLonTRMM':CETRMMList, 'TRMMArea': TRMMArea}
- else:
- cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min,}
-
- #current frame list of CEs
- currFrameCEs.append(cloudElementDict)
-
- #draw the graph node
- CLOUD_ELEMENT_GRAPH.add_node(CEuniqueID, cloudElementDict)
-
- if frameNum != 1:
- for cloudElementDict in prevFrameCEs:
- thisCElen = len(cloudElementLatLons)
- percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
-
- #change weights to integers because the built in shortest path chokes on floating pts
- #according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
- if percentageOverlap >= 0.95:
- CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
-
- elif percentageOverlap >= 0.90 and percentageOverlap < 0.95 :
- CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[1])
-
- elif areaOverlap >= MIN_OVERLAP:
- CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
-
- else:
- #TODO: remove this else as we only wish for the CE details
- #ensure only the non-zero elements are considered
- #store intel in allCE file
- labels, _ = ndimage.label(cloudElement)
- cloudElementsFile.write("\n-----------------------------------------------")
- cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
- # cloudElementLat = LAT[loc[0],0]
- # cloudElementLon = LON[0,loc[1]]
-
- #populate cloudElementLatLons by unpacking the original values from loc
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
- for index,value in np.ndenumerate(cloudElement):
- if value != 0 :
- lat_index,lon_index = index
- lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
- cloudElementLatLons.append(lat_lon_tuple)
-
- cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
- #latCenter and lonCenter are given according to the particular array defining this CE
- #so you need to convert this value to the overall domain truth
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
- cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
- cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
- cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
- cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
- cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
- cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
- cloudElementsFile.write("\n-----------------------------------------------")
-
- #reset list for the next CE
- nodeExist = False
- cloudElementCenter=[]
- cloudElement = []
- cloudElementCenter = []
- cloudElementLat=[]
- cloudElementLon =[]
- cloudElementLatLons =[]
- brightnesstemp1 =[]
- brightnesstemp =[]
- finalCETRMMvalues =[]
- CEprecipRate =[]
- CETRMMList =[]
- precipTotal = 0.0
- precip=[]
- TRMMCloudElementLatLons=[]
-
- #reset for the next time
- prevFrameCEs =[]
- prevFrameCEs = currFrameCEs
- currFrameCEs =[]
-
- cloudElementsFile.close
- cloudElementsUserFile.close
- #if using ARCGIS data store code, uncomment this file close line
- #cloudElementsTextFile.close
-
- #clean up graph - remove parent and childless nodes
- outAndInDeg = CLOUD_ELEMENT_GRAPH.degree_iter()
- toRemove = [node[0] for node in outAndInDeg if node[1]<1]
- CLOUD_ELEMENT_GRAPH.remove_nodes_from(toRemove)
-
- print "number of nodes are: ", CLOUD_ELEMENT_GRAPH.number_of_nodes()
- print "number of edges are: ", CLOUD_ELEMENT_GRAPH.number_of_edges()
- print ("*"*80)
-
- #hierachial graph output
- #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
- drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
-
- return CLOUD_ELEMENT_GRAPH
-#******************************************************************
-def findPrecipRate(TRMMdirName):
- '''
- TODO: needs fixing
- Purpose:: Determines the precipitation rates for MCSs found
-
- Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
-
- Output:: a list of dictionary of the TRMM data
- NB: also creates netCDF with TRMM data for each CE (for post processing) index
- in MAINDIRECTORY/TRMMnetcdfCEs
-
- Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
-
- '''
- allCEnodesTRMMdata =[]
- TRMMdataDict={}
- precipTotal = 0.0
-
- os.chdir((MAINDIRECTORY+'/MERGnetcdfCEs/'))
- imgFilename = ''
- temporalRes = 3 #3 hours for TRMM
-
- #sort files
- files = filter(os.path.isfile, glob.glob("*.nc"))
- files.sort(key=lambda x: os.path.getmtime(x))
-
- for afile in files:
- #try:
- print "in for ", afile
- fullFname = os.path.splitext(afile)[0]
- noFrameExtension = (fullFname.replace("_","")).split('F')[0]
- CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
- fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
- fileDateTime = fileDateTimeChar.replace("-","")
- fileDate = fileDateTime[:-6]
- fileHr=fileDateTime[-6:-4]
-
- cloudElementData = Dataset(afile,'r', format='NETCDF4')
- brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
- latsrawCloudElements = cloudElementData.variables['latitude'][:]
- lonsrawCloudElements = cloudElementData.variables['longitude'][:]
-
- # print "noFrameExtension", noFrameExtension
- # print "fileDateTime ", fileDateTime
- # print "fileDateTimeChar ", fileDateTimeChar
- # print "fullFname ", fullFname
- # print "time is: ", (fullFname.replace("_"," ").split('F')[0]).split('s')[1]
- # print "fileDate and fileHr ", fileDate, fileHr
- # print "CEuniqueID ", CEuniqueID
- # sys.exit()
- if int(fileHr) % 3 == 0:
- TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
- #print "TRMMfileName is ", TRMMfileName
- TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
- precipRate = TRMMData.variables['pcp'][:,:,:]
- latsrawTRMMData = TRMMData.variables['latitude'][:]
- lonsrawTRMMData = TRMMData.variables['longitude'][:]
- lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
- LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
-
- #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
-
- precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
- #---------regrid the TRMM data to the MERG dataset ----------------------------------
- #regrid using the do_regrid stuff from the Apache OCW
- regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
- regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
- #----------------------------------------------------------------------------------
-
- # #get the lat/lon info from cloudElement
- latCEStart = LAT[0][0]
- latCEEnd = LAT[-1][0]
- lonCEStart = LON[0][0]
- lonCEEnd = LON[0][-1]
-
- #get the lat/lon info for TRMM data (different resolution)
- latStartT = find_nearest(latsrawTRMMData, latCEStart)
- latEndT = find_nearest(latsrawTRMMData, latCEEnd)
- lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
- lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
- latStartIndex = np.where(latsrawTRMMData == latStartT)
- latEndIndex = np.where(latsrawTRMMData == latEndT)
- lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
- lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
-
- #get the relevant TRMM info
- CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
- TRMMData.close()
-
-
-
- # ------ NETCDF File stuff ------------------------------------
- thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
- # print "thisFileName ", thisFileName
- # sys.exit()
- currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
- currNetCDFData.calendar = 'standard'
- currNetCDFData.conventions = 'COARDS'
- # dimensions
- currNetCDFData.createDimension('time', None)
- currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
- currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
- # variables
- TRMMprecip = ('time','lat', 'lon',)
- times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
- longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
- rainFallacc.units = 'mm'
-
- longitude[:] = LON[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
-
- latitude[:] = LAT[:,0]
- latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
-
- finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
- #-----------End most of NETCDF file stuff ------------------------------------
- #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
- for index,value in np.ndenumerate(brightnesstemp):
- #print "index, value ", index, value
- time_index, lat_index, lon_index = index
- currTimeValue = 0
- if value > 0:
- finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
-
-
- # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
- # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
- # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
- # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
- # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
- # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
- # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
- # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
- # # precipTotal = value*temporalRes*1.0
- # # prevLatValue = currLatvalue
- # # prevLonValue = currLonValue
-
- TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
- TRMMArea = TRMMnumOfBoxes*XRES*YRES
- rainFallacc[:] = finalCETRMMvalues
- currNetCDFData.close()
- for index, value in np.ndenumerate(finalCETRMMvalues):
- #print "index, value ", index, value
- precipTotal += value
-
- # #add info to the dictionary and to the list
- # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
- # allCEnodesTRMMdata.append(TRMMdataDict)
- # print "precipTotal is ", precipTotal
- #add info to CLOUDELEMENTSGRAPH
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
- if eachdict[1]['uniqueID'] == CEuniqueID:
- if not 'cloudElementPrecipTotal' in eachdict[1].keys():
- eachdict[1]['cloudElementPrecipTotal'] = precipTotal
- if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
- eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
- if not 'TRMMArea' in eachdict[1].keys():
- eachdict[1]['TRMMArea'] = TRMMArea
- #clean up
- precipTotal = 0.0
- latsrawTRMMData =[]
- lonsrawTRMMData = []
- latsrawCloudElements=[]
- lonsrawCloudElements=[]
- finalCETRMMvalues =[]
- CEprecipRate =[]
- brightnesstemp =[]
- TRMMdataDict ={}
-
- return allCEnodesTRMMdata
-#******************************************************************
-def findCloudClusters(CEGraph):
- '''
- Purpose:: Determines the cloud clusters properties from the subgraphs in
- the graph i.e. prunes the graph according to the minimum depth
-
- Input:: 1 graph - CEGraph: a directed graph of the CEs with weighted edges
- according the area overlap between nodes (CEs) of consectuive frames
-
- Output:: a dictionary of all possible cloudClusters
-
- '''
-
- seenNode = []
- allMCSLists =[]
- pathDictList =[]
- pathList=[]
-
- cloudClustersFile = open((MAINDIRECTORY+'/textFiles/cloudClusters.txt'),'wb')
-
- for eachNode in CEGraph:
- #check if the node has been seen before
- if eachNode not in dict(enumerate(zip(*seenNode))):
- #look for all trees associated with node as the root
- thisPathDistanceAndLength = nx.single_source_dijkstra(CEGraph, eachNode)
- #determine the actual shortestPath and minimum depth/length
- maxDepthAndMinPath = findMaxDepthAndMinPath(thisPathDistanceAndLength)
- if maxDepthAndMinPath:
- maxPathLength = maxDepthAndMinPath[0]
- shortestPath = maxDepthAndMinPath[1]
-
- #add nodes and paths to PRUNED_GRAPH
- for i in xrange(len(shortestPath)):
- if PRUNED_GRAPH.has_node(shortestPath[i]) is False:
- PRUNED_GRAPH.add_node(shortestPath[i])
-
- #add edge if necessary
- if i < (len(shortestPath)-1) and PRUNED_GRAPH.has_edge(shortestPath[i], shortestPath[i+1]) is False:
- prunedGraphEdgeweight = CEGraph.get_edge_data(shortestPath[i], shortestPath[i+1])['weight']
- PRUNED_GRAPH.add_edge(shortestPath[i], shortestPath[i+1], weight=prunedGraphEdgeweight)
-
- #note information in a file for consideration later i.e. checking to see if it works
- cloudClustersFile.write("\nSubtree pathlength is %d and path is %s" %(maxPathLength, shortestPath))
- #update seenNode info
- seenNode.append(shortestPath)
-
- print "pruned graph"
- print "number of nodes are: ", PRUNED_GRAPH.number_of_nodes()
- print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
- print ("*"*80)
-
- #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
- drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
- cloudClustersFile.close
-
- return PRUNED_GRAPH
-#******************************************************************
-def findMCC (prunedGraph):
- '''
- Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
-
- Input:: a Networkx Graph - prunedGraph
-
- Output:: finalMCCList: a list of list of tuples representing a MCC
-
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
-
- '''
- MCCList = []
- MCSList = []
- definiteMCC = []
- definiteMCS = []
- eachList =[]
- eachMCCList =[]
- maturing = False
- decaying = False
- fNode = ''
- lNode = ''
- removeList =[]
- imgCount = 0
- imgTitle =''
-
- maxShieldNode = ''
- orderedPath =[]
- treeTraversalList =[]
- definiteMCCFlag = False
- unDirGraph = nx.Graph()
- aSubGraph = nx.DiGraph()
- definiteMCSFlag = False
-
-
- #connected_components is not available for DiGraph, so generate graph as undirected
- unDirGraph = PRUNED_GRAPH.to_undirected()
- subGraph = nx.connected_component_subgraphs(unDirGraph)
-
- #for each path in the subgraphs determined
- for path in subGraph:
- #definite is a subTree provided the duration is longer than 3 hours
-
- if len(path.nodes()) > MIN_MCS_DURATION:
- orderedPath = path.nodes()
- orderedPath.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
- #definiteMCS.append(orderedPath)
-
- #build back DiGraph for checking purposes/paper purposes
- aSubGraph.add_nodes_from(path.nodes())
- for eachNode in path.nodes():
- if prunedGraph.predecessors(eachNode):
- for node in prunedGraph.predecessors(eachNode):
- aSubGraph.add_edge(node,eachNode,weight=edgeWeight[0])
-
- if prunedGraph.successors(eachNode):
- for node in prunedGraph.successors(eachNode):
- aSubGraph.add_edge(eachNode,node,weight=edgeWeight[0])
- imgTitle = 'CC'+str(imgCount+1)
- drawGraph(aSubGraph, imgTitle, edgeWeight) #for eachNode in path:
- imgCount +=1
- #----------end build back ---------------------------------------------
-
- mergeList, splitList = hasMergesOrSplits(path)
- #add node behavior regarding neutral, merge, split or both
- for node in path:
- if node in mergeList and node in splitList:
- addNodeBehaviorIdentifier(node,'B')
- elif node in mergeList and not node in splitList:
- addNodeBehaviorIdentifier(node,'M')
- elif node in splitList and not node in mergeList:
- addNodeBehaviorIdentifier(node,'S')
- else:
- addNodeBehaviorIdentifier(node,'N')
-
-
- #Do the first part of checking for the MCC feature
- #find the path
- treeTraversalList = traverseTree(aSubGraph, orderedPath[0],[],[])
- print "treeTraversalList is ", treeTraversalList
- #check the nodes to determine if a MCC on just the area criteria (consecutive nodes meeting the area and temp requirements)
- MCCList = checkedNodesMCC(prunedGraph, treeTraversalList)
- for aDict in MCCList:
- for eachNode in aDict["fullMCSMCC"]:
- addNodeMCSIdentifier(eachNode[0],eachNode[1])
-
- #do check for if MCCs overlap
- if MCCList:
- if len(MCCList) > 1:
- for count in range(len(MCCList)): #for eachDict in MCCList:
- #if there are more than two lists
- if count >= 1:
- #and the first node in this list
- eachList = list(x[0] for x in MCCList[count]["possMCCList"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
- if eachList:
- fNode = eachList[0]
- #get the lastNode in the previous possMCC list
- eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
- if eachList:
- lNode = eachList[-1]
- if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
- for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
- if aNode in eachList and aNode == lNode:
- #if edge_data is equal or less than to the exisitng edge in the tree append one to the other
- if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']:
- MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"])
- MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"])
- MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"]
- MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
- MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
- MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"]
- removeList.append(count)
- #update the MCCList
- if removeList:
- for i in removeList:
- del MCCList[i]
- removeList =[]
-
- #check if the nodes also meet the duration criteria and the shape crieria
- for eachDict in MCCList:
- #order the fullMCSMCC list, then run maximum extent and eccentricity criteria
- if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION:
- eachList = list(x[0] for x in eachDict["fullMCSMCC"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
- eachMCCList = list(x[0] for x in eachDict["possMCCList"])
- eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
-
- #update the nodemcsidentifer behavior
- #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
- #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
- #ensure that everything between is listed as 'M'
- for eachNode in eachList[:(eachList.index(eachMCCList[0]))]:
- addNodeMCSIdentifier(eachNode,'I')
-
- addNodeMCSIdentifier(eachMCCList[0],'M')
-
- for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]:
- addNodeMCSIdentifier(eachNode, 'D')
-
- #update definiteMCS list
- for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]:
- addNodeMCSIdentifier(eachNode, 'D')
-
- #run maximum extent and eccentricity criteria
- maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList)
- #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag
- if definiteMCCFlag == True:
- definiteMCC.append(eachList)
-
-
- definiteMCS.append(orderedPath)
-
- #reset for next subGraph
- aSubGraph.clear()
- orderedPath=[]
- MCCList =[]
- MCSList =[]
- definiteMCSFlag = False
-
- return definiteMCC, definiteMCS
-#******************************************************************
-def traverseTree(subGraph,node, queue, checkedNodes=None):
- '''
- Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
-
- Input:: subGraph: a Networkx DiGraph representing a CC
- lengthOfsubGraph: an integer representing the length of the subgraph
- node: a string representing the node currently being checked
- queue: a list of strings representing a list of nodes in a stack functionality
- i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
- checkedNodes: a list of strings representing the list of the nodes in the traversal
-
- Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
-
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
-
- '''
- if len(checkedNodes) == len(subGraph):
- return checkedNodes
-
- if not checkedNodes:
- queue =[]
- checkedNodes.append(node)
-
- #check one level infront first...if something does exisit, stick it at the front of the stack
- upOneLevel = subGraph.predecessors(node)
- downOneLevel = subGraph.successors(node)
- for parent in upOneLevel:
- if parent not in checkedNodes and parent not in queue:
- for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
- queue.insert(0,child)
- #downCheckFlag = True
- queue.insert(0,parent)
-
- for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
- if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
- queue.insert(0,child)
- else:
- queue.append(child)
-
- #print "^^^^^stack ", stack
- for eachNode in queue:
- if eachNode not in checkedNodes:
- #print "if eachNode ", checkedNodes, eachNode, stack
- checkedNodes.append(eachNode)
- return traverseTree(subGraph, eachNode, queue, checkedNodes)
-
- return checkedNodes
-#******************************************************************
-def checkedNodesMCC (prunedGraph, nodeList):
- '''
- Purpose :: Determine if this path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
-
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- nodeList: list of strings (CE ID) from the traversal
-
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
- dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
- '''
-
- CounterCriteriaAFlag = False
- CounterCriteriaBFlag = False
- INITIATIONFLAG = False
- MATURITYFLAG = False
- DECAYFLAG = False
- thisdict = {} #will have the same items as the cloudElementDict
- cloudElementArea = 0.0
- epsilon = 0.0
- frameNum =0
- oldNode =''
- potentialMCCList =[]
- durationAandB = 0
- CounterCriteriaA = 0
- CountercriteriaB = 0
-
- #check for if the list contains only one string/node
- if type(nodeList) is str:
- oldNode=nodeList
- nodeList =[]
- nodeList.append(oldNode)
-
- for node in nodeList:
- thisdict = thisDict(node)
- CounterCriteriaAFlag = False
- CounterCriteriaBFlag = False
- existingFrameFlag = False
-
- if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
- #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- CounterCriteriaAFlag = True
- INITIATIONFLAG = True
- MATURITYFLAG = False
- #check if criteriaB is meet
- cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
-
- #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
- if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
- #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- CounterCriteriaBFlag = True
- #append this information on to the dictionary
- addInfothisDict(node, cloudElementArea, criteriaB)
- INITIATIONFLAG = False
- MATURITYFLAG = True
- stage = 'M'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- else:
- #criteria B failed
- CounterCriteriaBFlag = False
- if INITIATIONFLAG == True:
- stage = 'I'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
-
- elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
- DECAYFLAG = True
- MATURITYFLAG = False
- stage = 'D'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- else:
- #criteria A failed
- CounterCriteriaAFlag = False
- CounterCriteriaBFlag = False
- #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- #add as a CE before or after the main feature
- if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
- stage ="I"
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
- stage = "D"
- DECAYFLAG = True
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
- stage ="I"
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
-
- return potentialMCCList
-#******************************************************************
-def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
- '''
- Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
-
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- node: a string representing the cloud element currently being assessed
- CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
- CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
-
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
- dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
-
- '''
- existingFrameFlag = False
- existingMCSFrameFlag = False
- predecessorsFlag = False
- predecessorsMCSFlag = False
- successorsFlag = False
- successorsMCSFlag = False
- frameNum = 0
-
- frameNum = int((node.split('CE')[0]).split('F')[1])
- if potentialMCCList==[]:
- #list empty
- stage = 'I'
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
- elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
-
- else:
- #list not empty
- predecessorsFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,1)
-
- if predecessorsFlag == True:
-
- for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
- if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
- existingFrameFlag = True
-
- #this MUST come after the check for the existing frame
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- stage = 'M'
- potentialMCCList[index]["possMCCList"].append((node,stage))
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
-
-
- if existingFrameFlag == False:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- stage ='M'
- potentialMCCList[index]["CounterCriteriaA"]+= 1
- potentialMCCList[index]["durationAandB"]+=1
- if frameNum > potentialMCCList[index]["frameNum"]:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
- #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list
- if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0:
- stage = 'I'
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- return potentialMCCList
- elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- return potentialMCCList
-
- #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
- if existingFrameFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- if CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
-
- if predecessorsFlag == False:
- successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
-
- if successorsFlag == True:
- for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
- if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
- existingFrameFlag = True
-
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
- stage = 'M'
- potentialMCCList[index]["possMCCList"].append((node,stage))
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
-
- if existingFrameFlag == False:
- if stage == 'M':
- stage = 'D'
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- potentialMCCList[index]["CounterCriteriaA"]+= 1
- potentialMCCList[index]["durationAandB"]+=1
- elif CounterCriteriaAFlag == True:
- potentialMCCList[index]["CounterCriteriaA"] += 1
- elif CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
- #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
- else:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- if CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
-
- #if this node isn't connected to exisiting MCCs check if it is connected to exisiting MCSs ...
- if predecessorsFlag == False and successorsFlag == False:
- stage = 'I'
- predecessorsMCSFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,2)
- if predecessorsMCSFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
- potentialMCCList[index]["possMCCList"].append((node,'M'))
- potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
- potentialMCCList[index]["durationAandB"] += 1
- if frameNum > potentialMCCList[index]["frameNum"]:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
- if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- elif CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
- else:
- successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
- if successorsMCSFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
- potentialMCCList[index]["possMCCList"].append((node,'M'))
- potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
- potentialMCCList[index]["durationAandB"] += 1
- if frameNum > potentialMCCList[index]["frameNum"]:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
-
- if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- elif CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
-
- #if this node isn't connected to existing MCCs or MCSs, create a new one ...
- if predecessorsFlag == False and predecessorsMCSFlag == False and successorsFlag == False and successorsMCSFlag == False:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
- elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
-
- return potentialMCCList
-#******************************************************************
-def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
- '''
- Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
-
- Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
- upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
- node: a string representing the cloud element currently being assessed
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
-
- Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
- index: an integer representing the location in the potentialMCCList where thisFlag occurs
-
- '''
- thisFlag = False
- index = -1
- checkList =""
- if whichList == 1:
- checkList = "possMCCList"
- elif whichList ==2:
- checkList = "fullMCSMCC"
-
- #check parents
- if upOrDown == 1:
- for aNode in prunedGraph.predecessors(node):
- #reset the index counter for this node search through potentialMCCList
- index = -1
- for MCCDict in potentialMCCList:
- index += 1
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
- thisFlag = True
- #get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
- return thisFlag, index
-
- #check children
- if upOrDown == 2:
- for aNode in prunedGraph.successors(node):
- #reset the index counter for this node search through potentialMCCList
- index = -1
- for MCCDict in potentialMCCList:
- index += 1
-
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
- thisFlag = True
- return thisFlag, index
-
- return thisFlag, index
-#******************************************************************
-def maxExtentAndEccentricity(eachList):
- '''
- Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
-
- Input:: eachList: a list of strings representing the node of the possible MCCs within a path
-
- Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
- definiteMCCFlag: a boolean indicating that the MCC has met all requirements
- '''
- maxShieldNode =''
- maxShieldArea = 0.0
- maxShieldEccentricity = 0.0
- definiteMCCFlag = False
-
- if eachList:
- for eachNode in eachList:
- if (thisDict(eachNode)['nodeMCSIdentifier'] == 'M' or thisDict(eachNode)['nodeMCSIdentifier'] == 'D') and thisDict(eachNode)['cloudElementArea'] > maxShieldArea:
- maxShieldNode = eachNode
- maxShieldArea = thisDict(eachNode)['cloudElementArea']
-
- maxShieldEccentricity = thisDict(maxShieldNode)['cloudElementEccentricity']
- if thisDict(maxShieldNode)['cloudElementEccentricity'] >= ECCENTRICITY_THRESHOLD_MIN and thisDict(maxShieldNode)['cloudElementEccentricity'] <= ECCENTRICITY_THRESHOLD_MAX :
- #criteria met
- definiteMCCFlag = True
-
- return maxShieldNode, definiteMCCFlag
-#******************************************************************
-def findMaxDepthAndMinPath (thisPathDistanceAndLength):
- '''
- Purpose:: To determine the maximum depth and min path for the headnode
-
- Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
- thisPathDistanceAndLength({distance}, {path})
- {distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
-
- Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
- minDistanceAndMaxPath = ({distance},{path})
- '''
- maxPathLength = 0
- minPath = 0
-
- #maxPathLength for the node in question
- maxPathLength = max(len (values) for values in thisPathDistanceAndLength[1].values())
-
- #if the duration is shorter then the min MCS length, then don't store!
- if maxPathLength < MIN_MCS_DURATION: #MINIMUM_DURATION :
- minDistanceAndMaxPath = ()
-
- #else find the min path and max depth
- else:
- #max path distance for the node in question
- minPath = max(values for values in thisPathDistanceAndLength[0].values())
-
- #check to determine the shortest path from the longest paths returned
- for pathDistance, path in itertools.izip(thisPathDistanceAndLength[0].values(), thisPathDistanceAndLength[1].values()):
- pathLength = len(path)
- #if pathLength is the same as the maxPathLength, then look the pathDistance to determine if the min
- if pathLength == maxPathLength :
- if pathDistance <= minPath:
- minPath = pathLength
- #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
- minDistanceAndMaxPath = (pathDistance, path)
- return minDistanceAndMaxPath
-#******************************************************************
-def thisDict (thisNode):
- '''
- Purpose:: return dictionary from graph if node exist in tree
-
- Input:: String - thisNode
-
- Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- return eachdict[1]
-#******************************************************************
-def checkCriteriaB (thisCloudElementLatLon):
- '''
- Purpose:: Determine if criteria B is met for a CEGraph
-
- Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
-
- Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
-
- '''
- cloudElementCriteriaBLatLon=[]
-
- frame, CEcounter = ndimage.measurements.label(thisCloudElementLatLon, structure=STRUCTURING_ELEMENT)
- frameCEcounter=0
- #determine min and max values in lat and lon, then use this to generate teh array from LAT,LON meshgrid
- minLat = min(x[0] for x in thisCloudElementLatLon)
- maxLat = max(x[0]for x in thisCloudElementLatLon)
- minLon = min(x[1]for x in thisCloudElementLatLon)
- maxLon = max(x[1]for x in thisCloudElementLatLon)
-
- minLatIndex = np.argmax(LAT[:,0] == minLat)
- maxLatIndex = np.argmax(LAT[:,0]== maxLat)
- minLonIndex = np.argmax(LON[0,:] == minLon)
- maxLonIndex = np.argmax(LON[0,:] == maxLon)
-
- criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
-
- for x in thisCloudElementLatLon:
- #to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
- #index given in the original array to get the indices for the new array
- #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
- criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
-
- #print criteriaBframe
- #keep only those values < TBB_MAX
- tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
-
- #get the actual values that the mask returned
- criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
-
- for index, value in maenumerate(tempMask):
- lat_index, lon_index = index
- criteriaB[lat_index, lon_index]=value
-
- for count in xrange(CEcounter):
- #[0] is time dimension. Determine the actual values from the data
- #loc is a masked array
- #***** returns elements down then across thus (6,4) is 6 arrays deep of size 4
-
- loc = ndimage.find_objects(criteriaB)[0]
- cloudElementCriteriaB =criteriaB[loc]
-
- for index,value in np.ndenumerate(cloudElementCriteriaB):
- if value !=0:
- t,lat,lon = index
- #add back on the minLatIndex and minLonIndex to find the true lat, lon values
- lat_lon_tuple = (LAT[(lat),0], LON[0,(lon)],value)
- cloudElementCriteriaBLatLon.append(lat_lon_tuple)
-
- cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
-
- return cloudElementArea, cloudElementCriteriaBLatLon
-#******************************************************************
-def hasMergesOrSplits (nodeList):
- '''
- Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
- Input:: nodeList - list of nodes from a path
- Output:: two list_vars_in_file
- splitList list of all the nodes in the path that split
- mergeList list of all the nodes in the path that merged
- '''
- mergeList=[]
- splitList=[]
-
- for node,numParents in PRUNED_GRAPH.in_degree(nodeList).items():
- if numParents > 1:
- mergeList.append(node)
-
- for node, numChildren in PRUNED_GRAPH.out_degree(nodeList).items():
- if numChildren > 1:
- splitList.append(node)
- #sort
- splitList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
- mergeList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
-
- return mergeList,splitList
-#******************************************************************
-def allAncestors(path, aNode):
- '''
- Purpose:: Utility script to provide the path leading up to a nodeList
-
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for parents
-
- Output:: list of strings - path: the list of the nodes connected to aNode through its parents
- integer - numOfChildren: the number of parents of the node passed
- '''
-
- numOfParents = PRUNED_GRAPH.in_degree(aNode)
- try:
- if PRUNED_GRAPH.predecessors(aNode) and numOfParents <= 1:
- path = path + PRUNED_GRAPH.predecessors(aNode)
- thisNode = PRUNED_GRAPH.predecessors(aNode)[0]
- return allAncestors(path,thisNode)
- else:
- path = path+aNode
- return path, numOfParents
- except:
- return path, numOfParents
-#******************************************************************
-def allDescendants(path, aNode):
- '''
- Purpose:: Utility script to provide the path leading up to a nodeList
-
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for children
-
- Output:: list of strings - path: the list of the nodes connected to aNode through its children
- integer - numOfChildren: the number of children of the node passed
- '''
-
- numOfChildren = PRUNED_GRAPH.out_degree(aNode)
- try:
- if PRUNED_GRAPH.successors(aNode) and numOfChildren <= 1:
- path = path + PRUNED_GRAPH.successors(aNode)
- thisNode = PRUNED_GRAPH.successors(aNode)[0]
- return allDescendants(path,thisNode)
- else:
- path = path + aNode
- #i.e. PRUNED_GRAPH.predecessors(aNode) is empty
- return path, numOfChildren
- except:
- #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
- return path, numOfChildren
-#******************************************************************
-def addInfothisDict (thisNode, cloudElementArea,criteriaB):
- '''
- Purpose:: update original dictionary node with information
-
- Input:: String - thisNode
- float - cloudElementArea,
- masked array of floats meeting the criteria - criteriaB
-
- Output ::
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- eachdict[1]['CriteriaBArea'] = cloudElementArea
- eachdict[1]['CriteriaBLatLon'] = criteriaB
- return
-#******************************************************************
-def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
- '''
- Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
-
- Input:: String - thisNode
- String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
-
- Output :: None
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- if not 'nodeBehaviorIdentifier' in eachdict[1].keys():
- eachdict[1]['nodeBehaviorIdentifier'] = nodeBehaviorIdentifier
- return
-#******************************************************************
-def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
- '''
- Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
-
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
-
- Output :: None
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- if not 'nodeMCSIdentifier' in eachdict[1].keys():
- eachdict[1]['nodeMCSIdentifier'] = nodeMCSIdentifier
- return
-#******************************************************************
-def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
- '''
- Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
-
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
-
- Output :: None
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- eachdict[1]['nodeMCSIdentifier'] = nodeBehaviorIdentifier
-
- return
-#******************************************************************
-def eccentricity (cloudElementLatLon):
- '''
- Purpose::
- Determines the eccentricity (shape) of contiguous boxes
- Values tending to 1 are more circular by definition, whereas
- values tending to 0 are more linear
-
- Input::
- 1 variable
- cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
-
- Output::
- 1 variable
- epsilon: a float representing the eccentricity of the matrix passed
-
- '''
-
- epsilon = 0.0
-
- #loop over all lons and determine longest (non-zero) col
- #loop over all lats and determine longest (non-zero) row
- for latLon in cloudElementLatLon:
- #assign a matrix to determine the legit values
-
- nonEmptyLons = sum(sum(cloudElementLatLon)>0)
- nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
-
- lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
- latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
- epsilon = min(latEigenvalues,lonEigenvalues)
-
- return epsilon
-#******************************************************************
-def cloudElementOverlap (currentCELatLons, previousCELatLons):
- '''
- Purpose::
- Determines the percentage overlap between two list of lat-lons passed
-
- Input::
- 2 sorted list of tuples
- currentCELatLons - the list of tuples for the current CE
- previousCELatLons - the list of tuples for the other CE being considered
-
- Output::
- 2 variables
- percentageOverlap - a float representing the number of overlapping lat_lon tuples
- areaOverlap - a floating-point number representing the area overlapping
-
- '''
-
- latlonprev =[]
- latloncurr = []
- count = 0
- percentageOverlap = 0.0
- areaOverlap = 0.0
-
- #remove the temperature from the tuples for currentCELatLons and previousCELatLons then check for overlap
- latlonprev = [(x[0],x[1]) for x in previousCELatLons]
- latloncurr = [(x[0],x[1]) for x in currentCELatLons]
-
- #find overlap
- count = len(list(set(latloncurr)&set(latlonprev)))
-
- #find area overlap
- areaOverlap = count*XRES*YRES
-
- #find percentage
- percentageOverlap = max(((count*1.0)/(len(latloncurr)*1.0)),((count*1.0)/(len(latlonprev)*1.0)))
-
- return percentageOverlap, areaOverlap
-#******************************************************************
-def findCESpeed(node, MCSList):
- '''
- Purpose:: to determine the speed of the CEs
- uses vector displacement delta_lat/delta_lon (y/x)
-
- Input:: node: a string representing the CE
- MCSList: a list of strings representing the feature
-
- Output::
-
- '''
-
- delta_lon =0.0
- delta_lat =0.0
- CEspeed =[]
- theSpeed = 0.0
-
-
- theList = CLOUD_ELEMENT_GRAPH.successors(node)
- nodeLatLon=thisDict(node)
-
-
- for aNode in theList:
- if aNode in MCSList:
- #if aNode is part of the MCSList then determine distance
- aNodeLatLon = thisDict(aNode)
- #calculate CE speed
- #checking the lats
- nodeLatLon[0] += 90.0
- aNodeLatLon[0] += 90.0
- delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
- #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
-
- #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
-
- if nodeLatLon[1] < 0.0:
- nodeLatLon[1] += 360.0
- if aNodeLatLon[1] <= 0.0:
- delta_lon = aNodeLatLon[1] + 360.0
-
- delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
- #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
-
- theSpeed = abs(((delta_lat/delta_lon)/(TRES*3600))) #convert to s --> m/s
-
- CEspeed.append(theSpeed)
- print "aNode CE speed is ", aNode, ((delta_lat/delta_lon)/(TRES*3600)), theSpeed
-
- return min(CEspeed)
-#******************************************************************
-#
-# UTILITY SCRIPTS FOR MCCSEARCH.PY
-#
-#******************************************************************
-def maenumerate(mArray):
- '''
- Purpose::
- Utility script for returning the actual values from the masked array
- Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
-
- Input::
- 1 variable
- mArray - the masked array returned from the ma.array() command
-
-
- Output::
- 1 variable
- maskedValues - 3D (t,lat,lon), value of only masked values
-
- '''
-
- mask = ~mArray.mask.ravel()
- #beware yield fast, but generates a type called "generate" that does not allow for array methods
- for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
- if maskedValue:
- yield index
-#******************************************************************
-def createMainDirectory(mainDirStr):
- '''
- Purpose:: to create the main directory for storing information and
- the subdirectories for storing information
- Input:: mainDir: a directory for where all information generated from
- the program are to be stored
- Output:: None
-
- '''
- global MAINDIRECTORY
-
- MAINDIRECTORY = mainDirStr
- #if directory doesnt exist, creat it
- if not os.path.exists(MAINDIRECTORY):
- os.makedirs(MAINDIRECTORY)
-
- os.chdir((MAINDIRECTORY))
- #create the subdirectories
- try:
- os.makedirs('images')
- os.makedirs('textFiles')
- os.makedirs('MERGnetcdfCEs')
- os.makedirs('TRMMnetcdfCEs')
- except:
- print "Directory exists already!!!"
- #TODO: some nice way of prompting if it is ok to continue...or just leave
-
- return
-#******************************************************************
-def find_nearest(thisArray,value):
- '''
- Purpose :: to determine the value within an array closes to
- another value
-
- Input ::
- Output::
- '''
- idx = (np.abs(thisArray-value)).argmin()
- return thisArray[idx]
-#******************************************************************
-def preprocessingMERG(MERGdirname):
- '''
- Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format. The files end up in a folder called mergNETCDF in the directory
- where the raw MERG data is
- NOTE: VERY RAW AND DIRTY
-
- Input::
- Directory to the location of the raw MERG files, preferably zipped
-
- Output::
- none
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
-
- os.chdir((MERGdirname+'/'))
- imgFilename = ''
-
- #Just incase the X11 server is giving problems
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- for files in glob.glob("*-pixel"):
- #for files in glob.glob("*.Z"):
- fname = os.path.splitext(files)[0]
-
- #unzip it
- bash_cmd = 'gunzip ' + files
- subprocess.call(bash_cmd, shell=True)
-
- #determine the time from the filename
- ftime = re.search('\_(.*)\_',fname).group(1)
-
- yy = ftime[0:4]
- mm = ftime[4:6]
- day = ftime[6:8]
- hr = ftime [8:10]
-
- #TODO: must be something more efficient!
-
- if mm=='01':
- mth = 'Jan'
- if mm == '02':
- mth = 'Feb'
- if mm == '03':
- mth = 'Mar'
- if mm == '04':
- mth = 'Apr'
- if mm == '05':
- mth = 'May'
- if mm == '06':
- mth = 'Jun'
- if mm == '07':
- mth = 'Jul'
- if mm == '08':
- mth = 'Aug'
- if mm == '09':
- mth = 'Sep'
- if mm == '10':
- mth = 'Oct'
- if mm == '11':
- mth = 'Nov'
- if mm == '12':
- mth = 'Dec'
-
-
- subprocess.call('rm merg.ctl', shell=True)
- subprocess.call('touch merg.ctl', shell=True)
- replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
- replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
- subprocess.call(replaceExpDset, shell=True)
- subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
- subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
- subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
- subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
- subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
- subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
- subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
-
- #generate the lats4D command for GrADS
- lats4D = 'lats4d -v -q -lat '+LATMIN + ' ' +LATMAX +' -lon ' +LONMIN +' ' +LONMAX +' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- #lats4D = 'lats4d -v -q -lat -40 -15 -lon 10 40 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
- #lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
- #run grads and lats4d command
- subprocess.call(gradscmd, shell=True)
- imgFilename = hr+'Z'+day+mth+yy+'.gif'
- tempMaskedImages(imgFilename)
-
- #when all the files have benn converted, mv the netcdf files
- subprocess.call('mkdir mergNETCDF', shell=True)
- subprocess.call('mv *.nc mergNETCDF', shell=True)
- #mv all images
- subprocess.call('mkdir mergImgs', shell=True)
- subprocess.call('mv *.gif mergImgs', shell=True)
- return
-#******************************************************************
-def postProcessingNetCDF(dataset, dirName = None):
- '''
-
- TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
-
- Purpose::
- Utility script displaying the data in generated NETCDF4 files
- in GrADS
- NOTE: VERY RAW AND DIRTY
-
- Input::
- dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
- string: Directory to the location of the raw (MERG) files, preferably zipped
-
- Output::
- images in location as specfied in the code
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- '''
-
- coreDir = os.path.dirname(MAINDIRECTORY)
- #coreDir = os.path.dirname(os.path.abspath(__file__))
- ImgFilename = ''
- frameList=[]
- fileList =[]
- lines =[]
- var =''
- firstTime = True
- printLine = 0
- lineNum = 1
- #Just incase the X11 server is giving problems
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- print "coreDir is ", str(coreDir)
- prevFrameNum = 0
-
- if dataset == 1:
- var = 'ch4'
- dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
- ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
- ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- gsFile = coreDir+"/GrADSscripts/cs2.gs"
- sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
- lineNum = 50
-
- elif dataset ==2:
- var = 'precipAcc'
- dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
- ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
- ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
- origsFile = coreDir+"/GrADSscripts/cs3.gs"
- gsFile = coreDir+"/GrADSscripts/cs4.gs"
- sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
- lineNum = 10
-
- elif dataset ==3:
- var = 'ch4'
- ctlTitle = 'TITLE MERG DATA'
- ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
- if dirName is None:
- print "Enter directory for original files"
- return
- else:
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- sologsFile = coreDir+"/GrADSscripts/infrared.gs"
- lineNum = 54
-
- #sort files
- os.chdir((dirName+'/'))
- try:
- os.makedirs('ctlFiles')
- except:
- print "ctl file folder created already"
-
- files = filter(os.path.isfile, glob.glob("*.nc"))
- files.sort(key=lambda x: os.path.getmtime(x))
- for eachfile in files:
- fullFname = os.path.splitext(eachfile)[0]
- fnameNoExtension = fullFname.split('.nc')[0]
- if dataset == 1 or dataset == 2:
- frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
- #create the ctlFile
- ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
- #the ctl file
- subprocessCall = 'rm ' +ctlFile1
- subprocess.call(subprocessCall, shell=True)
- subprocessCall = 'touch '+ctlFile1
- subprocess.call(subprocessCall, shell=True)
- lineToWrite = 'echo DSET ' + dirName+'/'+fnameNoExtension+'.nc' +' >>' + ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo DTYPE netcdf >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo UNDEF 0 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo XDEF 413 LINEAR -9.984375 0.036378335 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo YDEF 412 LINEAR 5.03515625 0.036378335 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo ZDEF 01 LEVELS 1 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo TDEF 99999 linear 31aug2009 1hr >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo VARS 1 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite ='echo '+ctlLine+' >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo ENDVARS >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
-
- #create plot of just that data
- subprocessCall = 'cp '+ origsFile+' '+sologsFile
- subprocess.call(subprocessCall, shell=True)
-
- ImgFilename = fnameNoExtension + '.gif'
-
- displayCmd = '\''+'d '+ var+'\''+'\n'
- newFileCmd = '\''+'open '+ ctlFile1+'\''+'\n'
- colorbarCmd = '\''+'run cbarn'+'\''+'\n'
- printimCmd = '\''+'
<TRUNCATED>
[10/50] [abbrv] git commit: Merge branch '0.1-incubating' of
https://github.com/kwhitehall/climate into mccsearch
Posted by le...@apache.org.
Merge branch '0.1-incubating' of https://github.com/kwhitehall/climate into mccsearch
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/0747070d
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/0747070d
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/0747070d
Branch: refs/heads/master
Commit: 0747070d51599a0071a22dea8d0b8020d6e44f72
Parents: 5e1a3d1 8158c15
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 09:05:15 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Sun Apr 13 09:05:15 2014 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[33/50] [abbrv] initial mccsearch code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/96aad4bf/mccsearch/code/mccSearchGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchGoyens.py b/mccsearch/code/mccSearchGoyens.py
new file mode 100644
index 0000000..0f49a10
--- /dev/null
+++ b/mccsearch/code/mccSearchGoyens.py
@@ -0,0 +1,3606 @@
+'''
+# All the functions for the MCC search algorithm
+# RCMES data in format (t,lat,lon), value
+# Kim Whitehall Sep 2012
+# Last updated: 23Jan 2014, 6 Dec 2013, 21 May 2013, 4 Jun 2013
+# June 13 - called this merg.py and edited out comments. merg.py is where all "cleaned" functions are to go
+# June 27 - including the first part of graphing, according to the center_of_mass
+# July 1 - going back to brute force graph approach, but weigh the edges according to closeness, up to within 20_perc of the forecasted
+#
+
+Mimumum MCS is 3 hours
+'''
+
+import datetime
+from datetime import timedelta, datetime
+import calendar
+import fileinput
+import glob
+import itertools
+import json
+import math
+import Nio
+from netCDF4 import Dataset, num2date, date2num
+import numpy as np
+import numpy.ma as ma
+import os
+import pickle
+import re
+from scipy import ndimage
+import string
+import subprocess
+import sys
+import time
+
+import networkx as nx
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+from matplotlib.dates import DateFormatter,HourLocator
+from matplotlib import cm
+import matplotlib.cm as cm
+import matplotlib.colors as mcolors
+from matplotlib.ticker import FuncFormatter, FormatStrFormatter
+
+#existing modules in services
+import files
+import process
+#----------------------- GLOBAL VARIABLES --------------------------
+# --------------------- User defined variables ---------------------
+#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
+#the first point closest the the value (for the min) from the MERG data is used, etc.
+LATMIN = '5.0' #'5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '20.0' #'20.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
+LONMIN = '0.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '30.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+XRES = 4.0 #x direction spatial resolution in km
+YRES = 4.0 #y direction spatial resolution in km
+TRES = 1 #temporal resolution in hrs
+T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
+T_BB_MIN = 218 #221 #cooler temp for the center of the system
+STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
+ #have same rank of the matrix it is being compared against
+CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
+MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
+MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
+MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
+LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
+DIRECTION = 45.0 #general direction of the wind flow in degrees
+AREA_MIN = 3500.0 #2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
+MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+
+#---Assuming using the MCC function, these will have to be changed
+ECCENTRICITY_THRESHOLD_MAX = 0.75 #tending to 1 is a circle e.g. hurricane,
+ECCENTRICITY_THRESHOLD_MIN = 0.25 #0.65 #tending to 0 is a linear e.g. squall line
+OUTER_CLOUD_SHIELD_AREA = 30000.0 #80000.0 #100000.0 #km^2
+INNER_CLOUD_SHIELD_AREA = 3000.0#30000.0 #50000.0 #km^2
+OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
+INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
+MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
+MAXIMUM_DURATION = 100 #24 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
+
+#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+#------------------- End user defined Variables -------------------
+edgeWeight = [1,2,3] #weights for the graph edges
+#graph object fo the CEs meeting the criteria
+CLOUD_ELEMENT_GRAPH = nx.DiGraph()
+#graph meeting the CC criteria
+PRUNED_GRAPH = nx.DiGraph()
+#------------------------ End GLOBAL VARS -------------------------
+#************************ Begin Functions *************************
+#******************************************************************
+def readMergData(dirname):
+ '''
+ Purpose::
+ Read MERG data into RCMES format
+
+ Input::
+ Directory to the MERG files in NETCDF format
+
+ Output::
+ A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
+ criteria for each frame
+ **remove below**
+ A dictionary of all the MERG data from the files in the directory given.
+ The dictionary contains, a string representing the time (a datetime string), a
+ tuple (lat,lon,value) representing only the data that meets the temperature requirements
+ i.e. T_BB_MAX
+
+ Assumptions::
+ The MERG data has been converted to NETCDF using LATS4D
+ The data has the same lat/lon format
+ '''
+
+ global LAT
+ global LON
+
+ # these strings are specific to the MERG data
+ mergVarName = 'ch4'
+ mergTimeVarName = 'time'
+ mergLatVarName = 'latitude'
+ mergLonVarName = 'longitude'
+
+ filelistInstructions = dirname + '/*'
+ filelist = glob.glob(filelistInstructions)
+
+
+ #sat_img is the array that will contain all the masked frames
+ mergImgs = []
+ #timelist of python time strings
+ timelist = []
+ time2store = None
+ tempMaskedValueNp =[]
+
+
+ filelist.sort()
+ nfiles = len(filelist)
+
+ # Crash nicely if there are no netcdf files
+ if nfiles == 0:
+ print 'Error: no files in this directory! Exiting elegantly'
+ sys.exit()
+ else:
+ # Open the first file in the list to read in lats, lons and generate the grid for comparison
+ tmp = Nio.open_file(filelist[0], format='nc')
+ #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
+
+ #clip the lat/lon grid according to user input
+ #http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
+ latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
+ lonsraw = tmp.variables[mergLonVarName][mergLonVarName+"|"+LONMIN+":"+LONMAX].astype('f2')
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ LON, LAT = np.meshgrid(lonsraw, latsraw)
+ #clean up
+ latsraw =[]
+ lonsraw = []
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+ #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
+ tmp.close
+
+ for files in filelist:
+ try:
+ thisFile = Nio.open_file(files, format='nc')
+ #TODO: thisFile = netCDF4.Dataset(files)
+
+ #clip the dataset according to user lat,lon coordinates
+ #mask the data and fill with zeros for later
+ tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
+ +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
+
+ tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
+
+ #get the actual values that the mask returned
+ tempMaskedValue = ma.zeros((tempRaw.shape)).astype('int16')
+ for index, value in maenumerate(tempMask):
+ time_index, lat_index, lon_index = index
+ tempMaskedValue[time_index,lat_index, lon_index]=value
+
+ timesRaw = thisFile.variables[mergTimeVarName]
+ #convert this time to a python datastring
+ time2store, _ = process.getModelTimes(files, mergTimeVarName)
+ #extend instead of append because getModelTimes returns a list already and we don't
+ #want a list of list
+ timelist.extend(time2store)
+ #to get the actual masked data only
+ #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
+ mergImgs.extend(tempMaskedValue)
+ thisFile.close
+ thisFile = None
+
+ except:
+ print "bad file! ", file
+
+ mergImgs = ma.array(mergImgs)
+
+ return mergImgs, timelist
+#******************************************************************
+def findCloudElements(mergImgs,timelist,TRMMdirName=None):
+ '''
+ Purpose::
+ Determines the contiguous boxes for a given time of the satellite imaages i.e. each frame
+ It recursively calls another function to determine if the boxes are touching
+
+ Input::
+ 2 variables
+ sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
+ maximum acceptable temperature, T_BB_MAX
+ timelist: a list of python datatimes
+ TRMMdirName (optional): string representing the path where to find the TRMM datafiles
+
+ Output::
+ 1 variable
+ cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
+ The dictionary is cloudElements={"frame": datetime object,
+ "cloudElementNum": integer,
+ "CEuniqueID": string,
+ "cloudElementLatLon": 2D array of lat_lon}
+ "cloudElementCenter": 1D array [0] is lat, [1]is lon of the center_of_mass}
+ "cloudElementEccentricity: float"
+
+ Assumptions::
+ Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
+ required according to Vila et al. (2008) is 2400km^2
+ therefore, 2400/16 = 150 contiguous squares
+ '''
+
+ frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
+ CEcounter = 0
+ frameCEcounter = 0
+ frameNum = 0
+ cloudElementEpsilon = 0.0
+ cloudElementDict = {}
+ cloudElementCenter = [] #list with two elements [lat,lon] for the center of a CE
+ prevFrameCEs = [] #list for CEs in previous frame
+ currFrameCEs = [] #list for CEs in current frame
+ cloudElementLat = [] #list for a particular CE's lat values
+ cloudElementLon = [] #list for a particular CE's lon values
+ cloudElementLatLons = [] #list for a particular CE's (lat,lon) values
+
+ prevLatValue = 0.0
+ prevLonValue = 0.0
+ TIR_min = 0.0
+ TIR_max = 0.0
+ temporalRes = 3 # TRMM data is 3 hourly
+ precipTotal = 0.0
+ CETRMMList =[]
+ precip =[]
+ TRMMCloudElementLatLons =[]
+
+ #edgeWeight = [1,2]
+ minCELatLimit = 0.0
+ minCELonLimit = 0.0
+ maxCELatLimit = 0.0
+ maxCELonLimit = 0.0
+
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ #openfile for storing ALL cloudElement information
+ cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
+
+
+ #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
+ cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
+ #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
+
+ #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
+ for t in xrange(mergImgs.shape[0]):
+ #-------------------------------------------------
+ # #textfile name for saving the data for arcgis
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ #determine contiguous locations with temeperature below the warmest temp i.e. cloudElements in each frame
+ frame, CEcounter = ndimage.measurements.label(mergImgs[t,:,:], structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ frameNum += 1
+
+ #for each of the areas identified, check to determine if it a valid CE via an area and T requirement
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ loc = ndimage.find_objects(frame==(count+1))[0]
+ cloudElement = mergImgs[t,:,:][loc]
+ labels, lcounter = ndimage.label(cloudElement)
+
+ #determine the true lats and lons for this particular CE
+ cloudElementLat = LAT[loc[0],0]
+ cloudElementLon = LON[0,loc[1]]
+
+ #determine number of boxes in this cloudelement
+ numOfBoxes = np.count_nonzero(cloudElement)
+ cloudElementArea = numOfBoxes*XRES*YRES
+
+ #If the area is greater than the area required, then start considering as CE
+ #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
+ #if it does, still consider as CE
+
+ if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
+ #do a second pass on the identified area
+ # if lcounter > 1:
+ # #i.e.there is more than one item in the area identified
+ # for lcount in xrange(lcounter):
+ # print " in lcount for", lcount
+ # loc = ndimage.find_objects(labels ==(lcount+1))[0]
+ # lcloudElement = cloudElement[:,:][loc]
+ # llabels, _ = ndimage.label(lcloudElement)
+ # plt.imshow(llabels)
+ # plt.show()
+
+
+ #get some time information and labeling info
+ frameTime = str(timelist[t])
+ frameCEcounter +=1
+ CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
+
+ #-------------------------------------------------
+ #textfile name for accesing CE data using MATLAB code
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ # ------ NETCDF File stuff for brightness temp stuff ------------------------------------
+ thisFileName = MAINDIRECTORY +'/MERGnetcdfCEs/cloudElements'+ (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFCEData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFCEData.description = 'Cloud Element '+CEuniqueID + ' temperature data'
+ currNetCDFCEData.calendar = 'standard'
+ currNetCDFCEData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFCEData.createDimension('time', None)
+ currNetCDFCEData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFCEData.createDimension('lon', len(LON[0,:]))
+ # variables
+ tempDims = ('time','lat', 'lon',)
+ times = currNetCDFCEData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitudes = currNetCDFCEData.createVariable('latitude', 'f8', ('lat',))
+ longitudes = currNetCDFCEData.createVariable('longitude', 'f8', ('lon',))
+ brightnesstemp = currNetCDFCEData.createVariable('brightnesstemp', 'i16',tempDims )
+ brightnesstemp.units = 'Kelvin'
+ # NETCDF data
+ dates=[timelist[t]+timedelta(hours=0)]
+ times[:] = date2num(dates,units=times.units)
+ longitudes[:] = LON[0,:]
+ longitudes.units = "degrees_east"
+ longitudes.long_name = "Longitude"
+
+ latitudes[:] = LAT[:,0]
+ latitudes.units = "degrees_north"
+ latitudes.long_name ="Latitude"
+
+ #generate array of zeros for brightness temperature
+ brightnesstemp1 = ma.zeros((1,len(latitudes), len(longitudes))).astype('int16')
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #if other dataset (TRMM) assumed to be a precipitation dataset was entered
+ if TRMMdirName:
+ #------------------TRMM stuff -------------------------------------------------
+ fileDate = ((str(timelist[t])).replace(" ", "")[:-8]).replace("-","")
+ fileHr1 = (str(timelist[t])).replace(" ", "")[-8:-6]
+
+ if int(fileHr1) % temporalRes == 0:
+ fileHr = fileHr1
+ else:
+ fileHr = (int(fileHr1)/temporalRes) * temporalRes
+ if fileHr < 10:
+ fileHr = '0'+str(fileHr)
+ else:
+ str(fileHr)
+
+ #open TRMM file for the resolution info and to create the appropriate sized grid
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+str(fileHr)+".7A.nc"
+
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ nygrdTRMM = len(LATTRMM[:,0]); nxgrdTRMM = len(LONTRMM[0,:])
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ #get the lat/lon info from the file
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+ # ------ NETCDF File info for writing TRMM CE rainfall ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' precipitation data'
+ currNetCDFTRMMData.calendar = 'standard'
+ currNetCDFTRMMData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFTRMMData.createDimension('time', None)
+ currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
+
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value)
+
+ #generate the comma separated file for GIS
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ #temp data for CE NETCDF file
+ brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value
+
+ if TRMMdirName:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])]
+ CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]]))
+
+
+ brightnesstemp[:] = brightnesstemp1[:]
+ currNetCDFCEData.close()
+
+ if TRMMdirName:
+
+ #print finalCETRMMvalues
+ #calculate the total precip associated with the feature
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ precipTotal += value
+ precip.append(value)
+
+ rainFallacc[:] = finalCETRMMvalues[:]
+ currNetCDFTRMMData.close()
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ try:
+ maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ pass
+
+ #sort cloudElementLatLons by lats
+ cloudElementLatLons.sort(key=lambda tup: tup[0])
+
+ #determine if the cloud element the shape
+ cloudElementEpsilon = eccentricity (cloudElement)
+ cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementCenter.append(latCenter)
+ cloudElementCenter.append(lonCenter)
+ cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsUserFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ #populate the dictionary
+ if TRMMdirName:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min, 'cloudElementPrecipTotal':precipTotal,'cloudElementLatLonTRMM':CETRMMList, 'TRMMArea': TRMMArea}
+ else:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min,}
+
+ #current frame list of CEs
+ currFrameCEs.append(cloudElementDict)
+
+ #draw the graph node
+ CLOUD_ELEMENT_GRAPH.add_node(CEuniqueID, cloudElementDict)
+
+ if frameNum != 1:
+ for cloudElementDict in prevFrameCEs:
+ thisCElen = len(cloudElementLatLons)
+ percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
+
+ #change weights to integers because the built in shortest path chokes on floating pts
+ #according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
+ if percentageOverlap >= 0.95:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
+
+ elif percentageOverlap >= 0.90 and percentageOverlap < 0.95 :
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[1])
+
+ elif areaOverlap >= MIN_OVERLAP:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
+
+ else:
+ #TODO: remove this else as we only wish for the CE details
+ #ensure only the non-zero elements are considered
+ #store intel in allCE file
+ labels, _ = ndimage.label(cloudElement)
+ cloudElementsFile.write("\n-----------------------------------------------")
+ cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ # cloudElementLat = LAT[loc[0],0]
+ # cloudElementLon = LON[0,loc[1]]
+
+ #populate cloudElementLatLons by unpacking the original values from loc
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ cloudElementsFile.write("\n-----------------------------------------------")
+
+ #reset list for the next CE
+ nodeExist = False
+ cloudElementCenter=[]
+ cloudElement = []
+ cloudElementCenter = []
+ cloudElementLat=[]
+ cloudElementLon =[]
+ cloudElementLatLons =[]
+ brightnesstemp1 =[]
+ brightnesstemp =[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ CETRMMList =[]
+ precipTotal = 0.0
+ precip=[]
+ TRMMCloudElementLatLons=[]
+
+ #reset for the next time
+ prevFrameCEs =[]
+ prevFrameCEs = currFrameCEs
+ currFrameCEs =[]
+
+ cloudElementsFile.close
+ cloudElementsUserFile.close
+ #if using ARCGIS data store code, uncomment this file close line
+ #cloudElementsTextFile.close
+
+ #clean up graph - remove parent and childless nodes
+ outAndInDeg = CLOUD_ELEMENT_GRAPH.degree_iter()
+ toRemove = [node[0] for node in outAndInDeg if node[1]<1]
+ CLOUD_ELEMENT_GRAPH.remove_nodes_from(toRemove)
+
+ print "number of nodes are: ", CLOUD_ELEMENT_GRAPH.number_of_nodes()
+ print "number of edges are: ", CLOUD_ELEMENT_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #hierachial graph output
+ #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
+ drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
+
+ return CLOUD_ELEMENT_GRAPH
+#******************************************************************
+def findPrecipRate(TRMMdirName):
+ '''
+ TODO: needs fixing
+ Purpose:: Determines the precipitation rates for MCSs found
+
+ Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
+
+ Output:: a list of dictionary of the TRMM data
+ NB: also creates netCDF with TRMM data for each CE (for post processing) index
+ in MAINDIRECTORY/TRMMnetcdfCEs
+
+ Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
+
+ '''
+ allCEnodesTRMMdata =[]
+ TRMMdataDict={}
+ precipTotal = 0.0
+
+ os.chdir((MAINDIRECTORY+'/MERGnetcdfCEs/'))
+ imgFilename = ''
+ temporalRes = 3 #3 hours for TRMM
+
+ #sort files
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+
+ for afile in files:
+ #try:
+ print "in for ", afile
+ fullFname = os.path.splitext(afile)[0]
+ noFrameExtension = (fullFname.replace("_","")).split('F')[0]
+ CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
+ fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
+ fileDateTime = fileDateTimeChar.replace("-","")
+ fileDate = fileDateTime[:-6]
+ fileHr=fileDateTime[-6:-4]
+
+ cloudElementData = Dataset(afile,'r', format='NETCDF4')
+ brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
+ latsrawCloudElements = cloudElementData.variables['latitude'][:]
+ lonsrawCloudElements = cloudElementData.variables['longitude'][:]
+
+ # print "noFrameExtension", noFrameExtension
+ # print "fileDateTime ", fileDateTime
+ # print "fileDateTimeChar ", fileDateTimeChar
+ # print "fullFname ", fullFname
+ # print "time is: ", (fullFname.replace("_"," ").split('F')[0]).split('s')[1]
+ # print "fileDate and fileHr ", fileDate, fileHr
+ # print "CEuniqueID ", CEuniqueID
+ # sys.exit()
+ if int(fileHr) % 3 == 0:
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
+ #print "TRMMfileName is ", TRMMfileName
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+
+
+ # ------ NETCDF File stuff ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
+ # print "thisFileName ", thisFileName
+ # sys.exit()
+ currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
+ currNetCDFData.calendar = 'standard'
+ currNetCDFData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFData.createDimension('time', None)
+ currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
+ currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+ #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
+ for index,value in np.ndenumerate(brightnesstemp):
+ #print "index, value ", index, value
+ time_index, lat_index, lon_index = index
+ currTimeValue = 0
+ if value > 0:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
+
+
+ # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
+ # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
+ # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
+ # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
+ # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
+ # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
+ # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
+ # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
+ # # precipTotal = value*temporalRes*1.0
+ # # prevLatValue = currLatvalue
+ # # prevLonValue = currLonValue
+
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ rainFallacc[:] = finalCETRMMvalues
+ currNetCDFData.close()
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ #print "index, value ", index, value
+ precipTotal += value
+
+ # #add info to the dictionary and to the list
+ # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
+ # allCEnodesTRMMdata.append(TRMMdataDict)
+ # print "precipTotal is ", precipTotal
+ #add info to CLOUDELEMENTSGRAPH
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
+ if eachdict[1]['uniqueID'] == CEuniqueID:
+ if not 'cloudElementPrecipTotal' in eachdict[1].keys():
+ eachdict[1]['cloudElementPrecipTotal'] = precipTotal
+ if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
+ eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
+ if not 'TRMMArea' in eachdict[1].keys():
+ eachdict[1]['TRMMArea'] = TRMMArea
+ #clean up
+ precipTotal = 0.0
+ latsrawTRMMData =[]
+ lonsrawTRMMData = []
+ latsrawCloudElements=[]
+ lonsrawCloudElements=[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ brightnesstemp =[]
+ TRMMdataDict ={}
+
+ return allCEnodesTRMMdata
+#******************************************************************
+def findCloudClusters(CEGraph):
+ '''
+ Purpose:: Determines the cloud clusters properties from the subgraphs in
+ the graph i.e. prunes the graph according to the minimum depth
+
+ Input:: 1 graph - CEGraph: a directed graph of the CEs with weighted edges
+ according the area overlap between nodes (CEs) of consectuive frames
+
+ Output:: a dictionary of all possible cloudClusters
+
+ '''
+
+ seenNode = []
+ allMCSLists =[]
+ pathDictList =[]
+ pathList=[]
+
+ cloudClustersFile = open((MAINDIRECTORY+'/textFiles/cloudClusters.txt'),'wb')
+
+ for eachNode in CEGraph:
+ #check if the node has been seen before
+ if eachNode not in dict(enumerate(zip(*seenNode))):
+ #look for all trees associated with node as the root
+ thisPathDistanceAndLength = nx.single_source_dijkstra(CEGraph, eachNode)
+ #determine the actual shortestPath and minimum depth/length
+ maxDepthAndMinPath = findMaxDepthAndMinPath(thisPathDistanceAndLength)
+ if maxDepthAndMinPath:
+ maxPathLength = maxDepthAndMinPath[0]
+ shortestPath = maxDepthAndMinPath[1]
+
+ #add nodes and paths to PRUNED_GRAPH
+ for i in xrange(len(shortestPath)):
+ if PRUNED_GRAPH.has_node(shortestPath[i]) is False:
+ PRUNED_GRAPH.add_node(shortestPath[i])
+
+ #add edge if necessary
+ if i < (len(shortestPath)-1) and PRUNED_GRAPH.has_edge(shortestPath[i], shortestPath[i+1]) is False:
+ prunedGraphEdgeweight = CEGraph.get_edge_data(shortestPath[i], shortestPath[i+1])['weight']
+ PRUNED_GRAPH.add_edge(shortestPath[i], shortestPath[i+1], weight=prunedGraphEdgeweight)
+
+ #note information in a file for consideration later i.e. checking to see if it works
+ cloudClustersFile.write("\nSubtree pathlength is %d and path is %s" %(maxPathLength, shortestPath))
+ #update seenNode info
+ seenNode.append(shortestPath)
+
+ print "pruned graph"
+ print "number of nodes are: ", PRUNED_GRAPH.number_of_nodes()
+ print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
+ drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
+ cloudClustersFile.close
+
+ return PRUNED_GRAPH
+#******************************************************************
+def findMCC (prunedGraph):
+ '''
+ Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
+
+ Input:: a Networkx Graph - prunedGraph
+
+ Output:: finalMCCList: a list of list of tuples representing a MCC
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ MCCList = []
+ MCSList = []
+ definiteMCC = []
+ definiteMCS = []
+ eachList =[]
+ eachMCCList =[]
+ maturing = False
+ decaying = False
+ fNode = ''
+ lNode = ''
+ removeList =[]
+ imgCount = 0
+ imgTitle =''
+
+ maxShieldNode = ''
+ orderedPath =[]
+ treeTraversalList =[]
+ definiteMCCFlag = False
+ unDirGraph = nx.Graph()
+ aSubGraph = nx.DiGraph()
+ definiteMCSFlag = False
+
+
+ #connected_components is not available for DiGraph, so generate graph as undirected
+ unDirGraph = PRUNED_GRAPH.to_undirected()
+ subGraph = nx.connected_component_subgraphs(unDirGraph)
+
+ #for each path in the subgraphs determined
+ for path in subGraph:
+ #definite is a subTree provided the duration is longer than 3 hours
+
+ if len(path.nodes()) > MIN_MCS_DURATION:
+ orderedPath = path.nodes()
+ orderedPath.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ #definiteMCS.append(orderedPath)
+
+ #build back DiGraph for checking purposes/paper purposes
+ aSubGraph.add_nodes_from(path.nodes())
+ for eachNode in path.nodes():
+ if prunedGraph.predecessors(eachNode):
+ for node in prunedGraph.predecessors(eachNode):
+ aSubGraph.add_edge(node,eachNode,weight=edgeWeight[0])
+
+ if prunedGraph.successors(eachNode):
+ for node in prunedGraph.successors(eachNode):
+ aSubGraph.add_edge(eachNode,node,weight=edgeWeight[0])
+ imgTitle = 'CC'+str(imgCount+1)
+ drawGraph(aSubGraph, imgTitle, edgeWeight) #for eachNode in path:
+ imgCount +=1
+ #----------end build back ---------------------------------------------
+
+ mergeList, splitList = hasMergesOrSplits(path)
+ #add node behavior regarding neutral, merge, split or both
+ for node in path:
+ if node in mergeList and node in splitList:
+ addNodeBehaviorIdentifier(node,'B')
+ elif node in mergeList and not node in splitList:
+ addNodeBehaviorIdentifier(node,'M')
+ elif node in splitList and not node in mergeList:
+ addNodeBehaviorIdentifier(node,'S')
+ else:
+ addNodeBehaviorIdentifier(node,'N')
+
+
+ #Do the first part of checking for the MCC feature
+ #find the path
+ treeTraversalList = traverseTree(aSubGraph, orderedPath[0],[],[])
+ print "treeTraversalList is ", treeTraversalList
+ #check the nodes to determine if a MCC on just the area criteria (consecutive nodes meeting the area and temp requirements)
+ MCCList = checkedNodesMCC(prunedGraph, treeTraversalList)
+ for aDict in MCCList:
+ for eachNode in aDict["fullMCSMCC"]:
+ addNodeMCSIdentifier(eachNode[0],eachNode[1])
+
+ #do check for if MCCs overlap
+ if MCCList:
+ if len(MCCList) > 1:
+ for count in range(len(MCCList)): #for eachDict in MCCList:
+ #if there are more than two lists
+ if count >= 1:
+ #and the first node in this list
+ eachList = list(x[0] for x in MCCList[count]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ fNode = eachList[0]
+ #get the lastNode in the previous possMCC list
+ eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ lNode = eachList[-1]
+ if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ if aNode in eachList and aNode == lNode:
+ #if edge_data is equal or less than to the exisitng edge in the tree append one to the other
+ if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']:
+ MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"])
+ MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"])
+ MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"]
+ MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
+ MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
+ MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"]
+ removeList.append(count)
+ #update the MCCList
+ if removeList:
+ for i in removeList:
+ del MCCList[i]
+ removeList =[]
+
+ #check if the nodes also meet the duration criteria and the shape crieria
+ for eachDict in MCCList:
+ #order the fullMCSMCC list, then run maximum extent and eccentricity criteria
+ if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION:
+ eachList = list(x[0] for x in eachDict["fullMCSMCC"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ eachMCCList = list(x[0] for x in eachDict["possMCCList"])
+ eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+
+ #update the nodemcsidentifer behavior
+ #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
+ #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
+ #ensure that everything between is listed as 'M'
+ for eachNode in eachList[:(eachList.index(eachMCCList[0]))]:
+ addNodeMCSIdentifier(eachNode,'I')
+
+ addNodeMCSIdentifier(eachMCCList[0],'M')
+
+ for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #update definiteMCS list
+ for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #run maximum extent and eccentricity criteria
+ maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList)
+ #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag
+ if definiteMCCFlag == True:
+ definiteMCC.append(eachList)
+
+
+ definiteMCS.append(orderedPath)
+
+ #reset for next subGraph
+ aSubGraph.clear()
+ orderedPath=[]
+ MCCList =[]
+ MCSList =[]
+ definiteMCSFlag = False
+
+ return definiteMCC, definiteMCS
+#******************************************************************
+def traverseTree(subGraph,node, queue, checkedNodes=None):
+ '''
+ Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+
+ Input:: subGraph: a Networkx DiGraph representing a CC
+ lengthOfsubGraph: an integer representing the length of the subgraph
+ node: a string representing the node currently being checked
+ queue: a list of strings representing a list of nodes in a stack functionality
+ i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ if len(checkedNodes) == len(subGraph):
+ return checkedNodes
+
+ if not checkedNodes:
+ queue =[]
+ checkedNodes.append(node)
+
+ #check one level infront first...if something does exisit, stick it at the front of the stack
+ upOneLevel = subGraph.predecessors(node)
+ downOneLevel = subGraph.successors(node)
+ for parent in upOneLevel:
+ if parent not in checkedNodes and parent not in queue:
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ queue.insert(0,child)
+ #downCheckFlag = True
+ queue.insert(0,parent)
+
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
+ queue.insert(0,child)
+ else:
+ queue.append(child)
+
+ #print "^^^^^stack ", stack
+ for eachNode in queue:
+ if eachNode not in checkedNodes:
+ #print "if eachNode ", checkedNodes, eachNode, stack
+ checkedNodes.append(eachNode)
+ return traverseTree(subGraph, eachNode, queue, checkedNodes)
+
+ return checkedNodes
+#******************************************************************
+def checkedNodesMCC (prunedGraph, nodeList):
+ '''
+ Purpose :: Determine if this path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ nodeList: list of strings (CE ID) from the traversal
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+ '''
+
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ INITIATIONFLAG = False
+ MATURITYFLAG = False
+ DECAYFLAG = False
+ thisdict = {} #will have the same items as the cloudElementDict
+ cloudElementArea = 0.0
+ epsilon = 0.0
+ frameNum =0
+ oldNode =''
+ potentialMCCList =[]
+ durationAandB = 0
+ CounterCriteriaA = 0
+ CountercriteriaB = 0
+
+ #check for if the list contains only one string/node
+ if type(nodeList) is str:
+ oldNode=nodeList
+ nodeList =[]
+ nodeList.append(oldNode)
+
+ for node in nodeList:
+ thisdict = thisDict(node)
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ existingFrameFlag = False
+
+ if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
+ #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaAFlag = True
+ INITIATIONFLAG = True
+ MATURITYFLAG = False
+ #check if criteriaB is meet
+ cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
+
+ #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
+ if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
+ #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaBFlag = True
+ #append this information on to the dictionary
+ addInfothisDict(node, cloudElementArea, criteriaB)
+ INITIATIONFLAG = False
+ MATURITYFLAG = True
+ stage = 'M'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria B failed
+ CounterCriteriaBFlag = False
+ if INITIATIONFLAG == True:
+ stage = 'I'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
+ DECAYFLAG = True
+ MATURITYFLAG = False
+ stage = 'D'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria A failed
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ #add as a CE before or after the main feature
+ if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
+ stage = "D"
+ DECAYFLAG = True
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ return potentialMCCList
+#******************************************************************
+def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
+ '''
+ Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ node: a string representing the cloud element currently being assessed
+ CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
+ CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+
+ '''
+ existingFrameFlag = False
+ existingMCSFrameFlag = False
+ predecessorsFlag = False
+ predecessorsMCSFlag = False
+ successorsFlag = False
+ successorsMCSFlag = False
+ frameNum = 0
+
+ frameNum = int((node.split('CE')[0]).split('F')[1])
+ if potentialMCCList==[]:
+ #list empty
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ else:
+ #list not empty
+ predecessorsFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,1)
+
+ if predecessorsFlag == True:
+
+ for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
+ existingFrameFlag = True
+
+ #this MUST come after the check for the existing frame
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+
+
+ if existingFrameFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage ='M'
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0:
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ if existingFrameFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ if predecessorsFlag == False:
+ successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+
+ if successorsFlag == True:
+ for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
+ existingFrameFlag = True
+
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if existingFrameFlag == False:
+ if stage == 'M':
+ stage = 'D'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ elif CounterCriteriaAFlag == True:
+ potentialMCCList[index]["CounterCriteriaA"] += 1
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ else:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to exisiting MCCs check if it is connected to exisiting MCSs ...
+ if predecessorsFlag == False and successorsFlag == False:
+ stage = 'I'
+ predecessorsMCSFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,2)
+ if predecessorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ else:
+ successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+ if successorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to existing MCCs or MCSs, create a new one ...
+ if predecessorsFlag == False and predecessorsMCSFlag == False and successorsFlag == False and successorsMCSFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ return potentialMCCList
+#******************************************************************
+def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
+ '''
+ Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
+
+ Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
+ upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
+ node: a string representing the cloud element currently being assessed
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
+
+ Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
+ index: an integer representing the location in the potentialMCCList where thisFlag occurs
+
+ '''
+ thisFlag = False
+ index = -1
+ checkList =""
+ if whichList == 1:
+ checkList = "possMCCList"
+ elif whichList ==2:
+ checkList = "fullMCSMCC"
+
+ #check parents
+ if upOrDown == 1:
+ for aNode in prunedGraph.predecessors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ #get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
+ return thisFlag, index
+
+ #check children
+ if upOrDown == 2:
+ for aNode in prunedGraph.successors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ return thisFlag, index
+
+ return thisFlag, index
+#******************************************************************
+def maxExtentAndEccentricity(eachList):
+ '''
+ Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
+
+ Input:: eachList: a list of strings representing the node of the possible MCCs within a path
+
+ Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
+ '''
+ maxShieldNode =''
+ maxShieldArea = 0.0
+ maxShieldEccentricity = 0.0
+ definiteMCCFlag = False
+
+ if eachList:
+ for eachNode in eachList:
+ if (thisDict(eachNode)['nodeMCSIdentifier'] == 'M' or thisDict(eachNode)['nodeMCSIdentifier'] == 'D') and thisDict(eachNode)['cloudElementArea'] > maxShieldArea:
+ maxShieldNode = eachNode
+ maxShieldArea = thisDict(eachNode)['cloudElementArea']
+
+ maxShieldEccentricity = thisDict(maxShieldNode)['cloudElementEccentricity']
+ if thisDict(maxShieldNode)['cloudElementEccentricity'] >= ECCENTRICITY_THRESHOLD_MIN and thisDict(maxShieldNode)['cloudElementEccentricity'] <= ECCENTRICITY_THRESHOLD_MAX :
+ #criteria met
+ definiteMCCFlag = True
+
+ return maxShieldNode, definiteMCCFlag
+#******************************************************************
+def findMaxDepthAndMinPath (thisPathDistanceAndLength):
+ '''
+ Purpose:: To determine the maximum depth and min path for the headnode
+
+ Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
+ thisPathDistanceAndLength({distance}, {path})
+ {distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
+
+ Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
+ minDistanceAndMaxPath = ({distance},{path})
+ '''
+ maxPathLength = 0
+ minPath = 0
+
+ #maxPathLength for the node in question
+ maxPathLength = max(len (values) for values in thisPathDistanceAndLength[1].values())
+
+ #if the duration is shorter then the min MCS length, then don't store!
+ if maxPathLength < MIN_MCS_DURATION: #MINIMUM_DURATION :
+ minDistanceAndMaxPath = ()
+
+ #else find the min path and max depth
+ else:
+ #max path distance for the node in question
+ minPath = max(values for values in thisPathDistanceAndLength[0].values())
+
+ #check to determine the shortest path from the longest paths returned
+ for pathDistance, path in itertools.izip(thisPathDistanceAndLength[0].values(), thisPathDistanceAndLength[1].values()):
+ pathLength = len(path)
+ #if pathLength is the same as the maxPathLength, then look the pathDistance to determine if the min
+ if pathLength == maxPathLength :
+ if pathDistance <= minPath:
+ minPath = pathLength
+ #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
+ minDistanceAndMaxPath = (pathDistance, path)
+ return minDistanceAndMaxPath
+#******************************************************************
+def thisDict (thisNode):
+ '''
+ Purpose:: return dictionary from graph if node exist in tree
+
+ Input:: String - thisNode
+
+ Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ return eachdict[1]
+#******************************************************************
+def checkCriteriaB (thisCloudElementLatLon):
+ '''
+ Purpose:: Determine if criteria B is met for a CEGraph
+
+ Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
+
+ Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
+
+ '''
+ cloudElementCriteriaBLatLon=[]
+
+ frame, CEcounter = ndimage.measurements.label(thisCloudElementLatLon, structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ #determine min and max values in lat and lon, then use this to generate teh array from LAT,LON meshgrid
+ minLat = min(x[0] for x in thisCloudElementLatLon)
+ maxLat = max(x[0]for x in thisCloudElementLatLon)
+ minLon = min(x[1]for x in thisCloudElementLatLon)
+ maxLon = max(x[1]for x in thisCloudElementLatLon)
+
+ minLatIndex = np.argmax(LAT[:,0] == minLat)
+ maxLatIndex = np.argmax(LAT[:,0]== maxLat)
+ minLonIndex = np.argmax(LON[0,:] == minLon)
+ maxLonIndex = np.argmax(LON[0,:] == maxLon)
+
+ criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
+
+ for x in thisCloudElementLatLon:
+ #to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
+ #index given in the original array to get the indices for the new array
+ #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
+ criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
+
+ #print criteriaBframe
+ #keep only those values < TBB_MAX
+ tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
+
+ #get the actual values that the mask returned
+ criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
+
+ for index, value in maenumerate(tempMask):
+ lat_index, lon_index = index
+ criteriaB[lat_index, lon_index]=value
+
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ #***** returns elements down then across thus (6,4) is 6 arrays deep of size 4
+
+ loc = ndimage.find_objects(criteriaB)[0]
+ cloudElementCriteriaB =criteriaB[loc]
+
+ for index,value in np.ndenumerate(cloudElementCriteriaB):
+ if value !=0:
+ t,lat,lon = index
+ #add back on the minLatIndex and minLonIndex to find the true lat, lon values
+ lat_lon_tuple = (LAT[(lat),0], LON[0,(lon)],value)
+ cloudElementCriteriaBLatLon.append(lat_lon_tuple)
+
+ cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
+
+ return cloudElementArea, cloudElementCriteriaBLatLon
+#******************************************************************
+def hasMergesOrSplits (nodeList):
+ '''
+ Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
+ Input:: nodeList - list of nodes from a path
+ Output:: two list_vars_in_file
+ splitList list of all the nodes in the path that split
+ mergeList list of all the nodes in the path that merged
+ '''
+ mergeList=[]
+ splitList=[]
+
+ for node,numParents in PRUNED_GRAPH.in_degree(nodeList).items():
+ if numParents > 1:
+ mergeList.append(node)
+
+ for node, numChildren in PRUNED_GRAPH.out_degree(nodeList).items():
+ if numChildren > 1:
+ splitList.append(node)
+ #sort
+ splitList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ mergeList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+
+ return mergeList,splitList
+#******************************************************************
+def allAncestors(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for parents
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its parents
+ integer - numOfChildren: the number of parents of the node passed
+ '''
+
+ numOfParents = PRUNED_GRAPH.in_degree(aNode)
+ try:
+ if PRUNED_GRAPH.predecessors(aNode) and numOfParents <= 1:
+ path = path + PRUNED_GRAPH.predecessors(aNode)
+ thisNode = PRUNED_GRAPH.predecessors(aNode)[0]
+ return allAncestors(path,thisNode)
+ else:
+ path = path+aNode
+ return path, numOfParents
+ except:
+ return path, numOfParents
+#******************************************************************
+def allDescendants(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for children
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its children
+ integer - numOfChildren: the number of children of the node passed
+ '''
+
+ numOfChildren = PRUNED_GRAPH.out_degree(aNode)
+ try:
+ if PRUNED_GRAPH.successors(aNode) and numOfChildren <= 1:
+ path = path + PRUNED_GRAPH.successors(aNode)
+ thisNode = PRUNED_GRAPH.successors(aNode)[0]
+ return allDescendants(path,thisNode)
+ else:
+ path = path + aNode
+ #i.e. PRUNED_GRAPH.predecessors(aNode) is empty
+ return path, numOfChildren
+ except:
+ #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
+ return path, numOfChildren
+#******************************************************************
+def addInfothisDict (thisNode, cloudElementArea,criteriaB):
+ '''
+ Purpose:: update original dictionary node with information
+
+ Input:: String - thisNode
+ float - cloudElementArea,
+ masked array of floats meeting the criteria - criteriaB
+
+ Output ::
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['CriteriaBArea'] = cloudElementArea
+ eachdict[1]['CriteriaBLatLon'] = criteriaB
+ return
+#******************************************************************
+def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeBehaviorIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeBehaviorIdentifier'] = nodeBehaviorIdentifier
+ return
+#******************************************************************
+def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeMCSIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeMCSIdentifier'] = nodeMCSIdentifier
+ return
+#******************************************************************
+def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['nodeMCSIdentifier'] = nodeBehaviorIdentifier
+
+ return
+#******************************************************************
+def eccentricity (cloudElementLatLon):
+ '''
+ Purpose::
+ Determines the eccentricity (shape) of contiguous boxes
+ Values tending to 1 are more circular by definition, whereas
+ values tending to 0 are more linear
+
+ Input::
+ 1 variable
+ cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
+
+ Output::
+ 1 variable
+ epsilon: a float representing the eccentricity of the matrix passed
+
+ '''
+
+ epsilon = 0.0
+
+ #loop over all lons and determine longest (non-zero) col
+ #loop over all lats and determine longest (non-zero) row
+ for latLon in cloudElementLatLon:
+ #assign a matrix to determine the legit values
+
+ nonEmptyLons = sum(sum(cloudElementLatLon)>0)
+ nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
+
+ lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
+ latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
+ epsilon = min(latEigenvalues,lonEigenvalues)
+
+ return epsilon
+#******************************************************************
+def cloudElementOverlap (currentCELatLons, previousCELatLons):
+ '''
+ Purpose::
+ Determines the percentage overlap between two list of lat-lons passed
+
+ Input::
+ 2 sorted list of tuples
+ currentCELatLons - the list of tuples for the current CE
+ previousCELatLons - the list of tuples for the other CE being considered
+
+ Output::
+ 2 variables
+ percentageOverlap - a float representing the number of overlapping lat_lon tuples
+ areaOverlap - a floating-point number representing the area overlapping
+
+ '''
+
+ latlonprev =[]
+ latloncurr = []
+ count = 0
+ percentageOverlap = 0.0
+ areaOverlap = 0.0
+
+ #remove the temperature from the tuples for currentCELatLons and previousCELatLons then check for overlap
+ latlonprev = [(x[0],x[1]) for x in previousCELatLons]
+ latloncurr = [(x[0],x[1]) for x in currentCELatLons]
+
+ #find overlap
+ count = len(list(set(latloncurr)&set(latlonprev)))
+
+ #find area overlap
+ areaOverlap = count*XRES*YRES
+
+ #find percentage
+ percentageOverlap = max(((count*1.0)/(len(latloncurr)*1.0)),((count*1.0)/(len(latlonprev)*1.0)))
+
+ return percentageOverlap, areaOverlap
+#******************************************************************
+def findCESpeed(node, MCSList):
+ '''
+ Purpose:: to determine the speed of the CEs
+ uses vector displacement delta_lat/delta_lon (y/x)
+
+ Input:: node: a string representing the CE
+ MCSList: a list of strings representing the feature
+
+ Output::
+
+ '''
+
+ delta_lon =0.0
+ delta_lat =0.0
+ CEspeed =[]
+ theSpeed = 0.0
+
+
+ theList = CLOUD_ELEMENT_GRAPH.successors(node)
+ nodeLatLon=thisDict(node)
+
+
+ for aNode in theList:
+ if aNode in MCSList:
+ #if aNode is part of the MCSList then determine distance
+ aNodeLatLon = thisDict(aNode)
+ #calculate CE speed
+ #checking the lats
+ nodeLatLon[0] += 90.0
+ aNodeLatLon[0] += 90.0
+ delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
+ #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
+
+ #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ if nodeLatLon[1] < 0.0:
+ nodeLatLon[1] += 360.0
+ if aNodeLatLon[1] <= 0.0:
+ delta_lon = aNodeLatLon[1] + 360.0
+
+ delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
+ #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
+
+ theSpeed = abs(((delta_lat/delta_lon)/(TRES*3600))) #convert to s --> m/s
+
+ CEspeed.append(theSpeed)
+ print "aNode CE speed is ", aNode, ((delta_lat/delta_lon)/(TRES*3600)), theSpeed
+
+ return min(CEspeed)
+#******************************************************************
+#
+# UTILITY SCRIPTS FOR MCCSEARCH.PY
+#
+#******************************************************************
+def maenumerate(mArray):
+ '''
+ Purpose::
+ Utility script for returning the actual values from the masked array
+ Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
+
+ Input::
+ 1 variable
+ mArray - the masked array returned from the ma.array() command
+
+
+ Output::
+ 1 variable
+ maskedValues - 3D (t,lat,lon), value of only masked values
+
+ '''
+
+ mask = ~mArray.mask.ravel()
+ #beware yield fast, but generates a type called "generate" that does not allow for array methods
+ for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
+ if maskedValue:
+ yield index
+#******************************************************************
+def createMainDirectory(mainDirStr):
+ '''
+ Purpose:: to create the main directory for storing information and
+ the subdirectories for storing information
+ Input:: mainDir: a directory for where all information generated from
+ the program are to be stored
+ Output:: None
+
+ '''
+ global MAINDIRECTORY
+
+ MAINDIRECTORY = mainDirStr
+ #if directory doesnt exist, creat it
+ if not os.path.exists(MAINDIRECTORY):
+ os.makedirs(MAINDIRECTORY)
+
+ os.chdir((MAINDIRECTORY))
+ #create the subdirectories
+ try:
+ os.makedirs('images')
+ os.makedirs('textFiles')
+ os.makedirs('MERGnetcdfCEs')
+ os.makedirs('TRMMnetcdfCEs')
+ except:
+ print "Directory exists already!!!"
+ #TODO: some nice way of prompting if it is ok to continue...or just leave
+
+ return
+#******************************************************************
+def find_nearest(thisArray,value):
+ '''
+ Purpose :: to determine the value within an array closes to
+ another value
+
+ Input ::
+ Output::
+ '''
+ idx = (np.abs(thisArray-value)).argmin()
+ return thisArray[idx]
+#******************************************************************
+def preprocessingMERG(MERGdirname):
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ Directory to the location of the raw MERG files, preferably zipped
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ os.chdir((MERGdirname+'/'))
+ imgFilename = ''
+
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ for files in glob.glob("*-pixel"):
+ #for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+ #TODO: must be something more efficient!
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+ #generate the lats4D command for GrADS
+ lats4D = 'lats4d -v -q -lat '+LATMIN + ' ' +LATMAX +' -lon ' +LONMIN +' ' +LONMAX +' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ #lats4D = 'lats4d -v -q -lat -40 -15 -lon 10 40 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+ #lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+ imgFilename = hr+'Z'+day+mth+yy+'.gif'
+ tempMaskedImages(imgFilename)
+
+ #when all the files have benn converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ #mv all images
+ subprocess.call('mkdir mergImgs', shell=True)
+ subprocess.call('mv *.gif mergImgs', shell=True)
+ return
+#******************************************************************
+def postProcessingNetCDF(dataset, dirName = None):
+ '''
+
+ TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
+
+ Purpose::
+ Utility script displaying the data in generated NETCDF4 files
+ in GrADS
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
+ string: Directory to the location of the raw (MERG) files, preferably zipped
+
+ Output::
+ images in location as specfied in the code
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ '''
+
+ coreDir = os.path.dirname(MAINDIRECTORY)
+ #coreDir = os.path.dirname(os.path.abspath(__file__))
+ ImgFilename = ''
+ frameList=[]
+ fileList =[]
+ lines =[]
+ var =''
+ firstTime = True
+ printLine = 0
+ lineNum = 1
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ print "coreDir is ", str(coreDir)
+ prevFrameNum = 0
+
+ if dataset == 1:
+ var = 'ch4'
+ dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
+ ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
+ ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ gsFile = coreDir+"/GrADSscripts/cs2.gs"
+ sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
+ lineNum = 50
+
+ elif dataset ==2:
+ var = 'precipAcc'
+ dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
+ ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
+ ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
+ origsFile = coreDir+"/GrADSscripts/cs3.gs"
+ gsFile = coreDir+"/GrADSscripts/cs4.gs"
+ sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
+ lineNum = 10
+
+ elif dataset ==3:
+ var = 'ch4'
+ ctlTitle = 'TITLE MERG DATA'
+ ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
+ if dirName is None:
+ print "Enter directory for original files"
+ return
+ else:
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ sologsFile = coreDir+"/GrADSscripts/infrared.gs"
+ lineNum = 54
+
+ #sort files
+ os.chdir((dirName+'/'))
+ try:
+ os.makedirs('ctlFiles')
+ except:
+ print "ctl file folder created already"
+
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+ for eachfile in files:
+ fullFname = os.path.splitext(eachfile)[0]
+ fnameNoExtension = fullFname.split('.nc')[0]
+ if dataset == 1 or dataset == 2:
+ frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
+ #create the ctlFile
+ ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
+ #the ctl file
+ subprocessCall = 'rm ' +ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ subprocessCall = 'touch '+ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ lineToWrite = 'echo DSET ' + dirName+'/'+fnameNoExtension+'.nc' +' >>' + ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo DTYPE netcdf >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo UNDEF 0 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo XDEF 413 LINEAR -9.984375 0.036378335 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo YDEF 412 LINEAR 5.03515625 0.036378335 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo ZDEF 01 LEVELS 1 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo TDEF 99999 linear 31aug2009 1hr >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo VARS 1 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite ='echo '+ctlLine+' >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo ENDVARS >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+
+ #create plot of just that data
+ subprocessCall = 'cp '+ origsFile+' '+sologsFile
+ subprocess.call(subprocessCall, shell=True)
+
+ ImgFilename = fnameNoExtension + '.gif'
+
+ displayCmd = '\''+'d '+ var+'\''+'\n'
+ newFileCmd = '\''+'open '+ ctlFile1+'\''+'\n'
+ colorbarCmd = '\''+'run cbarn'+'\''+'\n'
+ printimCmd = '\''+'prin
<TRUNCATED>
[14/50] [abbrv] git commit: remove unnecessary files
Posted by le...@apache.org.
remove unnecessary files
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/1d0b5a5b
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/1d0b5a5b
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/1d0b5a5b
Branch: refs/heads/master
Commit: 1d0b5a5b7e6769d12a4777f0a945400fff9a8883
Parents: 0b6b8a2
Author: kwhitehall <k_...@yahoo.com>
Authored: Sun Apr 13 20:59:22 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Sun Apr 13 20:59:22 2014 -0400
----------------------------------------------------------------------
mccsearch/code/files.pyc | Bin 18453 -> 0 bytes
mccsearch/code/mainProgGoyens.py | 126 -
mccsearch/code/mainProgTemp.py | 99 -
mccsearch/code/mainProgWorks.py | 124 -
mccsearch/code/mccSearch.pyc | Bin 98227 -> 0 bytes
mccsearch/code/mccSearchGoyens.py | 3606 ---------------------------
mccsearch/code/newPreprocessingMERG.py | 175 --
mccsearch/code/preprocessingMERG.py | 125 -
mccsearch/code/process.pyc | Bin 29786 -> 0 bytes
9 files changed, 4255 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/files.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.pyc b/mccsearch/code/files.pyc
deleted file mode 100644
index f8a5577..0000000
Binary files a/mccsearch/code/files.pyc and /dev/null differ
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/mainProgGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgGoyens.py b/mccsearch/code/mainProgGoyens.py
deleted file mode 100644
index ad54ab1..0000000
--- a/mccsearch/code/mainProgGoyens.py
+++ /dev/null
@@ -1,126 +0,0 @@
-'''
-# running the program
-# Kim Whitehall Nov 2012
-# Last updated: 15 May 2013
-'''
-
-#import rcmes
-import sys
-import networkx as nx
-import mccSearchGoyens
-#import cloudclusters
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
-
-def main():
- CEGraph = nx.DiGraph()
- prunedGraph = nx.DiGraph()
- MCCList =[]
- MCSList=[]
- MCSMCCNodesList =[]
- allMCSsList =[]
- allCETRMMList =[]
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccSearchGoyens/caseStudy1"
- #directories for the original data
- TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
- CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
- #allMCSsDict ={{}}
-
- # mccSearchGoyens.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # sys.exit()
-
- # # mccSearchGoyens.plotAccTRMM(TRMMCEdirName)
- # sys.exit()
-
- #create main directory and file structure for storing intel
- mccSearchGoyens.createMainDirectory(mainDirStr)
- TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
- CEdirName = mainDirStr+'/MERGnetcdfCEs'
-
- # #mccSearchGoyens.postProcessingNetCDF(3,CEoriDirName)
- # mccSearchGoyens.postProcessingNetCDF(2)
- # sys.exit()
-
- #using merg data ***********
- print "\n -------------- Read MERG Data ----------"
- mergImgs, timeList = mccSearchGoyens.readMergData(CEoriDirName)
- print ("-"*80)
-
- print 'in main', len(mergImgs)
- print "\n -------------- TESTING findCloudElements ----------"
- CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName isnt entered for what ever reason
- # CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearchGoyens.findPrecipRate(TRMMdirName)
- # sys.exit()
- print ("-"*80)
- print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
- print ("-"*80)
- print "\n -------------- TESTING findCloudClusters ----------"
- prunedGraph = mccSearchGoyens.findCloudClusters(CEGraph)
- print ("-"*80)
- print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING findMCCs ----------"
- MCCList,MCSList = mccSearchGoyens.findMCC(prunedGraph)
- print ("-"*80)
- print "MCC List has been acquired ", len(MCCList)
- # for eachList in MCCList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
- print "MCS List has been acquired ", len(MCSList)
- # for eachList in MCSList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING TRMM ----------"
- #now ready to perform various calculations/metrics
- print "\n -------------- TESTING METRICS ----------"
- #MCCTimes, tdelta = mccSearchGoyens.temporalAndAreaInfoMetric(MCCList)
- print ("-"*80)
- #print "MCCTimes is: ", MCCTimes
- print "creating the MCS userfile ", mccSearchGoyens.createTextFile(MCCList)
-
- # print "number of MCCs is: ", mccSearchGoyens.numberOfFeatures(MCCList)
- # print "longest duration is: ", mccSearchGoyens.longestDuration(MCCTimes), "hrs"
- # print "shortest duration is: ", mccSearchGoyens.shortestDuration(MCCTimes), "hrs"
- # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
- # print "Average duration is: ", mccSearchGoyens.averageDuration(MCCTimes), "hrs"
- # print "Average size is: ", mccSearchGoyens.averageFeatureSize(MCCList), "km^2"
-
- # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
- # hist, bin_edges = mccSearchGoyens.commonFeatureSize(allMCSsList)
- # print "Histogram is ", hist, bin_edges
- # plt.bar(bin_edges[:-1], hist, width = 10)
- # plt.xlim(min(bin_edges), max(bin_edges))
- # plt.show()
-
- #some plots that work
- # mccSearchGoyens.plotAccTRMM(MCCList)
- # mccSearchGoyens.displayPrecip(MCCList)
- # mccSearchGoyens.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- # mccSearchGoyens.displaySize(MCCList)
- # mccSearchGoyens.displayPrecip(MCCList)
- # mccSearchGoyens.plotHistograms(MCCList)
- #sys.exit()
- #
- print ("-"*80)
-
- # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
- # print "********** findPrecipRate found"
- # allCETRMMList = mccSearchGoyens.findPrecipRate(CEdirName,TRMMdirName)
- # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
-
- # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
- # print ("-"*80)
- # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
- # print ("-"*80)
- # # sys.exit()
-
-main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/mainProgTemp.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgTemp.py b/mccsearch/code/mainProgTemp.py
deleted file mode 100644
index 9be65b5..0000000
--- a/mccsearch/code/mainProgTemp.py
+++ /dev/null
@@ -1,99 +0,0 @@
-'''
-# running the program
-'''
-
-import sys
-import networkx as nx
-import mccSearch
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
-import subprocess
-
-
-def main():
- CEGraph = nx.DiGraph()
- prunedGraph = nx.DiGraph()
- MCCList =[]
- MCSList=[]
- MCSMCCNodesList =[]
- allMCSsList =[]
- allCETRMMList =[]
-
- #for GrADs
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- mainDirStr= "/directory/to/where/to/store/outputs"
- TRMMdirName = "/directory/to/the/TRMM/netCDF/files"
- CEoriDirName = "/directory/to/the/MERG/netCDF/files"
-
- #for first time working with the raw MERG zipped files
- # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # ---------------------------------------------------------------------------------
-
-
- #create main directory and file structure for storing intel
- mccSearch.createMainDirectory(mainDirStr)
- TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
- CEdirName = mainDirStr+'/MERGnetcdfCEs'
-
- # for doing some postprocessing with the clipped datasets instead of running the full program, e.g.
- # mccSearch.postProcessingNetCDF(3,CEoriDirName)
- # mccSearch.postProcessingNetCDF(2)
- # -------------------------------------------------------------------------------------------------
-
- #let's go!
- print "\n -------------- Read MERG Data ----------"
- mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
- print ("-"*80)
-
- print 'in main', len(mergImgs)
- #print 'timeList', timeList
- print 'TRMMdirName ', TRMMdirName
- print "\n -------------- TESTING findCloudElements ----------"
- CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
- # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName,timeList)
- # ----------------------------------------------------------------------------------------------
- print ("-"*80)
- print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
- print ("-"*80)
- print "\n -------------- TESTING findCloudClusters ----------"
- prunedGraph = mccSearch.findCloudClusters(CEGraph)
- print ("-"*80)
- print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING findMCCs ----------"
- MCCList,MCSList = mccSearch.findMCC(prunedGraph)
- print ("-"*80)
- print "MCC List has been acquired ", len(MCCList)
- print "MCS List has been acquired ", len(MCSList)
- print ("-"*80)
- #now ready to perform various calculations/metrics
- print "\n -------------- TESTING METRICS ----------"
-
- #some calculations/metrics that work that work
- # print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
- # print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
- # MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
- # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
- # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
- # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
- # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
- # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
-
- #some plots that work
- # mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_09:00:00')
- # mccSearch.displaySize(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistogram(MCCList)
- #
- print ("-"*80)
-
-main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/mainProgWorks.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgWorks.py b/mccsearch/code/mainProgWorks.py
deleted file mode 100644
index f6eb55d..0000000
--- a/mccsearch/code/mainProgWorks.py
+++ /dev/null
@@ -1,124 +0,0 @@
-'''
-# running the program
-# Kim Whitehall Nov 2012
-# Last updated: 15 May 2013
-'''
-
-#import rcmes
-import sys
-import networkx as nx
-import mccSearch
-#import cloudclusters
-import numpy as np
-import numpy.ma as ma
-import files
-import matplotlib.pyplot as plt
-
-def main():
- CEGraph = nx.DiGraph()
- prunedGraph = nx.DiGraph()
- MCCList =[]
- MCSList=[]
- MCSMCCNodesList =[]
- allMCSsList =[]
- allCETRMMList =[]
- mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
- #directories for the original data
- TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
- CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
- #allMCSsDict ={{}}
-
- # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
- # sys.exit()
-
- # # mccSearch.plotAccTRMM(TRMMCEdirName)
- # sys.exit()
-
- #create main directory and file structure for storing intel
- mccSearch.createMainDirectory(mainDirStr)
- TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
- CEdirName = mainDirStr+'/MERGnetcdfCEs'
-
- # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
- # mccSearch.postProcessingNetCDF(2)
- # sys.exit()
-
- #using merg data ***********
- print "\n -------------- Read MERG Data ----------"
- mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
- print ("-"*80)
-
- print 'in main', len(mergImgs)
- print "\n -------------- TESTING findCloudElements ----------"
- CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
- #if the TRMMdirName isnt entered for what ever reason
- # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
- # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
- # sys.exit()
- print ("-"*80)
- print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
- print ("-"*80)
- print "\n -------------- TESTING findCloudClusters ----------"
- prunedGraph = mccSearch.findCloudClusters(CEGraph)
- print ("-"*80)
- print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING findMCCs ----------"
- MCCList,MCSList = mccSearch.findMCC(prunedGraph)
- print ("-"*80)
- print "MCC List has been acquired ", len(MCCList)
- # for eachList in MCCList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
- print "MCS List has been acquired ", len(MCSList)
- # for eachList in MCSList:
- # print eachList
- # for eachNode in eachList:
- # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
- print ("-"*80)
- #sys.exit()
- print "\n -------------- TESTING TRMM ----------"
- #now ready to perform various calculations/metrics
- print "\n -------------- TESTING METRICS ----------"
- MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
- print ("-"*80)
- print "MCCTimes is: ", MCCTimes
- print ("-"*80)
- print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
- print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
- print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
- #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
- print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
-
- # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
- # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
- # print "Histogram is ", hist, bin_edges
- # plt.bar(bin_edges[:-1], hist, width = 10)
- # plt.xlim(min(bin_edges), max(bin_edges))
- # plt.show()
-
- #some plots that work
- # mccSearch.plotAccTRMM(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
- # mccSearch.displaySize(MCCList)
- # mccSearch.displayPrecip(MCCList)
- # mccSearch.plotHistograms(MCCList)
- #sys.exit()
- #
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print "********** findPrecipRate found"
- # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
-
- # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
- # print ("-"*80)
- # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
- # print ("-"*80)
- # # sys.exit()
-
-main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/1d0b5a5b/mccsearch/code/mccSearch.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.pyc b/mccsearch/code/mccSearch.pyc
deleted file mode 100644
index dc462d1..0000000
Binary files a/mccsearch/code/mccSearch.pyc and /dev/null differ
[04/50] [abbrv] initial mccsearch code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/mccSearchGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchGoyens.py b/mccsearch/code/mccSearchGoyens.py
new file mode 100644
index 0000000..0f49a10
--- /dev/null
+++ b/mccsearch/code/mccSearchGoyens.py
@@ -0,0 +1,3606 @@
+'''
+# All the functions for the MCC search algorithm
+# RCMES data in format (t,lat,lon), value
+# Kim Whitehall Sep 2012
+# Last updated: 23Jan 2014, 6 Dec 2013, 21 May 2013, 4 Jun 2013
+# June 13 - called this merg.py and edited out comments. merg.py is where all "cleaned" functions are to go
+# June 27 - including the first part of graphing, according to the center_of_mass
+# July 1 - going back to brute force graph approach, but weigh the edges according to closeness, up to within 20_perc of the forecasted
+#
+
+Mimumum MCS is 3 hours
+'''
+
+import datetime
+from datetime import timedelta, datetime
+import calendar
+import fileinput
+import glob
+import itertools
+import json
+import math
+import Nio
+from netCDF4 import Dataset, num2date, date2num
+import numpy as np
+import numpy.ma as ma
+import os
+import pickle
+import re
+from scipy import ndimage
+import string
+import subprocess
+import sys
+import time
+
+import networkx as nx
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+from matplotlib.dates import DateFormatter,HourLocator
+from matplotlib import cm
+import matplotlib.cm as cm
+import matplotlib.colors as mcolors
+from matplotlib.ticker import FuncFormatter, FormatStrFormatter
+
+#existing modules in services
+import files
+import process
+#----------------------- GLOBAL VARIABLES --------------------------
+# --------------------- User defined variables ---------------------
+#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
+#the first point closest the the value (for the min) from the MERG data is used, etc.
+LATMIN = '5.0' #'5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '20.0' #'20.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
+LONMIN = '0.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '30.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+XRES = 4.0 #x direction spatial resolution in km
+YRES = 4.0 #y direction spatial resolution in km
+TRES = 1 #temporal resolution in hrs
+T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
+T_BB_MIN = 218 #221 #cooler temp for the center of the system
+STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
+ #have same rank of the matrix it is being compared against
+CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
+MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
+MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
+MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
+LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
+LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
+DIRECTION = 45.0 #general direction of the wind flow in degrees
+AREA_MIN = 3500.0 #2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
+MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+
+#---Assuming using the MCC function, these will have to be changed
+ECCENTRICITY_THRESHOLD_MAX = 0.75 #tending to 1 is a circle e.g. hurricane,
+ECCENTRICITY_THRESHOLD_MIN = 0.25 #0.65 #tending to 0 is a linear e.g. squall line
+OUTER_CLOUD_SHIELD_AREA = 30000.0 #80000.0 #100000.0 #km^2
+INNER_CLOUD_SHIELD_AREA = 3000.0#30000.0 #50000.0 #km^2
+OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
+INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
+MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
+MAXIMUM_DURATION = 100 #24 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
+
+#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+#------------------- End user defined Variables -------------------
+edgeWeight = [1,2,3] #weights for the graph edges
+#graph object fo the CEs meeting the criteria
+CLOUD_ELEMENT_GRAPH = nx.DiGraph()
+#graph meeting the CC criteria
+PRUNED_GRAPH = nx.DiGraph()
+#------------------------ End GLOBAL VARS -------------------------
+#************************ Begin Functions *************************
+#******************************************************************
+def readMergData(dirname):
+ '''
+ Purpose::
+ Read MERG data into RCMES format
+
+ Input::
+ Directory to the MERG files in NETCDF format
+
+ Output::
+ A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
+ criteria for each frame
+ **remove below**
+ A dictionary of all the MERG data from the files in the directory given.
+ The dictionary contains, a string representing the time (a datetime string), a
+ tuple (lat,lon,value) representing only the data that meets the temperature requirements
+ i.e. T_BB_MAX
+
+ Assumptions::
+ The MERG data has been converted to NETCDF using LATS4D
+ The data has the same lat/lon format
+ '''
+
+ global LAT
+ global LON
+
+ # these strings are specific to the MERG data
+ mergVarName = 'ch4'
+ mergTimeVarName = 'time'
+ mergLatVarName = 'latitude'
+ mergLonVarName = 'longitude'
+
+ filelistInstructions = dirname + '/*'
+ filelist = glob.glob(filelistInstructions)
+
+
+ #sat_img is the array that will contain all the masked frames
+ mergImgs = []
+ #timelist of python time strings
+ timelist = []
+ time2store = None
+ tempMaskedValueNp =[]
+
+
+ filelist.sort()
+ nfiles = len(filelist)
+
+ # Crash nicely if there are no netcdf files
+ if nfiles == 0:
+ print 'Error: no files in this directory! Exiting elegantly'
+ sys.exit()
+ else:
+ # Open the first file in the list to read in lats, lons and generate the grid for comparison
+ tmp = Nio.open_file(filelist[0], format='nc')
+ #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
+
+ #clip the lat/lon grid according to user input
+ #http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
+ latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
+ lonsraw = tmp.variables[mergLonVarName][mergLonVarName+"|"+LONMIN+":"+LONMAX].astype('f2')
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ LON, LAT = np.meshgrid(lonsraw, latsraw)
+ #clean up
+ latsraw =[]
+ lonsraw = []
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+ #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
+ tmp.close
+
+ for files in filelist:
+ try:
+ thisFile = Nio.open_file(files, format='nc')
+ #TODO: thisFile = netCDF4.Dataset(files)
+
+ #clip the dataset according to user lat,lon coordinates
+ #mask the data and fill with zeros for later
+ tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
+ +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
+
+ tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
+
+ #get the actual values that the mask returned
+ tempMaskedValue = ma.zeros((tempRaw.shape)).astype('int16')
+ for index, value in maenumerate(tempMask):
+ time_index, lat_index, lon_index = index
+ tempMaskedValue[time_index,lat_index, lon_index]=value
+
+ timesRaw = thisFile.variables[mergTimeVarName]
+ #convert this time to a python datastring
+ time2store, _ = process.getModelTimes(files, mergTimeVarName)
+ #extend instead of append because getModelTimes returns a list already and we don't
+ #want a list of list
+ timelist.extend(time2store)
+ #to get the actual masked data only
+ #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
+ mergImgs.extend(tempMaskedValue)
+ thisFile.close
+ thisFile = None
+
+ except:
+ print "bad file! ", file
+
+ mergImgs = ma.array(mergImgs)
+
+ return mergImgs, timelist
+#******************************************************************
+def findCloudElements(mergImgs,timelist,TRMMdirName=None):
+ '''
+ Purpose::
+ Determines the contiguous boxes for a given time of the satellite imaages i.e. each frame
+ It recursively calls another function to determine if the boxes are touching
+
+ Input::
+ 2 variables
+ sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
+ maximum acceptable temperature, T_BB_MAX
+ timelist: a list of python datatimes
+ TRMMdirName (optional): string representing the path where to find the TRMM datafiles
+
+ Output::
+ 1 variable
+ cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
+ The dictionary is cloudElements={"frame": datetime object,
+ "cloudElementNum": integer,
+ "CEuniqueID": string,
+ "cloudElementLatLon": 2D array of lat_lon}
+ "cloudElementCenter": 1D array [0] is lat, [1]is lon of the center_of_mass}
+ "cloudElementEccentricity: float"
+
+ Assumptions::
+ Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
+ required according to Vila et al. (2008) is 2400km^2
+ therefore, 2400/16 = 150 contiguous squares
+ '''
+
+ frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
+ CEcounter = 0
+ frameCEcounter = 0
+ frameNum = 0
+ cloudElementEpsilon = 0.0
+ cloudElementDict = {}
+ cloudElementCenter = [] #list with two elements [lat,lon] for the center of a CE
+ prevFrameCEs = [] #list for CEs in previous frame
+ currFrameCEs = [] #list for CEs in current frame
+ cloudElementLat = [] #list for a particular CE's lat values
+ cloudElementLon = [] #list for a particular CE's lon values
+ cloudElementLatLons = [] #list for a particular CE's (lat,lon) values
+
+ prevLatValue = 0.0
+ prevLonValue = 0.0
+ TIR_min = 0.0
+ TIR_max = 0.0
+ temporalRes = 3 # TRMM data is 3 hourly
+ precipTotal = 0.0
+ CETRMMList =[]
+ precip =[]
+ TRMMCloudElementLatLons =[]
+
+ #edgeWeight = [1,2]
+ minCELatLimit = 0.0
+ minCELonLimit = 0.0
+ maxCELatLimit = 0.0
+ maxCELonLimit = 0.0
+
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ #openfile for storing ALL cloudElement information
+ cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
+
+
+ #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
+ cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
+ #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
+
+ #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
+ for t in xrange(mergImgs.shape[0]):
+ #-------------------------------------------------
+ # #textfile name for saving the data for arcgis
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ #determine contiguous locations with temeperature below the warmest temp i.e. cloudElements in each frame
+ frame, CEcounter = ndimage.measurements.label(mergImgs[t,:,:], structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ frameNum += 1
+
+ #for each of the areas identified, check to determine if it a valid CE via an area and T requirement
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ loc = ndimage.find_objects(frame==(count+1))[0]
+ cloudElement = mergImgs[t,:,:][loc]
+ labels, lcounter = ndimage.label(cloudElement)
+
+ #determine the true lats and lons for this particular CE
+ cloudElementLat = LAT[loc[0],0]
+ cloudElementLon = LON[0,loc[1]]
+
+ #determine number of boxes in this cloudelement
+ numOfBoxes = np.count_nonzero(cloudElement)
+ cloudElementArea = numOfBoxes*XRES*YRES
+
+ #If the area is greater than the area required, then start considering as CE
+ #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
+ #if it does, still consider as CE
+
+ if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
+ #do a second pass on the identified area
+ # if lcounter > 1:
+ # #i.e.there is more than one item in the area identified
+ # for lcount in xrange(lcounter):
+ # print " in lcount for", lcount
+ # loc = ndimage.find_objects(labels ==(lcount+1))[0]
+ # lcloudElement = cloudElement[:,:][loc]
+ # llabels, _ = ndimage.label(lcloudElement)
+ # plt.imshow(llabels)
+ # plt.show()
+
+
+ #get some time information and labeling info
+ frameTime = str(timelist[t])
+ frameCEcounter +=1
+ CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
+
+ #-------------------------------------------------
+ #textfile name for accesing CE data using MATLAB code
+ # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
+ # cloudElementsTextFile = open(thisFileName,'w')
+ #-------------------------------------------------
+
+ # ------ NETCDF File stuff for brightness temp stuff ------------------------------------
+ thisFileName = MAINDIRECTORY +'/MERGnetcdfCEs/cloudElements'+ (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFCEData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFCEData.description = 'Cloud Element '+CEuniqueID + ' temperature data'
+ currNetCDFCEData.calendar = 'standard'
+ currNetCDFCEData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFCEData.createDimension('time', None)
+ currNetCDFCEData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFCEData.createDimension('lon', len(LON[0,:]))
+ # variables
+ tempDims = ('time','lat', 'lon',)
+ times = currNetCDFCEData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitudes = currNetCDFCEData.createVariable('latitude', 'f8', ('lat',))
+ longitudes = currNetCDFCEData.createVariable('longitude', 'f8', ('lon',))
+ brightnesstemp = currNetCDFCEData.createVariable('brightnesstemp', 'i16',tempDims )
+ brightnesstemp.units = 'Kelvin'
+ # NETCDF data
+ dates=[timelist[t]+timedelta(hours=0)]
+ times[:] = date2num(dates,units=times.units)
+ longitudes[:] = LON[0,:]
+ longitudes.units = "degrees_east"
+ longitudes.long_name = "Longitude"
+
+ latitudes[:] = LAT[:,0]
+ latitudes.units = "degrees_north"
+ latitudes.long_name ="Latitude"
+
+ #generate array of zeros for brightness temperature
+ brightnesstemp1 = ma.zeros((1,len(latitudes), len(longitudes))).astype('int16')
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #if other dataset (TRMM) assumed to be a precipitation dataset was entered
+ if TRMMdirName:
+ #------------------TRMM stuff -------------------------------------------------
+ fileDate = ((str(timelist[t])).replace(" ", "")[:-8]).replace("-","")
+ fileHr1 = (str(timelist[t])).replace(" ", "")[-8:-6]
+
+ if int(fileHr1) % temporalRes == 0:
+ fileHr = fileHr1
+ else:
+ fileHr = (int(fileHr1)/temporalRes) * temporalRes
+ if fileHr < 10:
+ fileHr = '0'+str(fileHr)
+ else:
+ str(fileHr)
+
+ #open TRMM file for the resolution info and to create the appropriate sized grid
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+str(fileHr)+".7A.nc"
+
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ nygrdTRMM = len(LATTRMM[:,0]); nxgrdTRMM = len(LONTRMM[0,:])
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ #get the lat/lon info from the file
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+ # ------ NETCDF File info for writing TRMM CE rainfall ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
+ currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' precipitation data'
+ currNetCDFTRMMData.calendar = 'standard'
+ currNetCDFTRMMData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFTRMMData.createDimension('time', None)
+ currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
+ currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
+
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+
+ #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value)
+
+ #generate the comma separated file for GIS
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ #temp data for CE NETCDF file
+ brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value
+
+ if TRMMdirName:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])]
+ CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]]))
+
+
+ brightnesstemp[:] = brightnesstemp1[:]
+ currNetCDFCEData.close()
+
+ if TRMMdirName:
+
+ #print finalCETRMMvalues
+ #calculate the total precip associated with the feature
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ precipTotal += value
+ precip.append(value)
+
+ rainFallacc[:] = finalCETRMMvalues[:]
+ currNetCDFTRMMData.close()
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ try:
+ maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except:
+ pass
+
+ #sort cloudElementLatLons by lats
+ cloudElementLatLons.sort(key=lambda tup: tup[0])
+
+ #determine if the cloud element the shape
+ cloudElementEpsilon = eccentricity (cloudElement)
+ cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementCenter.append(latCenter)
+ cloudElementCenter.append(lonCenter)
+ cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsUserFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsUserFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ #populate the dictionary
+ if TRMMdirName:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min, 'cloudElementPrecipTotal':precipTotal,'cloudElementLatLonTRMM':CETRMMList, 'TRMMArea': TRMMArea}
+ else:
+ cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min,}
+
+ #current frame list of CEs
+ currFrameCEs.append(cloudElementDict)
+
+ #draw the graph node
+ CLOUD_ELEMENT_GRAPH.add_node(CEuniqueID, cloudElementDict)
+
+ if frameNum != 1:
+ for cloudElementDict in prevFrameCEs:
+ thisCElen = len(cloudElementLatLons)
+ percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
+
+ #change weights to integers because the built in shortest path chokes on floating pts
+ #according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
+ if percentageOverlap >= 0.95:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
+
+ elif percentageOverlap >= 0.90 and percentageOverlap < 0.95 :
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[1])
+
+ elif areaOverlap >= MIN_OVERLAP:
+ CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
+
+ else:
+ #TODO: remove this else as we only wish for the CE details
+ #ensure only the non-zero elements are considered
+ #store intel in allCE file
+ labels, _ = ndimage.label(cloudElement)
+ cloudElementsFile.write("\n-----------------------------------------------")
+ cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
+ # cloudElementLat = LAT[loc[0],0]
+ # cloudElementLon = LON[0,loc[1]]
+
+ #populate cloudElementLatLons by unpacking the original values from loc
+ #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # as cloudElement is masked
+ for index,value in np.ndenumerate(cloudElement):
+ if value != 0 :
+ lat_index,lon_index = index
+ lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
+ cloudElementLatLons.append(lat_lon_tuple)
+
+ cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
+ #latCenter and lonCenter are given according to the particular array defining this CE
+ #so you need to convert this value to the overall domain truth
+ latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
+ latCenter = cloudElementLat[round(latCenter)]
+ lonCenter = cloudElementLon[round(lonCenter)]
+ cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
+ cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
+ cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
+ cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
+ cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
+ cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
+ cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
+ cloudElementsFile.write("\n-----------------------------------------------")
+
+ #reset list for the next CE
+ nodeExist = False
+ cloudElementCenter=[]
+ cloudElement = []
+ cloudElementCenter = []
+ cloudElementLat=[]
+ cloudElementLon =[]
+ cloudElementLatLons =[]
+ brightnesstemp1 =[]
+ brightnesstemp =[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ CETRMMList =[]
+ precipTotal = 0.0
+ precip=[]
+ TRMMCloudElementLatLons=[]
+
+ #reset for the next time
+ prevFrameCEs =[]
+ prevFrameCEs = currFrameCEs
+ currFrameCEs =[]
+
+ cloudElementsFile.close
+ cloudElementsUserFile.close
+ #if using ARCGIS data store code, uncomment this file close line
+ #cloudElementsTextFile.close
+
+ #clean up graph - remove parent and childless nodes
+ outAndInDeg = CLOUD_ELEMENT_GRAPH.degree_iter()
+ toRemove = [node[0] for node in outAndInDeg if node[1]<1]
+ CLOUD_ELEMENT_GRAPH.remove_nodes_from(toRemove)
+
+ print "number of nodes are: ", CLOUD_ELEMENT_GRAPH.number_of_nodes()
+ print "number of edges are: ", CLOUD_ELEMENT_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #hierachial graph output
+ #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
+ drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
+
+ return CLOUD_ELEMENT_GRAPH
+#******************************************************************
+def findPrecipRate(TRMMdirName):
+ '''
+ TODO: needs fixing
+ Purpose:: Determines the precipitation rates for MCSs found
+
+ Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
+
+ Output:: a list of dictionary of the TRMM data
+ NB: also creates netCDF with TRMM data for each CE (for post processing) index
+ in MAINDIRECTORY/TRMMnetcdfCEs
+
+ Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
+
+ '''
+ allCEnodesTRMMdata =[]
+ TRMMdataDict={}
+ precipTotal = 0.0
+
+ os.chdir((MAINDIRECTORY+'/MERGnetcdfCEs/'))
+ imgFilename = ''
+ temporalRes = 3 #3 hours for TRMM
+
+ #sort files
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+
+ for afile in files:
+ #try:
+ print "in for ", afile
+ fullFname = os.path.splitext(afile)[0]
+ noFrameExtension = (fullFname.replace("_","")).split('F')[0]
+ CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
+ fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
+ fileDateTime = fileDateTimeChar.replace("-","")
+ fileDate = fileDateTime[:-6]
+ fileHr=fileDateTime[-6:-4]
+
+ cloudElementData = Dataset(afile,'r', format='NETCDF4')
+ brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
+ latsrawCloudElements = cloudElementData.variables['latitude'][:]
+ lonsrawCloudElements = cloudElementData.variables['longitude'][:]
+
+ # print "noFrameExtension", noFrameExtension
+ # print "fileDateTime ", fileDateTime
+ # print "fileDateTimeChar ", fileDateTimeChar
+ # print "fullFname ", fullFname
+ # print "time is: ", (fullFname.replace("_"," ").split('F')[0]).split('s')[1]
+ # print "fileDate and fileHr ", fileDate, fileHr
+ # print "CEuniqueID ", CEuniqueID
+ # sys.exit()
+ if int(fileHr) % 3 == 0:
+ TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
+ #print "TRMMfileName is ", TRMMfileName
+ TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
+ precipRate = TRMMData.variables['pcp'][:,:,:]
+ latsrawTRMMData = TRMMData.variables['latitude'][:]
+ lonsrawTRMMData = TRMMData.variables['longitude'][:]
+ lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
+ LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
+
+ #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
+ nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
+
+ precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
+ #---------regrid the TRMM data to the MERG dataset ----------------------------------
+ #regrid using the do_regrid stuff from the Apache OCW
+ regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
+ regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
+ #----------------------------------------------------------------------------------
+
+ # #get the lat/lon info from cloudElement
+ latCEStart = LAT[0][0]
+ latCEEnd = LAT[-1][0]
+ lonCEStart = LON[0][0]
+ lonCEEnd = LON[0][-1]
+
+ #get the lat/lon info for TRMM data (different resolution)
+ latStartT = find_nearest(latsrawTRMMData, latCEStart)
+ latEndT = find_nearest(latsrawTRMMData, latCEEnd)
+ lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
+ lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
+ latStartIndex = np.where(latsrawTRMMData == latStartT)
+ latEndIndex = np.where(latsrawTRMMData == latEndT)
+ lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
+ lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
+
+ #get the relevant TRMM info
+ CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
+ TRMMData.close()
+
+
+
+ # ------ NETCDF File stuff ------------------------------------
+ thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
+ # print "thisFileName ", thisFileName
+ # sys.exit()
+ currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
+ currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
+ currNetCDFData.calendar = 'standard'
+ currNetCDFData.conventions = 'COARDS'
+ # dimensions
+ currNetCDFData.createDimension('time', None)
+ currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
+ currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
+ # variables
+ TRMMprecip = ('time','lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
+ times.units = 'hours since '+ str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ rainFallacc.units = 'mm'
+
+ longitude[:] = LON[0,:]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
+
+ latitude[:] = LAT[:,0]
+ latitude.units = "degrees_north"
+ latitude.long_name ="Latitude"
+
+ finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
+ #-----------End most of NETCDF file stuff ------------------------------------
+ #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
+ for index,value in np.ndenumerate(brightnesstemp):
+ #print "index, value ", index, value
+ time_index, lat_index, lon_index = index
+ currTimeValue = 0
+ if value > 0:
+ finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
+
+
+ # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
+ # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
+ # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
+ # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
+ # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
+ # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
+ # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
+ # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
+ # # precipTotal = value*temporalRes*1.0
+ # # prevLatValue = currLatvalue
+ # # prevLonValue = currLonValue
+
+ TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
+ TRMMArea = TRMMnumOfBoxes*XRES*YRES
+ rainFallacc[:] = finalCETRMMvalues
+ currNetCDFData.close()
+ for index, value in np.ndenumerate(finalCETRMMvalues):
+ #print "index, value ", index, value
+ precipTotal += value
+
+ # #add info to the dictionary and to the list
+ # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
+ # allCEnodesTRMMdata.append(TRMMdataDict)
+ # print "precipTotal is ", precipTotal
+ #add info to CLOUDELEMENTSGRAPH
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
+ if eachdict[1]['uniqueID'] == CEuniqueID:
+ if not 'cloudElementPrecipTotal' in eachdict[1].keys():
+ eachdict[1]['cloudElementPrecipTotal'] = precipTotal
+ if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
+ eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
+ if not 'TRMMArea' in eachdict[1].keys():
+ eachdict[1]['TRMMArea'] = TRMMArea
+ #clean up
+ precipTotal = 0.0
+ latsrawTRMMData =[]
+ lonsrawTRMMData = []
+ latsrawCloudElements=[]
+ lonsrawCloudElements=[]
+ finalCETRMMvalues =[]
+ CEprecipRate =[]
+ brightnesstemp =[]
+ TRMMdataDict ={}
+
+ return allCEnodesTRMMdata
+#******************************************************************
+def findCloudClusters(CEGraph):
+ '''
+ Purpose:: Determines the cloud clusters properties from the subgraphs in
+ the graph i.e. prunes the graph according to the minimum depth
+
+ Input:: 1 graph - CEGraph: a directed graph of the CEs with weighted edges
+ according the area overlap between nodes (CEs) of consectuive frames
+
+ Output:: a dictionary of all possible cloudClusters
+
+ '''
+
+ seenNode = []
+ allMCSLists =[]
+ pathDictList =[]
+ pathList=[]
+
+ cloudClustersFile = open((MAINDIRECTORY+'/textFiles/cloudClusters.txt'),'wb')
+
+ for eachNode in CEGraph:
+ #check if the node has been seen before
+ if eachNode not in dict(enumerate(zip(*seenNode))):
+ #look for all trees associated with node as the root
+ thisPathDistanceAndLength = nx.single_source_dijkstra(CEGraph, eachNode)
+ #determine the actual shortestPath and minimum depth/length
+ maxDepthAndMinPath = findMaxDepthAndMinPath(thisPathDistanceAndLength)
+ if maxDepthAndMinPath:
+ maxPathLength = maxDepthAndMinPath[0]
+ shortestPath = maxDepthAndMinPath[1]
+
+ #add nodes and paths to PRUNED_GRAPH
+ for i in xrange(len(shortestPath)):
+ if PRUNED_GRAPH.has_node(shortestPath[i]) is False:
+ PRUNED_GRAPH.add_node(shortestPath[i])
+
+ #add edge if necessary
+ if i < (len(shortestPath)-1) and PRUNED_GRAPH.has_edge(shortestPath[i], shortestPath[i+1]) is False:
+ prunedGraphEdgeweight = CEGraph.get_edge_data(shortestPath[i], shortestPath[i+1])['weight']
+ PRUNED_GRAPH.add_edge(shortestPath[i], shortestPath[i+1], weight=prunedGraphEdgeweight)
+
+ #note information in a file for consideration later i.e. checking to see if it works
+ cloudClustersFile.write("\nSubtree pathlength is %d and path is %s" %(maxPathLength, shortestPath))
+ #update seenNode info
+ seenNode.append(shortestPath)
+
+ print "pruned graph"
+ print "number of nodes are: ", PRUNED_GRAPH.number_of_nodes()
+ print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
+ print ("*"*80)
+
+ #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
+ graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
+ drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
+ cloudClustersFile.close
+
+ return PRUNED_GRAPH
+#******************************************************************
+def findMCC (prunedGraph):
+ '''
+ Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
+
+ Input:: a Networkx Graph - prunedGraph
+
+ Output:: finalMCCList: a list of list of tuples representing a MCC
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ MCCList = []
+ MCSList = []
+ definiteMCC = []
+ definiteMCS = []
+ eachList =[]
+ eachMCCList =[]
+ maturing = False
+ decaying = False
+ fNode = ''
+ lNode = ''
+ removeList =[]
+ imgCount = 0
+ imgTitle =''
+
+ maxShieldNode = ''
+ orderedPath =[]
+ treeTraversalList =[]
+ definiteMCCFlag = False
+ unDirGraph = nx.Graph()
+ aSubGraph = nx.DiGraph()
+ definiteMCSFlag = False
+
+
+ #connected_components is not available for DiGraph, so generate graph as undirected
+ unDirGraph = PRUNED_GRAPH.to_undirected()
+ subGraph = nx.connected_component_subgraphs(unDirGraph)
+
+ #for each path in the subgraphs determined
+ for path in subGraph:
+ #definite is a subTree provided the duration is longer than 3 hours
+
+ if len(path.nodes()) > MIN_MCS_DURATION:
+ orderedPath = path.nodes()
+ orderedPath.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ #definiteMCS.append(orderedPath)
+
+ #build back DiGraph for checking purposes/paper purposes
+ aSubGraph.add_nodes_from(path.nodes())
+ for eachNode in path.nodes():
+ if prunedGraph.predecessors(eachNode):
+ for node in prunedGraph.predecessors(eachNode):
+ aSubGraph.add_edge(node,eachNode,weight=edgeWeight[0])
+
+ if prunedGraph.successors(eachNode):
+ for node in prunedGraph.successors(eachNode):
+ aSubGraph.add_edge(eachNode,node,weight=edgeWeight[0])
+ imgTitle = 'CC'+str(imgCount+1)
+ drawGraph(aSubGraph, imgTitle, edgeWeight) #for eachNode in path:
+ imgCount +=1
+ #----------end build back ---------------------------------------------
+
+ mergeList, splitList = hasMergesOrSplits(path)
+ #add node behavior regarding neutral, merge, split or both
+ for node in path:
+ if node in mergeList and node in splitList:
+ addNodeBehaviorIdentifier(node,'B')
+ elif node in mergeList and not node in splitList:
+ addNodeBehaviorIdentifier(node,'M')
+ elif node in splitList and not node in mergeList:
+ addNodeBehaviorIdentifier(node,'S')
+ else:
+ addNodeBehaviorIdentifier(node,'N')
+
+
+ #Do the first part of checking for the MCC feature
+ #find the path
+ treeTraversalList = traverseTree(aSubGraph, orderedPath[0],[],[])
+ print "treeTraversalList is ", treeTraversalList
+ #check the nodes to determine if a MCC on just the area criteria (consecutive nodes meeting the area and temp requirements)
+ MCCList = checkedNodesMCC(prunedGraph, treeTraversalList)
+ for aDict in MCCList:
+ for eachNode in aDict["fullMCSMCC"]:
+ addNodeMCSIdentifier(eachNode[0],eachNode[1])
+
+ #do check for if MCCs overlap
+ if MCCList:
+ if len(MCCList) > 1:
+ for count in range(len(MCCList)): #for eachDict in MCCList:
+ #if there are more than two lists
+ if count >= 1:
+ #and the first node in this list
+ eachList = list(x[0] for x in MCCList[count]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ fNode = eachList[0]
+ #get the lastNode in the previous possMCC list
+ eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ if eachList:
+ lNode = eachList[-1]
+ if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ if aNode in eachList and aNode == lNode:
+ #if edge_data is equal or less than to the exisitng edge in the tree append one to the other
+ if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']:
+ MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"])
+ MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"])
+ MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"]
+ MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
+ MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
+ MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"]
+ removeList.append(count)
+ #update the MCCList
+ if removeList:
+ for i in removeList:
+ del MCCList[i]
+ removeList =[]
+
+ #check if the nodes also meet the duration criteria and the shape crieria
+ for eachDict in MCCList:
+ #order the fullMCSMCC list, then run maximum extent and eccentricity criteria
+ if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION:
+ eachList = list(x[0] for x in eachDict["fullMCSMCC"])
+ eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ eachMCCList = list(x[0] for x in eachDict["possMCCList"])
+ eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+
+ #update the nodemcsidentifer behavior
+ #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
+ #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
+ #ensure that everything between is listed as 'M'
+ for eachNode in eachList[:(eachList.index(eachMCCList[0]))]:
+ addNodeMCSIdentifier(eachNode,'I')
+
+ addNodeMCSIdentifier(eachMCCList[0],'M')
+
+ for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #update definiteMCS list
+ for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]:
+ addNodeMCSIdentifier(eachNode, 'D')
+
+ #run maximum extent and eccentricity criteria
+ maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList)
+ #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag
+ if definiteMCCFlag == True:
+ definiteMCC.append(eachList)
+
+
+ definiteMCS.append(orderedPath)
+
+ #reset for next subGraph
+ aSubGraph.clear()
+ orderedPath=[]
+ MCCList =[]
+ MCSList =[]
+ definiteMCSFlag = False
+
+ return definiteMCC, definiteMCS
+#******************************************************************
+def traverseTree(subGraph,node, queue, checkedNodes=None):
+ '''
+ Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+
+ Input:: subGraph: a Networkx DiGraph representing a CC
+ lengthOfsubGraph: an integer representing the length of the subgraph
+ node: a string representing the node currently being checked
+ queue: a list of strings representing a list of nodes in a stack functionality
+ i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
+
+ Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
+
+ '''
+ if len(checkedNodes) == len(subGraph):
+ return checkedNodes
+
+ if not checkedNodes:
+ queue =[]
+ checkedNodes.append(node)
+
+ #check one level infront first...if something does exisit, stick it at the front of the stack
+ upOneLevel = subGraph.predecessors(node)
+ downOneLevel = subGraph.successors(node)
+ for parent in upOneLevel:
+ if parent not in checkedNodes and parent not in queue:
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ queue.insert(0,child)
+ #downCheckFlag = True
+ queue.insert(0,parent)
+
+ for child in downOneLevel:
+ if child not in checkedNodes and child not in queue:
+ if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
+ queue.insert(0,child)
+ else:
+ queue.append(child)
+
+ #print "^^^^^stack ", stack
+ for eachNode in queue:
+ if eachNode not in checkedNodes:
+ #print "if eachNode ", checkedNodes, eachNode, stack
+ checkedNodes.append(eachNode)
+ return traverseTree(subGraph, eachNode, queue, checkedNodes)
+
+ return checkedNodes
+#******************************************************************
+def checkedNodesMCC (prunedGraph, nodeList):
+ '''
+ Purpose :: Determine if this path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ nodeList: list of strings (CE ID) from the traversal
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+ '''
+
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ INITIATIONFLAG = False
+ MATURITYFLAG = False
+ DECAYFLAG = False
+ thisdict = {} #will have the same items as the cloudElementDict
+ cloudElementArea = 0.0
+ epsilon = 0.0
+ frameNum =0
+ oldNode =''
+ potentialMCCList =[]
+ durationAandB = 0
+ CounterCriteriaA = 0
+ CountercriteriaB = 0
+
+ #check for if the list contains only one string/node
+ if type(nodeList) is str:
+ oldNode=nodeList
+ nodeList =[]
+ nodeList.append(oldNode)
+
+ for node in nodeList:
+ thisdict = thisDict(node)
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ existingFrameFlag = False
+
+ if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
+ #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaAFlag = True
+ INITIATIONFLAG = True
+ MATURITYFLAG = False
+ #check if criteriaB is meet
+ cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
+
+ #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
+ if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
+ #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ CounterCriteriaBFlag = True
+ #append this information on to the dictionary
+ addInfothisDict(node, cloudElementArea, criteriaB)
+ INITIATIONFLAG = False
+ MATURITYFLAG = True
+ stage = 'M'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria B failed
+ CounterCriteriaBFlag = False
+ if INITIATIONFLAG == True:
+ stage = 'I'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
+ DECAYFLAG = True
+ MATURITYFLAG = False
+ stage = 'D'
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ else:
+ #criteria A failed
+ CounterCriteriaAFlag = False
+ CounterCriteriaBFlag = False
+ #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
+ #add as a CE before or after the main feature
+ if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
+ stage = "D"
+ DECAYFLAG = True
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+ elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
+ stage ="I"
+ potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
+
+ return potentialMCCList
+#******************************************************************
+def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
+ '''
+ Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
+ preliminary information regarding the stages of the feature
+
+ Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ node: a string representing the cloud element currently being assessed
+ CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
+ CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
+
+ Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
+ dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
+
+ '''
+ existingFrameFlag = False
+ existingMCSFrameFlag = False
+ predecessorsFlag = False
+ predecessorsMCSFlag = False
+ successorsFlag = False
+ successorsMCSFlag = False
+ frameNum = 0
+
+ frameNum = int((node.split('CE')[0]).split('F')[1])
+ if potentialMCCList==[]:
+ #list empty
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ else:
+ #list not empty
+ predecessorsFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,1)
+
+ if predecessorsFlag == True:
+
+ for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
+ existingFrameFlag = True
+
+ #this MUST come after the check for the existing frame
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+
+
+ if existingFrameFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ stage ='M'
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0:
+ stage = 'I'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ return potentialMCCList
+
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ if existingFrameFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ if predecessorsFlag == False:
+ successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+
+ if successorsFlag == True:
+ for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
+ if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
+ existingFrameFlag = True
+
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ stage = 'M'
+ potentialMCCList[index]["possMCCList"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if existingFrameFlag == False:
+ if stage == 'M':
+ stage = 'D'
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList[index]["CounterCriteriaA"]+= 1
+ potentialMCCList[index]["durationAandB"]+=1
+ elif CounterCriteriaAFlag == True:
+ potentialMCCList[index]["CounterCriteriaA"] += 1
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
+ else:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ if CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to exisiting MCCs check if it is connected to exisiting MCSs ...
+ if predecessorsFlag == False and successorsFlag == False:
+ stage = 'I'
+ predecessorsMCSFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,2)
+ if predecessorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+ else:
+ successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
+ if successorsMCSFlag == True:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+ potentialMCCList[index]["possMCCList"].append((node,'M'))
+ potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ potentialMCCList[index]["durationAandB"] += 1
+ if frameNum > potentialMCCList[index]["frameNum"]:
+ potentialMCCList[index]["frameNum"] = frameNum
+ potentialMCCList[index]["highestMCCnode"] = node
+ return potentialMCCList
+
+
+ if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["CounterCriteriaA"] +=1
+ return potentialMCCList
+ elif CounterCriteriaAFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ return potentialMCCList
+
+ #if this node isn't connected to existing MCCs or MCSs, create a new one ...
+ if predecessorsFlag == False and predecessorsMCSFlag == False and successorsFlag == False and successorsMCSFlag == False:
+ if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
+ potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
+ elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
+ potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+
+ return potentialMCCList
+#******************************************************************
+def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
+ '''
+ Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
+
+ Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
+ upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
+ node: a string representing the cloud element currently being assessed
+ potentialMCCList: a list of dictionaries representing the possible MCCs within a path
+ whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
+
+ Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
+ index: an integer representing the location in the potentialMCCList where thisFlag occurs
+
+ '''
+ thisFlag = False
+ index = -1
+ checkList =""
+ if whichList == 1:
+ checkList = "possMCCList"
+ elif whichList ==2:
+ checkList = "fullMCSMCC"
+
+ #check parents
+ if upOrDown == 1:
+ for aNode in prunedGraph.predecessors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ #get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
+ return thisFlag, index
+
+ #check children
+ if upOrDown == 2:
+ for aNode in prunedGraph.successors(node):
+ #reset the index counter for this node search through potentialMCCList
+ index = -1
+ for MCCDict in potentialMCCList:
+ index += 1
+
+ if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
+ thisFlag = True
+ return thisFlag, index
+
+ return thisFlag, index
+#******************************************************************
+def maxExtentAndEccentricity(eachList):
+ '''
+ Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
+
+ Input:: eachList: a list of strings representing the node of the possible MCCs within a path
+
+ Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
+ definiteMCCFlag: a boolean indicating that the MCC has met all requirements
+ '''
+ maxShieldNode =''
+ maxShieldArea = 0.0
+ maxShieldEccentricity = 0.0
+ definiteMCCFlag = False
+
+ if eachList:
+ for eachNode in eachList:
+ if (thisDict(eachNode)['nodeMCSIdentifier'] == 'M' or thisDict(eachNode)['nodeMCSIdentifier'] == 'D') and thisDict(eachNode)['cloudElementArea'] > maxShieldArea:
+ maxShieldNode = eachNode
+ maxShieldArea = thisDict(eachNode)['cloudElementArea']
+
+ maxShieldEccentricity = thisDict(maxShieldNode)['cloudElementEccentricity']
+ if thisDict(maxShieldNode)['cloudElementEccentricity'] >= ECCENTRICITY_THRESHOLD_MIN and thisDict(maxShieldNode)['cloudElementEccentricity'] <= ECCENTRICITY_THRESHOLD_MAX :
+ #criteria met
+ definiteMCCFlag = True
+
+ return maxShieldNode, definiteMCCFlag
+#******************************************************************
+def findMaxDepthAndMinPath (thisPathDistanceAndLength):
+ '''
+ Purpose:: To determine the maximum depth and min path for the headnode
+
+ Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
+ thisPathDistanceAndLength({distance}, {path})
+ {distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
+
+ Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
+ minDistanceAndMaxPath = ({distance},{path})
+ '''
+ maxPathLength = 0
+ minPath = 0
+
+ #maxPathLength for the node in question
+ maxPathLength = max(len (values) for values in thisPathDistanceAndLength[1].values())
+
+ #if the duration is shorter then the min MCS length, then don't store!
+ if maxPathLength < MIN_MCS_DURATION: #MINIMUM_DURATION :
+ minDistanceAndMaxPath = ()
+
+ #else find the min path and max depth
+ else:
+ #max path distance for the node in question
+ minPath = max(values for values in thisPathDistanceAndLength[0].values())
+
+ #check to determine the shortest path from the longest paths returned
+ for pathDistance, path in itertools.izip(thisPathDistanceAndLength[0].values(), thisPathDistanceAndLength[1].values()):
+ pathLength = len(path)
+ #if pathLength is the same as the maxPathLength, then look the pathDistance to determine if the min
+ if pathLength == maxPathLength :
+ if pathDistance <= minPath:
+ minPath = pathLength
+ #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
+ minDistanceAndMaxPath = (pathDistance, path)
+ return minDistanceAndMaxPath
+#******************************************************************
+def thisDict (thisNode):
+ '''
+ Purpose:: return dictionary from graph if node exist in tree
+
+ Input:: String - thisNode
+
+ Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ return eachdict[1]
+#******************************************************************
+def checkCriteriaB (thisCloudElementLatLon):
+ '''
+ Purpose:: Determine if criteria B is met for a CEGraph
+
+ Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
+
+ Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
+
+ '''
+ cloudElementCriteriaBLatLon=[]
+
+ frame, CEcounter = ndimage.measurements.label(thisCloudElementLatLon, structure=STRUCTURING_ELEMENT)
+ frameCEcounter=0
+ #determine min and max values in lat and lon, then use this to generate teh array from LAT,LON meshgrid
+ minLat = min(x[0] for x in thisCloudElementLatLon)
+ maxLat = max(x[0]for x in thisCloudElementLatLon)
+ minLon = min(x[1]for x in thisCloudElementLatLon)
+ maxLon = max(x[1]for x in thisCloudElementLatLon)
+
+ minLatIndex = np.argmax(LAT[:,0] == minLat)
+ maxLatIndex = np.argmax(LAT[:,0]== maxLat)
+ minLonIndex = np.argmax(LON[0,:] == minLon)
+ maxLonIndex = np.argmax(LON[0,:] == maxLon)
+
+ criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
+
+ for x in thisCloudElementLatLon:
+ #to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
+ #index given in the original array to get the indices for the new array
+ #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
+ criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
+
+ #print criteriaBframe
+ #keep only those values < TBB_MAX
+ tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
+
+ #get the actual values that the mask returned
+ criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
+
+ for index, value in maenumerate(tempMask):
+ lat_index, lon_index = index
+ criteriaB[lat_index, lon_index]=value
+
+ for count in xrange(CEcounter):
+ #[0] is time dimension. Determine the actual values from the data
+ #loc is a masked array
+ #***** returns elements down then across thus (6,4) is 6 arrays deep of size 4
+
+ loc = ndimage.find_objects(criteriaB)[0]
+ cloudElementCriteriaB =criteriaB[loc]
+
+ for index,value in np.ndenumerate(cloudElementCriteriaB):
+ if value !=0:
+ t,lat,lon = index
+ #add back on the minLatIndex and minLonIndex to find the true lat, lon values
+ lat_lon_tuple = (LAT[(lat),0], LON[0,(lon)],value)
+ cloudElementCriteriaBLatLon.append(lat_lon_tuple)
+
+ cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
+
+ return cloudElementArea, cloudElementCriteriaBLatLon
+#******************************************************************
+def hasMergesOrSplits (nodeList):
+ '''
+ Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
+ Input:: nodeList - list of nodes from a path
+ Output:: two list_vars_in_file
+ splitList list of all the nodes in the path that split
+ mergeList list of all the nodes in the path that merged
+ '''
+ mergeList=[]
+ splitList=[]
+
+ for node,numParents in PRUNED_GRAPH.in_degree(nodeList).items():
+ if numParents > 1:
+ mergeList.append(node)
+
+ for node, numChildren in PRUNED_GRAPH.out_degree(nodeList).items():
+ if numChildren > 1:
+ splitList.append(node)
+ #sort
+ splitList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+ mergeList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
+
+ return mergeList,splitList
+#******************************************************************
+def allAncestors(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for parents
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its parents
+ integer - numOfChildren: the number of parents of the node passed
+ '''
+
+ numOfParents = PRUNED_GRAPH.in_degree(aNode)
+ try:
+ if PRUNED_GRAPH.predecessors(aNode) and numOfParents <= 1:
+ path = path + PRUNED_GRAPH.predecessors(aNode)
+ thisNode = PRUNED_GRAPH.predecessors(aNode)[0]
+ return allAncestors(path,thisNode)
+ else:
+ path = path+aNode
+ return path, numOfParents
+ except:
+ return path, numOfParents
+#******************************************************************
+def allDescendants(path, aNode):
+ '''
+ Purpose:: Utility script to provide the path leading up to a nodeList
+
+ Input:: list of strings - path: the nodes in the path
+ string - aNode: a string representing a node to be checked for children
+
+ Output:: list of strings - path: the list of the nodes connected to aNode through its children
+ integer - numOfChildren: the number of children of the node passed
+ '''
+
+ numOfChildren = PRUNED_GRAPH.out_degree(aNode)
+ try:
+ if PRUNED_GRAPH.successors(aNode) and numOfChildren <= 1:
+ path = path + PRUNED_GRAPH.successors(aNode)
+ thisNode = PRUNED_GRAPH.successors(aNode)[0]
+ return allDescendants(path,thisNode)
+ else:
+ path = path + aNode
+ #i.e. PRUNED_GRAPH.predecessors(aNode) is empty
+ return path, numOfChildren
+ except:
+ #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
+ return path, numOfChildren
+#******************************************************************
+def addInfothisDict (thisNode, cloudElementArea,criteriaB):
+ '''
+ Purpose:: update original dictionary node with information
+
+ Input:: String - thisNode
+ float - cloudElementArea,
+ masked array of floats meeting the criteria - criteriaB
+
+ Output ::
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['CriteriaBArea'] = cloudElementArea
+ eachdict[1]['CriteriaBLatLon'] = criteriaB
+ return
+#******************************************************************
+def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeBehaviorIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeBehaviorIdentifier'] = nodeBehaviorIdentifier
+ return
+#******************************************************************
+def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ if not 'nodeMCSIdentifier' in eachdict[1].keys():
+ eachdict[1]['nodeMCSIdentifier'] = nodeMCSIdentifier
+ return
+#******************************************************************
+def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
+ '''
+ Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
+
+ Input:: String - thisNode
+ String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
+
+
+ Output :: None
+
+ '''
+ for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
+ if eachdict[1]['uniqueID'] == thisNode:
+ eachdict[1]['nodeMCSIdentifier'] = nodeBehaviorIdentifier
+
+ return
+#******************************************************************
+def eccentricity (cloudElementLatLon):
+ '''
+ Purpose::
+ Determines the eccentricity (shape) of contiguous boxes
+ Values tending to 1 are more circular by definition, whereas
+ values tending to 0 are more linear
+
+ Input::
+ 1 variable
+ cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
+
+ Output::
+ 1 variable
+ epsilon: a float representing the eccentricity of the matrix passed
+
+ '''
+
+ epsilon = 0.0
+
+ #loop over all lons and determine longest (non-zero) col
+ #loop over all lats and determine longest (non-zero) row
+ for latLon in cloudElementLatLon:
+ #assign a matrix to determine the legit values
+
+ nonEmptyLons = sum(sum(cloudElementLatLon)>0)
+ nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
+
+ lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
+ latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
+ epsilon = min(latEigenvalues,lonEigenvalues)
+
+ return epsilon
+#******************************************************************
+def cloudElementOverlap (currentCELatLons, previousCELatLons):
+ '''
+ Purpose::
+ Determines the percentage overlap between two list of lat-lons passed
+
+ Input::
+ 2 sorted list of tuples
+ currentCELatLons - the list of tuples for the current CE
+ previousCELatLons - the list of tuples for the other CE being considered
+
+ Output::
+ 2 variables
+ percentageOverlap - a float representing the number of overlapping lat_lon tuples
+ areaOverlap - a floating-point number representing the area overlapping
+
+ '''
+
+ latlonprev =[]
+ latloncurr = []
+ count = 0
+ percentageOverlap = 0.0
+ areaOverlap = 0.0
+
+ #remove the temperature from the tuples for currentCELatLons and previousCELatLons then check for overlap
+ latlonprev = [(x[0],x[1]) for x in previousCELatLons]
+ latloncurr = [(x[0],x[1]) for x in currentCELatLons]
+
+ #find overlap
+ count = len(list(set(latloncurr)&set(latlonprev)))
+
+ #find area overlap
+ areaOverlap = count*XRES*YRES
+
+ #find percentage
+ percentageOverlap = max(((count*1.0)/(len(latloncurr)*1.0)),((count*1.0)/(len(latlonprev)*1.0)))
+
+ return percentageOverlap, areaOverlap
+#******************************************************************
+def findCESpeed(node, MCSList):
+ '''
+ Purpose:: to determine the speed of the CEs
+ uses vector displacement delta_lat/delta_lon (y/x)
+
+ Input:: node: a string representing the CE
+ MCSList: a list of strings representing the feature
+
+ Output::
+
+ '''
+
+ delta_lon =0.0
+ delta_lat =0.0
+ CEspeed =[]
+ theSpeed = 0.0
+
+
+ theList = CLOUD_ELEMENT_GRAPH.successors(node)
+ nodeLatLon=thisDict(node)
+
+
+ for aNode in theList:
+ if aNode in MCSList:
+ #if aNode is part of the MCSList then determine distance
+ aNodeLatLon = thisDict(aNode)
+ #calculate CE speed
+ #checking the lats
+ nodeLatLon[0] += 90.0
+ aNodeLatLon[0] += 90.0
+ delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
+ #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
+
+ #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ if nodeLatLon[1] < 0.0:
+ nodeLatLon[1] += 360.0
+ if aNodeLatLon[1] <= 0.0:
+ delta_lon = aNodeLatLon[1] + 360.0
+
+ delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
+ #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
+
+ theSpeed = abs(((delta_lat/delta_lon)/(TRES*3600))) #convert to s --> m/s
+
+ CEspeed.append(theSpeed)
+ print "aNode CE speed is ", aNode, ((delta_lat/delta_lon)/(TRES*3600)), theSpeed
+
+ return min(CEspeed)
+#******************************************************************
+#
+# UTILITY SCRIPTS FOR MCCSEARCH.PY
+#
+#******************************************************************
+def maenumerate(mArray):
+ '''
+ Purpose::
+ Utility script for returning the actual values from the masked array
+ Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
+
+ Input::
+ 1 variable
+ mArray - the masked array returned from the ma.array() command
+
+
+ Output::
+ 1 variable
+ maskedValues - 3D (t,lat,lon), value of only masked values
+
+ '''
+
+ mask = ~mArray.mask.ravel()
+ #beware yield fast, but generates a type called "generate" that does not allow for array methods
+ for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
+ if maskedValue:
+ yield index
+#******************************************************************
+def createMainDirectory(mainDirStr):
+ '''
+ Purpose:: to create the main directory for storing information and
+ the subdirectories for storing information
+ Input:: mainDir: a directory for where all information generated from
+ the program are to be stored
+ Output:: None
+
+ '''
+ global MAINDIRECTORY
+
+ MAINDIRECTORY = mainDirStr
+ #if directory doesnt exist, creat it
+ if not os.path.exists(MAINDIRECTORY):
+ os.makedirs(MAINDIRECTORY)
+
+ os.chdir((MAINDIRECTORY))
+ #create the subdirectories
+ try:
+ os.makedirs('images')
+ os.makedirs('textFiles')
+ os.makedirs('MERGnetcdfCEs')
+ os.makedirs('TRMMnetcdfCEs')
+ except:
+ print "Directory exists already!!!"
+ #TODO: some nice way of prompting if it is ok to continue...or just leave
+
+ return
+#******************************************************************
+def find_nearest(thisArray,value):
+ '''
+ Purpose :: to determine the value within an array closes to
+ another value
+
+ Input ::
+ Output::
+ '''
+ idx = (np.abs(thisArray-value)).argmin()
+ return thisArray[idx]
+#******************************************************************
+def preprocessingMERG(MERGdirname):
+ '''
+ Purpose::
+ Utility script for unzipping and converting the merg*.Z files from Mirador to
+ NETCDF format. The files end up in a folder called mergNETCDF in the directory
+ where the raw MERG data is
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ Directory to the location of the raw MERG files, preferably zipped
+
+ Output::
+ none
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ 3 the files havent been unzipped
+ '''
+
+ os.chdir((MERGdirname+'/'))
+ imgFilename = ''
+
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ for files in glob.glob("*-pixel"):
+ #for files in glob.glob("*.Z"):
+ fname = os.path.splitext(files)[0]
+
+ #unzip it
+ bash_cmd = 'gunzip ' + files
+ subprocess.call(bash_cmd, shell=True)
+
+ #determine the time from the filename
+ ftime = re.search('\_(.*)\_',fname).group(1)
+
+ yy = ftime[0:4]
+ mm = ftime[4:6]
+ day = ftime[6:8]
+ hr = ftime [8:10]
+
+ #TODO: must be something more efficient!
+
+ if mm=='01':
+ mth = 'Jan'
+ if mm == '02':
+ mth = 'Feb'
+ if mm == '03':
+ mth = 'Mar'
+ if mm == '04':
+ mth = 'Apr'
+ if mm == '05':
+ mth = 'May'
+ if mm == '06':
+ mth = 'Jun'
+ if mm == '07':
+ mth = 'Jul'
+ if mm == '08':
+ mth = 'Aug'
+ if mm == '09':
+ mth = 'Sep'
+ if mm == '10':
+ mth = 'Oct'
+ if mm == '11':
+ mth = 'Nov'
+ if mm == '12':
+ mth = 'Dec'
+
+
+ subprocess.call('rm merg.ctl', shell=True)
+ subprocess.call('touch merg.ctl', shell=True)
+ replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
+ replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
+ subprocess.call(replaceExpDset, shell=True)
+ subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
+ subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
+ subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
+ subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
+ subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
+ subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
+ subprocess.call(replaceExpTdef, shell=True)
+ subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
+ subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
+ subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
+
+ #generate the lats4D command for GrADS
+ lats4D = 'lats4d -v -q -lat '+LATMIN + ' ' +LATMAX +' -lon ' +LONMIN +' ' +LONMAX +' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ #lats4D = 'lats4d -v -q -lat -40 -15 -lon 10 40 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+ #lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
+
+ gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
+ #run grads and lats4d command
+ subprocess.call(gradscmd, shell=True)
+ imgFilename = hr+'Z'+day+mth+yy+'.gif'
+ tempMaskedImages(imgFilename)
+
+ #when all the files have benn converted, mv the netcdf files
+ subprocess.call('mkdir mergNETCDF', shell=True)
+ subprocess.call('mv *.nc mergNETCDF', shell=True)
+ #mv all images
+ subprocess.call('mkdir mergImgs', shell=True)
+ subprocess.call('mv *.gif mergImgs', shell=True)
+ return
+#******************************************************************
+def postProcessingNetCDF(dataset, dirName = None):
+ '''
+
+ TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
+
+ Purpose::
+ Utility script displaying the data in generated NETCDF4 files
+ in GrADS
+ NOTE: VERY RAW AND DIRTY
+
+ Input::
+ dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
+ string: Directory to the location of the raw (MERG) files, preferably zipped
+
+ Output::
+ images in location as specfied in the code
+
+ Assumptions::
+ 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
+ have been installed on the system and the user can access
+ 2 User can write files in location where script is being called
+ '''
+
+ coreDir = os.path.dirname(MAINDIRECTORY)
+ #coreDir = os.path.dirname(os.path.abspath(__file__))
+ ImgFilename = ''
+ frameList=[]
+ fileList =[]
+ lines =[]
+ var =''
+ firstTime = True
+ printLine = 0
+ lineNum = 1
+ #Just incase the X11 server is giving problems
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ print "coreDir is ", str(coreDir)
+ prevFrameNum = 0
+
+ if dataset == 1:
+ var = 'ch4'
+ dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
+ ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
+ ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ gsFile = coreDir+"/GrADSscripts/cs2.gs"
+ sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
+ lineNum = 50
+
+ elif dataset ==2:
+ var = 'precipAcc'
+ dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
+ ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
+ ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
+ origsFile = coreDir+"/GrADSscripts/cs3.gs"
+ gsFile = coreDir+"/GrADSscripts/cs4.gs"
+ sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
+ lineNum = 10
+
+ elif dataset ==3:
+ var = 'ch4'
+ ctlTitle = 'TITLE MERG DATA'
+ ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
+ if dirName is None:
+ print "Enter directory for original files"
+ return
+ else:
+ origsFile = coreDir+"/GrADSscripts/cs1.gs"
+ sologsFile = coreDir+"/GrADSscripts/infrared.gs"
+ lineNum = 54
+
+ #sort files
+ os.chdir((dirName+'/'))
+ try:
+ os.makedirs('ctlFiles')
+ except:
+ print "ctl file folder created already"
+
+ files = filter(os.path.isfile, glob.glob("*.nc"))
+ files.sort(key=lambda x: os.path.getmtime(x))
+ for eachfile in files:
+ fullFname = os.path.splitext(eachfile)[0]
+ fnameNoExtension = fullFname.split('.nc')[0]
+ if dataset == 1 or dataset == 2:
+ frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
+ #create the ctlFile
+ ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
+ #the ctl file
+ subprocessCall = 'rm ' +ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ subprocessCall = 'touch '+ctlFile1
+ subprocess.call(subprocessCall, shell=True)
+ lineToWrite = 'echo DSET ' + dirName+'/'+fnameNoExtension+'.nc' +' >>' + ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo DTYPE netcdf >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo UNDEF 0 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo XDEF 413 LINEAR -9.984375 0.036378335 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo YDEF 412 LINEAR 5.03515625 0.036378335 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo ZDEF 01 LEVELS 1 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo TDEF 99999 linear 31aug2009 1hr >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo VARS 1 >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite ='echo '+ctlLine+' >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo ENDVARS >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+ lineToWrite = 'echo >> '+ctlFile1
+ subprocess.call(lineToWrite, shell=True)
+
+ #create plot of just that data
+ subprocessCall = 'cp '+ origsFile+' '+sologsFile
+ subprocess.call(subprocessCall, shell=True)
+
+ ImgFilename = fnameNoExtension + '.gif'
+
+ displayCmd = '\''+'d '+ var+'\''+'\n'
+ newFileCmd = '\''+'open '+ ctlFile1+'\''+'\n'
+ colorbarCmd = '\''+'run cbarn'+'\''+'\n'
+ printimCmd = '\''+'prin
<TRUNCATED>
[36/50] [abbrv] remove unnecessary files
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/newPreprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/newPreprocessingMERG.py b/mccsearch/code/newPreprocessingMERG.py
deleted file mode 100755
index cedc155..0000000
--- a/mccsearch/code/newPreprocessingMERG.py
+++ /dev/null
@@ -1,175 +0,0 @@
-
-'''
- The scripts for processing MERG raw data
- #******************************************************************
- '''
-
-import fileinput
-import glob
-import os
-import re
-import string
-import subprocess
-import sys
-
-#----------------------- GLOBAL VARIABLES --------------------------
-# --------------------- User defined variables ---------------------
-LATMIN = '-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
-LATMAX = '20.0' #max latitude; -ve values in the SH e.g. 5S = -5
-LONMIN = '-30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
-LONMAX = '30.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8
-T_BB_MAX = '250' #warmest temp to allow in K
-T_BB_MIN = '180' #coolest temp to allow in K
-#------------------- End user defined Variables -------------------
-
-#------------------------ End GLOBAL VARS -------------------------
-
-def preprocessingMERG(MERGdirname):
- '''
- Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format. The files end up in a folder called mergNETCDF in the directory
- where the raw MERG data is
- NOTE: VERY RAW AND DIRTY
-
- Input::
- Directory to the location of the raw MERG files, preferably zipped
-
- Output::
- none
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
-
- os.chdir((MERGdirname+'/'))
- imgFilename = ''
-
- #Just incase the X11 server is giving problems
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- #for files in glob.glob("*-pixel"):
- for files in glob.glob("*.Z"):
- fname = os.path.splitext(files)[0]
-
- #unzip it
- bash_cmd = 'gunzip ' + files
- subprocess.call(bash_cmd, shell=True)
-
- #determine the time from the filename
- ftime = re.search('\_(.*)\_',fname).group(1)
-
- yy = ftime[0:4]
- mm = ftime[4:6]
- day = ftime[6:8]
- hr = ftime [8:10]
-
- #TODO: must be something more pythonic!
-
- if mm=='01':
- mth = 'Jan'
- if mm == '02':
- mth = 'Feb'
- if mm == '03':
- mth = 'Mar'
- if mm == '04':
- mth = 'Apr'
- if mm == '05':
- mth = 'May'
- if mm == '06':
- mth = 'Jun'
- if mm == '07':
- mth = 'Jul'
- if mm == '08':
- mth = 'Aug'
- if mm == '09':
- mth = 'Sep'
- if mm == '10':
- mth = 'Oct'
- if mm == '11':
- mth = 'Nov'
- if mm == '12':
- mth = 'Dec'
-
-
- subprocess.call('rm merg.ctl', shell=True)
- subprocess.call('touch merg.ctl', shell=True)
- replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
- replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
- subprocess.call(replaceExpDset, shell=True)
- subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
- subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
- subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
- subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
- subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
- subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
- subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
-
- #generate the lats4D command for GrADS
- lats4D = 'lats4d -v -q -lat ' + LATMIN +' ' + LATMAX + ' ' + '-lon ' + LONMIN + ' ' + LONMAX + ' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- gradscmd = 'grads -lbc ' + '\'' +lats4D + '\''
- #run grads and lats4d command
- subprocess.call(gradscmd, shell=True)
-
- #----- Generate GrADS images--------------
- imgFilename = hr+'Z'+day+mth+yy+'.gif'
- tempMaskedImages(imgFilename)
- #-------End Generate GrADS images ---------
-
- #remove the raw data file to save some resources
- rmCmd= "rm " + fname
- subprocess.call(rmCmd, shell=True)
-
- #when all the files have been converted, mv the netcdf files
- subprocess.call('mkdir mergNETCDF', shell=True)
- subprocess.call('mv *.nc mergNETCDF', shell=True)
- #mv all images
- subprocess.call('mkdir mergImgs', shell=True)
- subprocess.call('mv *.gif mergImgs', shell=True)
-
- return
-
-#******************************************************************
-def tempMaskedImages(imgFilename):
- '''
- Purpose:: To generate temperature-masked images for a first pass verification
-
- Input::
- filename for the gif file
-
- Output::
- None - gif images for each file of IR temperature < T_BB_MAX are generated in folder called mergImgs
-
- Assumptions::
- Same as for preprocessingMERG
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
-
- subprocess.call('rm tempMaskedImages.gs', shell=True)
- subprocess.call('touch tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'open merg.ctl''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set mpdset hires''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set lat ' +LATMIN + ' ' +LATMAX +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set lon ' +LONMIN + ' ' +LONMAX +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set cint 10''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set cmax '+T_BB_MAX +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'set cmin '+T_BB_MIN +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'d ch4+75''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'draw title Masked Temp @ '+imgFilename +'\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'printim '+imgFilename +' x1000 y800''\'" >> tempMaskedImages.gs', shell=True)
- subprocess.call('echo "''\'quit''\'" >> tempMaskedImages.gs', shell=True)
- gradscmd = 'grads -blc ' + '\'run tempMaskedImages.gs''\''
- subprocess.call(gradscmd, shell=True)
- return
-
-#******************************************************************
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/preprocessingMERG.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/preprocessingMERG.py b/mccsearch/code/preprocessingMERG.py
deleted file mode 100755
index 04964a8..0000000
--- a/mccsearch/code/preprocessingMERG.py
+++ /dev/null
@@ -1,125 +0,0 @@
-'''
-#preprocessing for MERG data
-# unzips files and generates the NETCDF files using GrADS lats4D
-# http://opengrads.org/doc/scripts/lats4d/
-# This script requires the merg_template.ctl resides with the folder
-# TODO: generate this file
-# Kim Whitehall
-# Last updated: June 4th 2013
-'''
-import subprocess
-import fileinput
-import json
-import os
-import glob
-import re
-import sys
-
-
-def preprocessing_merg():
- '''
- Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format
- NOTE: VERY RAW AND DIRTY
- Input::
- none
-
- Output::
- none
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
- os.chdir("/Users/whitehal/Documents/kimSum2013/mergData1/")
- searchExpDset = "DSET "
- searchExpTdef = "TDEF "
- #for files in glob.glob("*-pixel"):
- for files in glob.glob("*.Z"):
- fname = os.path.splitext(files)[0]
-
- #unzip it
- bash_cmd = 'gunzip ' + files
- subprocess.call(bash_cmd, shell=True)
-
- #determine the time from the filename
- ftime = re.search('\_(.*)\_',fname).group(1)
-
- yy = ftime[0:4]
- mm = ftime[4:6]
- day = ftime[6:8]
- hr = ftime [8:10]
-
-
- if mm=='01':
- mth = 'Jan'
- if mm == '02':
- mth = 'Feb'
- if mm == '03':
- mth = 'Mar'
- if mm == '04':
- mth = 'Apr'
- if mm == '05':
- mth = 'May'
- if mm == '06':
- mth = 'Jun'
- if mm == '07':
- mth = 'Jul'
- if mm == '08':
- mth = 'Aug'
- if mm == '09':
- mth = 'Sep'
- if mm == '10':
- mth = 'Oct'
- if mm == '11':
- mth = 'Nov'
- if mm == '12':
- mth = 'Dec'
-
-
- subprocess.call('rm merg.ctl', shell=True)
- subprocess.call('touch merg.ctl', shell=True)
- replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
- replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
- subprocess.call(replaceExpDset, shell=True)
- subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
- subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
- subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
- subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
- subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
- subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
- subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
-
-
-
- # #cp template merg.ctl
- # subprocess.call('cp -f merg_template.ctl merg.ctl', shell=True)
-
- # #edit merg.ctl dset line to contain filename by running file
- # #http://stackoverflow.com/questions/39086/search-and-replace-a-line-in-a-file-in-python
- # replaceExpDset = 'DSET ' + fname
- # replaceExpTdef = 'TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn'
- # for line in fileinput.input("merg.ctl", inplace=True):
- # if searchExpDset in line:
- # line = line.replace(searchExpDset, replaceExpDset)
- # if searchExpTdef in line:
- # line = line.replace(searchExpTdef, replaceExpTdef)
- # sys.stdout.write(line)
-
- #lats4d cmd
- lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
- #run grads and lats4d command
- subprocess.call(gradscmd, shell=True)
-
- #when all the files have benn converted, mv the netcdf files
- subprocess.call('mkdir mergNETCDF', shell=True)
- subprocess.call('mv *.nc mergNETCDF', shell=True)
- return
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/process.pyc
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.pyc b/mccsearch/code/process.pyc
deleted file mode 100644
index d0133fc..0000000
Binary files a/mccsearch/code/process.pyc and /dev/null differ
[21/50] [abbrv] git commit: Smarten up documentation for mccsearch
branch and move it into a docs directory
Posted by le...@apache.org.
Smarten up documentation for mccsearch branch and move it into a docs directory
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/5070312c
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/5070312c
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/5070312c
Branch: refs/heads/master
Commit: 5070312cb1eac272c515b0df1bb46c597421ba61
Parents: 73f8208
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 10:57:52 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Mon Aug 25 10:57:52 2014 -0700
----------------------------------------------------------------------
mccsearch/README.docx | Bin 192118 -> 0 bytes
mccsearch/README.md | 3 +++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/5070312c/mccsearch/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/README.docx b/mccsearch/README.docx
deleted file mode 100644
index 648637e..0000000
Binary files a/mccsearch/README.docx and /dev/null differ
http://git-wip-us.apache.org/repos/asf/climate/blob/5070312c/mccsearch/README.md
----------------------------------------------------------------------
diff --git a/mccsearch/README.md b/mccsearch/README.md
index e69de29..6ee45e9 100644
--- a/mccsearch/README.md
+++ b/mccsearch/README.md
@@ -0,0 +1,3 @@
+#Graph-based Search for the Identification and Chatacterization of Mesoscale Convective Complexes
+
+Please see /docs directory for all current documentation relating to this module.
\ No newline at end of file
[48/50] [abbrv] git commit: fixed bug regarding ability to access a
subset of files in a folder
Posted by le...@apache.org.
fixed bug regarding ability to access a subset of files in a folder
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/1f2948d8
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/1f2948d8
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/1f2948d8
Branch: refs/heads/master
Commit: 1f2948d863875ad411ddeaba76a42c5a1b3402cc
Parents: 345d234
Author: georgette <k_...@yahoo.com>
Authored: Tue Jun 3 09:37:52 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/.DS_Store | Bin 0 -> 6148 bytes
mccsearch/GrADsScripts/.DS_Store | Bin 0 -> 6148 bytes
mccsearch/code/.DS_Store | Bin 0 -> 6148 bytes
mccsearch/code/mccSearch.py | 17 ++++++++++++-----
mccsearch/code/mccSearchUI.py | 10 +++++-----
5 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/1f2948d8/mccsearch/.DS_Store
----------------------------------------------------------------------
diff --git a/mccsearch/.DS_Store b/mccsearch/.DS_Store
new file mode 100644
index 0000000..9cc3ce1
Binary files /dev/null and b/mccsearch/.DS_Store differ
http://git-wip-us.apache.org/repos/asf/climate/blob/1f2948d8/mccsearch/GrADsScripts/.DS_Store
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/.DS_Store b/mccsearch/GrADsScripts/.DS_Store
new file mode 100644
index 0000000..5008ddf
Binary files /dev/null and b/mccsearch/GrADsScripts/.DS_Store differ
http://git-wip-us.apache.org/repos/asf/climate/blob/1f2948d8/mccsearch/code/.DS_Store
----------------------------------------------------------------------
diff --git a/mccsearch/code/.DS_Store b/mccsearch/code/.DS_Store
new file mode 100644
index 0000000..5008ddf
Binary files /dev/null and b/mccsearch/code/.DS_Store differ
http://git-wip-us.apache.org/repos/asf/climate/blob/1f2948d8/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py
index 8c2c801..b7dc4eb 100644
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@ -77,13 +77,14 @@ PRUNED_GRAPH = nx.DiGraph()
#------------------------ End GLOBAL VARS -------------------------
#************************ Begin Functions *************************
#******************************************************************
-def readMergData(dirname):
+def readMergData(dirname, filelist = None):
'''
Purpose::
Read MERG data into RCMES format
Input::
- Directory to the MERG files in NETCDF format
+ dirname: a string representing the directory to the MERG files in NETCDF format
+ filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
Output::
A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
@@ -107,7 +108,8 @@ def readMergData(dirname):
mergLonVarName = 'longitude'
filelistInstructions = dirname + '/*'
- filelist = glob.glob(filelistInstructions)
+ if filelist == None:
+ filelist = glob.glob(filelistInstructions)
#sat_img is the array that will contain all the masked frames
@@ -1893,6 +1895,7 @@ def checkForFiles(startTime, endTime, thisDir, fileType):
status: a boolean representing whether all files exists
'''
+ filelist =[]
startFilename = ''
endFilename =''
currFilename = ''
@@ -1951,10 +1954,14 @@ def checkForFiles(startTime, endTime, thisDir, fileType):
currFilename = thisDir+"/"+startFilename
while currFilename is not endFilename:
+
if not os.path.isfile(currFilename):
print "file is missing! Filename: ", currFilename
status = False
- return status
+ return status, filelist
+ else:
+ #create filelist
+ filelist.append(currFilename)
status = True
if currFilename == endFilename:
@@ -1973,7 +1980,7 @@ def checkForFiles(startTime, endTime, thisDir, fileType):
if fileType == 2:
currFilename = thisDir+"/"+"3B42."+str(curryr)+currmmStr+currddStr+"."+currhrStr+".7A.nc"
- return status
+ return status,filelist
#******************************************************************
def findTime(curryr, currmm, currdd, currhr):
'''
http://git-wip-us.apache.org/repos/asf/climate/blob/1f2948d8/mccsearch/code/mccSearchUI.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchUI.py b/mccsearch/code/mccSearchUI.py
index 9dbb727..45dc73a 100644
--- a/mccsearch/code/mccSearchUI.py
+++ b/mccsearch/code/mccSearchUI.py
@@ -85,13 +85,13 @@ def main():
endDateTime = raw_input("> Please enter the end date and time yyyymmddhr: \n")
#check if all the files exisits in the MERG and TRMM directories entered
- test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['TRMMdirName'], 2)
+ test, _ = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['TRMMdirName'], 2)
if test == False:
- print "Error with files in the original MERG directory entered. Please check your files before restarting. "
+ print "Error with files in the original TRMM directory entered. Please check your files before restarting. "
return
- test = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['CEoriDirName'],1)
+ test, filelist = mccSearch.checkForFiles(startDateTime, endDateTime, DIRS['CEoriDirName'],1)
if test == False:
- print "Error with files in the original TRMM directory entered. Please check your files before restarting. "
+ print "Error with files in the original MERG directory entered. Please check your files before restarting. "
return
#create main directory and file structure for storing intel
@@ -116,7 +116,7 @@ def main():
print "\t\t Starting the MCCSearch Analysis "
print ("-"*80)
print "\n -------------- Reading MERG and TRMM Data ----------"
- mergImgs, timeList = mccSearch.readMergData(DIRS['CEoriDirName'])
+ mergImgs, timeList = mccSearch.readMergData(DIRS['CEoriDirName'], filelist)
print "\n -------------- findCloudElements ----------"
CEGraph = mccSearch.findCloudElements(mergImgs,timeList,DIRS['TRMMdirName'])
#if the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
[44/50] [abbrv] git commit: mccSearchUI.py
Posted by le...@apache.org.
mccSearchUI.py
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/345d234b
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/345d234b
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/345d234b
Branch: refs/heads/master
Commit: 345d234bd7370a5e2449af72bc5ceb3a40216a6a
Parents: fce6fec
Author: kwhitehall <k_...@yahoo.com>
Authored: Thu May 15 15:04:28 2014 -0400
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/GrADsScripts/TRMMCE.gs | 15 -----
mccsearch/GrADsScripts/cs2.gs | 104 --------------------------------
mccsearch/GrADsScripts/cs4.gs | 27 ---------
mccsearch/GrADsScripts/infrared.gs | 88 ---------------------------
mccsearch/GrADsScripts/mergeCE.gs | 88 ---------------------------
5 files changed, 322 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/345d234b/mccsearch/GrADsScripts/TRMMCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/TRMMCE.gs b/mccsearch/GrADsScripts/TRMMCE.gs
deleted file mode 100644
index cd1dbd6..0000000
--- a/mccsearch/GrADsScripts/TRMMCE.gs
+++ /dev/null
@@ -1,15 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/20090831000000F1CE1.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-'set datawarn off'
-'set gxout shaded on'
-'set csmooth on'
-
-'d precipAcc'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/20090831000000F1CE1.gif x800 y600 white'
-'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/345d234b/mccsearch/GrADsScripts/cs2.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs2.gs b/mccsearch/GrADsScripts/cs2.gs
deleted file mode 100644
index 9edcf5a..0000000
--- a/mccsearch/GrADsScripts/cs2.gs
+++ /dev/null
@@ -1,104 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE1.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE2.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE3.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE4.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE5.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE6.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE7.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE8.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE9.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-*'set lat 12.0234 16.8594'
-*'set lon -7.98438 7.91406'
-
-*set colors taken from colormap.gs
-*Blues
- 'set rgb 16 255 255 255'
- 'set rgb 17 108 20 156'
- 'set rgb 18 77 50 183'
- 'set rgb 19 48 83 213'
- 'set rgb 20 22 107 236'
- 'set rgb 21 0 193 254'
- 'set rgb 22 42 166 255'
- 'set rgb 23 66 197 249'
- 'set rgb 24 92 226 255'
- 'set rgb 25 124 255 249'
-
-*Greens
- 'set rgb 26 132 252 204'
- 'set rgb 27 135 252 145'
- 'set rgb 28 151 255 130'
- 'set rgb 29 209 255 128'
-
-*Yellows/Reds
- 'set rgb 30 255 246 117'
- 'set rgb 31 255 189 58'
- 'set rgb 32 249 136 6'
- 'set rgb 33 241 110 0'
- 'set rgb 34 212 93 1'
- 'set rgb 35 208 68 0'
- 'set rgb 36 182 48 10'
- 'set rgb 37 163 29 2'
- 'set rgb 38 138 15 0'
- 'set rgb 39 255 255 255'
-
-
- set_plot(198,312,5)
- say '-------------------------------'
- say 'Color Scale set to Color Infrared 1.'
- say '-------------------------------'
-
-*'set gxout shaded on'
-
-*'run cbarn'
-
-'d ch4.9'
-'d ch4.8'
-'d ch4.7'
-'d ch4.6'
-'d ch4.5'
-'d ch4.4'
-'d ch4.3'
-'d ch4.2'
-'d ch4.1'
-
-
-
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-08-31_23:00:00F7.gif x800 y600 white'
-'quit'
-
-
-
-
-
-function set_plot(min,max,int)
-
- value = min
- cval=16
- c_levs = ''
- c_cols = ''
-
- while( value <= max )
- c_levs = c_levs ' ' value
- c_cols = c_cols ' ' cval
- value = value + int
- cval=cval+1
- endwhile
- c_cols=c_cols' 'cval-1
-
- say '-------------------------------'
- say 'Contour levels set to: 'c_levs
- say 'Color Values set to: 'c_cols
- say '-------------------------------'
-
- 'set clevs 'c_levs
- 'set ccols 'c_cols
-
-return
-
http://git-wip-us.apache.org/repos/asf/climate/blob/345d234b/mccsearch/GrADsScripts/cs4.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs4.gs b/mccsearch/GrADsScripts/cs4.gs
deleted file mode 100644
index e6396f7..0000000
--- a/mccsearch/GrADsScripts/cs4.gs
+++ /dev/null
@@ -1,27 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE1.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE2.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE3.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE4.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE5.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE6.ctl'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE7.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-'set datawarn off'
-'set gxout shaded on'
-'set csmooth on'
-
-'d precipAcc.7'
-'d precipAcc.6'
-'d precipAcc.5'
-'d precipAcc.4'
-'d precipAcc.3'
-'d precipAcc.2'
-'d precipAcc.1'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/TRMM2009-09-01_23:00:00F48.gif x800 y600 white'
-'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/345d234b/mccsearch/GrADsScripts/infrared.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/infrared.gs b/mccsearch/GrADsScripts/infrared.gs
deleted file mode 100644
index 5c22b96..0000000
--- a/mccsearch/GrADsScripts/infrared.gs
+++ /dev/null
@@ -1,88 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF/ctlFiles/merg_2009090123_4km-pixel.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-*'set lat 12.0234 16.8594'
-*'set lon -7.98438 7.91406'
-
-*set colors taken from colormap.gs
-*Blues
- 'set rgb 16 255 255 255'
- 'set rgb 17 108 20 156'
- 'set rgb 18 77 50 183'
- 'set rgb 19 48 83 213'
- 'set rgb 20 22 107 236'
- 'set rgb 21 0 193 254'
- 'set rgb 22 42 166 255'
- 'set rgb 23 66 197 249'
- 'set rgb 24 92 226 255'
- 'set rgb 25 124 255 249'
-
-*Greens
- 'set rgb 26 132 252 204'
- 'set rgb 27 135 252 145'
- 'set rgb 28 151 255 130'
- 'set rgb 29 209 255 128'
-
-*Yellows/Reds
- 'set rgb 30 255 246 117'
- 'set rgb 31 255 189 58'
- 'set rgb 32 249 136 6'
- 'set rgb 33 241 110 0'
- 'set rgb 34 212 93 1'
- 'set rgb 35 208 68 0'
- 'set rgb 36 182 48 10'
- 'set rgb 37 163 29 2'
- 'set rgb 38 138 15 0'
- 'set rgb 39 255 255 255'
-
-
- set_plot(198,312,5)
- say '-------------------------------'
- say 'Color Scale set to Color Infrared 1.'
- say '-------------------------------'
-
-*'set gxout shaded on'
-
-*'run cbarn'
-
-
-
-
-
-'d ch4'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/merg_2009090123_4km-pixel.gif x800 y600 white'
-'quit'
-
-
-
-
-function set_plot(min,max,int)
-
- value = min
- cval=16
- c_levs = ''
- c_cols = ''
-
- while( value <= max )
- c_levs = c_levs ' ' value
- c_cols = c_cols ' ' cval
- value = value + int
- cval=cval+1
- endwhile
- c_cols=c_cols' 'cval-1
-
- say '-------------------------------'
- say 'Contour levels set to: 'c_levs
- say 'Color Values set to: 'c_cols
- say '-------------------------------'
-
- 'set clevs 'c_levs
- 'set ccols 'c_cols
-
-return
-
http://git-wip-us.apache.org/repos/asf/climate/blob/345d234b/mccsearch/GrADsScripts/mergeCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/mergeCE.gs b/mccsearch/GrADsScripts/mergeCE.gs
deleted file mode 100644
index b89945c..0000000
--- a/mccsearch/GrADsScripts/mergeCE.gs
+++ /dev/null
@@ -1,88 +0,0 @@
-'reinit'
-'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-09-01_00:00:00F8CE5.ctl'
-'set grads off'
-'set mpdset hires'
-'set gxout shaded'
-'set csmooth on'
-'set gxout shaded on'
-*'set lat 12.0234 16.8594'
-*'set lon -7.98438 7.91406'
-
-*set colors taken from colormap.gs
-*Blues
- 'set rgb 16 255 255 255'
- 'set rgb 17 108 20 156'
- 'set rgb 18 77 50 183'
- 'set rgb 19 48 83 213'
- 'set rgb 20 22 107 236'
- 'set rgb 21 0 193 254'
- 'set rgb 22 42 166 255'
- 'set rgb 23 66 197 249'
- 'set rgb 24 92 226 255'
- 'set rgb 25 124 255 249'
-
-*Greens
- 'set rgb 26 132 252 204'
- 'set rgb 27 135 252 145'
- 'set rgb 28 151 255 130'
- 'set rgb 29 209 255 128'
-
-*Yellows/Reds
- 'set rgb 30 255 246 117'
- 'set rgb 31 255 189 58'
- 'set rgb 32 249 136 6'
- 'set rgb 33 241 110 0'
- 'set rgb 34 212 93 1'
- 'set rgb 35 208 68 0'
- 'set rgb 36 182 48 10'
- 'set rgb 37 163 29 2'
- 'set rgb 38 138 15 0'
- 'set rgb 39 255 255 255'
-
-
- set_plot(198,312,5)
- say '-------------------------------'
- say 'Color Scale set to Color Infrared 1.'
- say '-------------------------------'
-
-*'set gxout shaded on'
-
-*'run cbarn'
-
-'d ch4'
-'run cbarn'
-'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-09-01_00:00:00F8CE5.gif x800 y600 white'
-'quit'
-
-
-
-
-
-
-
-
-function set_plot(min,max,int)
-
- value = min
- cval=16
- c_levs = ''
- c_cols = ''
-
- while( value <= max )
- c_levs = c_levs ' ' value
- c_cols = c_cols ' ' cval
- value = value + int
- cval=cval+1
- endwhile
- c_cols=c_cols' 'cval-1
-
- say '-------------------------------'
- say 'Contour levels set to: 'c_levs
- say 'Color Values set to: 'c_cols
- say '-------------------------------'
-
- 'set clevs 'c_levs
- 'set ccols 'c_cols
-
-return
-
[39/50] [abbrv] git commit: Move docsgit status
Posted by le...@apache.org.
Move docsgit status
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/37d837f2
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/37d837f2
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/37d837f2
Branch: refs/heads/master
Commit: 37d837f2edbab90bd71848f4f73e56a8315c6da2
Parents: ce04f72
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Mon Aug 25 10:56:57 2014 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Wed Oct 15 15:18:05 2014 -0700
----------------------------------------------------------------------
mccsearch/README.md | 0
mccsearch/docs/README.docx | Bin 0 -> 192118 bytes
mccsearch/docs/mccsearch.md | 58 +++++++++++++++++++++++++++++
mccsearch/docs/mccsearch_workflow.png | Bin 0 -> 132430 bytes
4 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/37d837f2/mccsearch/README.md
----------------------------------------------------------------------
diff --git a/mccsearch/README.md b/mccsearch/README.md
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/climate/blob/37d837f2/mccsearch/docs/README.docx
----------------------------------------------------------------------
diff --git a/mccsearch/docs/README.docx b/mccsearch/docs/README.docx
new file mode 100644
index 0000000..648637e
Binary files /dev/null and b/mccsearch/docs/README.docx differ
http://git-wip-us.apache.org/repos/asf/climate/blob/37d837f2/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
new file mode 100644
index 0000000..4948228
--- /dev/null
+++ b/mccsearch/docs/mccsearch.md
@@ -0,0 +1,58 @@
+#Graph-based Search for the Identification and Chatacterization of Mesoscale Convective Complexes
+
+##Very Brief Introduction
+If you are in this file it means that you wish to use the mccSearch.py program, or learn more about it.
+
+##What is this project about?
+This program searches for a weather feature known as a Mesoscale Convective Complex (MCC) in gridded infrared and precipitation rate satellite data (MERG and TRMM datasets have been tested).
+The data is read from netCDF files into arrays with the dimensions time, latitude, longitude, value.
+
+##What is needed?
+ * Python 2.7.4 (We used Anaconda 1.5.1 64 bit libraries, which installed most of the dependencies.) Other module dependences:
+ * Nio
+ * netCDF
+ * sciPy
+ * NumPy
+ * Networkx
+ * matplotlib
+ * GrADS (We used OpenGrADS grads2 Version 2.0.1.oga.1)
+ * LATS4D
+
+##Download the source code and store in a folder
+ (1) mccSearch.py contains all the function needed
+ (2) mccSearchUI.py contains a wizard type Q&A for running the program
+ (3) process.py contains some needed functions (from older version of Apache OCW)
+ (4) file.py contains some needed functions (from older version of Apache OCW)
+ (5) mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
+ * mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be store
+ * TRMMdirName : this is the directory where the original TRMM data in netCDF format is stored
+ * CEoriDirName : this is the directory where the original MERG data in netCDF format is stored
+ (6) Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
+
+##Download data in one folder (with only the data files)
+This is a very crude program. The following assumptions are made:
+ (1) input data are in one folder. For MERG data this is CEoriDirName and for the TRMM data this is TRMMdirName in mainProg.py. These directories cannot be the same.
+ (2) THERE IS NO FILE CHECKING. So please ensure ALL your files are there in netCDF format.
+ (3) THERE IS NO FILE ERROR HANDLING. Please ensure that the MERG data and the TRMM data files are correlated temporally and spatially
+
+##Run mccSearchUI.py
+As a first try to determine the workflow, run the mccSearchUI.py wizard.
+
+##Configure mainProg.py
+Guidance on the workflow can be seen in the figure below. More details on individual functions in mccSearch.py can be found in the DocStrings.
+
+
+The general workflow of the program. The dashed lines indicate optional paths.
+
+##Run mainProg.py
+Keep your fingers & toes crossed!! Once everything went well, the directory you indicated where outputs should be stored will be generated, and four folders should appear in it.
+ * The image folder will store all images generated from plots.
+ * The textFiles folder will store all the text files generated during the run, e.g. cloudElementsUserFile.txt that contains information about each cloud element identified
+ * The MERGnetcdfCEs folder contains the infrared data in clipped netCDF files that have been generated for each cloud element identified
+ * The TRMMnetcdfCEs folder contains the precipitation data in clipped netCDF files that have been generated for each cloud element identified.
+
+##Anticipated future work
+ * Implement functionality for file checking to ensure all files are there
+ * Create a user interface based on mainProg.py.
+ * Plug the project into the Regional Climate Model Evaluation Database (RCMED) that is available at Apache OCW.
+With increased usage, it is anticipated that more metrics and visualizations will be provided.
http://git-wip-us.apache.org/repos/asf/climate/blob/37d837f2/mccsearch/docs/mccsearch_workflow.png
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch_workflow.png b/mccsearch/docs/mccsearch_workflow.png
new file mode 100644
index 0000000..42a2680
Binary files /dev/null and b/mccsearch/docs/mccsearch_workflow.png differ
[06/50] [abbrv] git commit: initial mccsearch code
Posted by le...@apache.org.
initial mccsearch code
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/5f3a8521
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/5f3a8521
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/5f3a8521
Branch: refs/heads/master
Commit: 5f3a85211af19cf34cdd653f46f8745938c432b1
Parents: 4fb8d50
Author: kwhitehall <k_...@yahoo.com>
Authored: Mon Apr 7 18:14:54 2014 -0400
Committer: kwhitehall <k_...@yahoo.com>
Committed: Mon Apr 7 18:14:54 2014 -0400
----------------------------------------------------------------------
mccsearch/GrADsScripts/TRMMCE.gs | 15 +
mccsearch/GrADsScripts/accuTRMM.gs | 11 +
mccsearch/GrADsScripts/c3.gs | 11 +
.../GrADsScripts/cloudElementVars_template | 12 +
.../GrADsScripts/cloudElementVars_template2 | 11 +
mccsearch/GrADsScripts/cs1.gs | 83 +
mccsearch/GrADsScripts/cs2.gs | 104 +
mccsearch/GrADsScripts/cs3.gs | 10 +
mccsearch/GrADsScripts/cs4.gs | 27 +
mccsearch/GrADsScripts/infrared.gs | 88 +
mccsearch/GrADsScripts/mergeCE.gs | 88 +
mccsearch/code/files.py | 696 +++
mccsearch/code/mainProg.py | 142 +
mccsearch/code/mainProgGoyens.py | 126 +
mccsearch/code/mainProgWorks.py | 124 +
mccsearch/code/mccSearch.py | 4340 ++++++++++++++++++
mccsearch/code/mccSearchGoyens.py | 3606 +++++++++++++++
mccsearch/code/newPreprocessingMERG.py | 175 +
mccsearch/code/preprocessingMERG.py | 125 +
mccsearch/code/process.py | 1284 ++++++
20 files changed, 11078 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/TRMMCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/TRMMCE.gs b/mccsearch/GrADsScripts/TRMMCE.gs
new file mode 100644
index 0000000..cd1dbd6
--- /dev/null
+++ b/mccsearch/GrADsScripts/TRMMCE.gs
@@ -0,0 +1,15 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/20090831000000F1CE1.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
+'d precipAcc'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/20090831000000F1CE1.gif x800 y600 white'
+'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/accuTRMM.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/accuTRMM.gs b/mccsearch/GrADsScripts/accuTRMM.gs
new file mode 100644
index 0000000..01eda71
--- /dev/null
+++ b/mccsearch/GrADsScripts/accuTRMM.gs
@@ -0,0 +1,11 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+'set datawarn off'
+'draw title TRMM Accumulated Precipitation [mm]'
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/c3.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/c3.gs b/mccsearch/GrADsScripts/c3.gs
new file mode 100644
index 0000000..f19e5e5
--- /dev/null
+++ b/mccsearch/GrADsScripts/c3.gs
@@ -0,0 +1,11 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/cloudElementVars_template
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cloudElementVars_template b/mccsearch/GrADsScripts/cloudElementVars_template
new file mode 100644
index 0000000..bb572f6
--- /dev/null
+++ b/mccsearch/GrADsScripts/cloudElementVars_template
@@ -0,0 +1,12 @@
+DSET
+dtype netcdf
+undef 0
+TITLE MCC search Output Grid: Time lat lon
+XDEF 413 LINEAR -9.984375 0.036378335
+YDEF 412 LINEAR 5.03515625 0.036378335
+tdef 99999 linear 31aug2009 1hr
+zdef 01 LEVELS 1
+vars 1
+brightnesstemp=>ch4 1 t,y,x brightnesstemperature
+endvars
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/cloudElementVars_template2
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cloudElementVars_template2 b/mccsearch/GrADsScripts/cloudElementVars_template2
new file mode 100644
index 0000000..42746f5
--- /dev/null
+++ b/mccsearch/GrADsScripts/cloudElementVars_template2
@@ -0,0 +1,11 @@
+DSET
+undef 0
+TITLE MCC search Output Grid: Time lat lon
+XDEF 413 LINEAR -9.984375 0.036378335
+YDEF 412 LINEAR 5.03515625 0.036378335
+tdef 99999 linear 31aug2009 1hr
+zdef 01 LEVELS 1
+vars 1
+precipitation_Accumulation=>precipAcc 1 t,y,x precipAccu
+endvars
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/cs1.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs1.gs b/mccsearch/GrADsScripts/cs1.gs
new file mode 100644
index 0000000..8a790ed
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs1.gs
@@ -0,0 +1,83 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+
+
+
+
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/cs2.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs2.gs b/mccsearch/GrADsScripts/cs2.gs
new file mode 100644
index 0000000..9edcf5a
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs2.gs
@@ -0,0 +1,104 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE1.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE2.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE3.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE4.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE5.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE6.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE7.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE8.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-08-31_23:00:00F7CE9.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+'d ch4.9'
+'d ch4.8'
+'d ch4.7'
+'d ch4.6'
+'d ch4.5'
+'d ch4.4'
+'d ch4.3'
+'d ch4.2'
+'d ch4.1'
+
+
+
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-08-31_23:00:00F7.gif x800 y600 white'
+'quit'
+
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/cs3.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs3.gs b/mccsearch/GrADsScripts/cs3.gs
new file mode 100644
index 0000000..1c23d96
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs3.gs
@@ -0,0 +1,10 @@
+'reinit'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/cs4.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/cs4.gs b/mccsearch/GrADsScripts/cs4.gs
new file mode 100644
index 0000000..e6396f7
--- /dev/null
+++ b/mccsearch/GrADsScripts/cs4.gs
@@ -0,0 +1,27 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE1.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE2.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE3.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE4.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE5.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE6.ctl'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/TRMMnetcdfCEs/ctlFiles/TRMM2009-09-01_23:00:00F48CE7.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+'set datawarn off'
+'set gxout shaded on'
+'set csmooth on'
+
+'d precipAcc.7'
+'d precipAcc.6'
+'d precipAcc.5'
+'d precipAcc.4'
+'d precipAcc.3'
+'d precipAcc.2'
+'d precipAcc.1'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/TRMM2009-09-01_23:00:00F48.gif x800 y600 white'
+'quit'
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/infrared.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/infrared.gs b/mccsearch/GrADsScripts/infrared.gs
new file mode 100644
index 0000000..5c22b96
--- /dev/null
+++ b/mccsearch/GrADsScripts/infrared.gs
@@ -0,0 +1,88 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF/ctlFiles/merg_2009090123_4km-pixel.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+
+
+
+
+'d ch4'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/merg_2009090123_4km-pixel.gif x800 y600 white'
+'quit'
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/GrADsScripts/mergeCE.gs
----------------------------------------------------------------------
diff --git a/mccsearch/GrADsScripts/mergeCE.gs b/mccsearch/GrADsScripts/mergeCE.gs
new file mode 100644
index 0000000..b89945c
--- /dev/null
+++ b/mccsearch/GrADsScripts/mergeCE.gs
@@ -0,0 +1,88 @@
+'reinit'
+'open /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/MERGnetcdfCEs/ctlFiles/cloudElements2009-09-01_00:00:00F8CE5.ctl'
+'set grads off'
+'set mpdset hires'
+'set gxout shaded'
+'set csmooth on'
+'set gxout shaded on'
+*'set lat 12.0234 16.8594'
+*'set lon -7.98438 7.91406'
+
+*set colors taken from colormap.gs
+*Blues
+ 'set rgb 16 255 255 255'
+ 'set rgb 17 108 20 156'
+ 'set rgb 18 77 50 183'
+ 'set rgb 19 48 83 213'
+ 'set rgb 20 22 107 236'
+ 'set rgb 21 0 193 254'
+ 'set rgb 22 42 166 255'
+ 'set rgb 23 66 197 249'
+ 'set rgb 24 92 226 255'
+ 'set rgb 25 124 255 249'
+
+*Greens
+ 'set rgb 26 132 252 204'
+ 'set rgb 27 135 252 145'
+ 'set rgb 28 151 255 130'
+ 'set rgb 29 209 255 128'
+
+*Yellows/Reds
+ 'set rgb 30 255 246 117'
+ 'set rgb 31 255 189 58'
+ 'set rgb 32 249 136 6'
+ 'set rgb 33 241 110 0'
+ 'set rgb 34 212 93 1'
+ 'set rgb 35 208 68 0'
+ 'set rgb 36 182 48 10'
+ 'set rgb 37 163 29 2'
+ 'set rgb 38 138 15 0'
+ 'set rgb 39 255 255 255'
+
+
+ set_plot(198,312,5)
+ say '-------------------------------'
+ say 'Color Scale set to Color Infrared 1.'
+ say '-------------------------------'
+
+*'set gxout shaded on'
+
+*'run cbarn'
+
+'d ch4'
+'run cbarn'
+'printim /Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1/images/cloudElements2009-09-01_00:00:00F8CE5.gif x800 y600 white'
+'quit'
+
+
+
+
+
+
+
+
+function set_plot(min,max,int)
+
+ value = min
+ cval=16
+ c_levs = ''
+ c_cols = ''
+
+ while( value <= max )
+ c_levs = c_levs ' ' value
+ c_cols = c_cols ' ' cval
+ value = value + int
+ cval=cval+1
+ endwhile
+ c_cols=c_cols' 'cval-1
+
+ say '-------------------------------'
+ say 'Contour levels set to: 'c_levs
+ say 'Color Values set to: 'c_cols
+ say '-------------------------------'
+
+ 'set clevs 'c_levs
+ 'set ccols 'c_cols
+
+return
+
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/files.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/files.py b/mccsearch/code/files.py
new file mode 100644
index 0000000..82b508a
--- /dev/null
+++ b/mccsearch/code/files.py
@@ -0,0 +1,696 @@
+"""
+Module for handling data input files. Requires PyNIO and Numpy be
+installed.
+
+This module can easily open NetCDF, HDF and Grib files. Search the PyNIO
+documentation for a complete list of supported formats.
+"""
+
+from os import path
+
+try:
+ import Nio
+except ImportError:
+ import nio as Nio
+
+import numpy as np
+import numpy.ma as ma
+import sys
+import process #KDW added
+#from toolkit import process
+#from utils import fortranfile
+#from utils import misc
+
+
+VARIABLE_NAMES = {'time': ['time', 'times', 'date', 'dates', 'julian'],
+ 'latitude': ['latitude', 'lat', 'lats', 'latitudes'],
+ 'longitude': ['longitude', 'lon', 'lons', 'longitudes']
+ }
+
+
+def findunique(seq):
+ keys = {}
+ for e in seq:
+ keys[e] = 1
+ return keys.keys()
+
+def getVariableByType(filename, variableType):
+ """
+ Function that will try to return the variable from a file based on a provided
+ parameter type.
+
+ Input::
+ filename - the file to inspect
+ variableType - time | latitude | longitude
+
+ Output::
+ variable name OR list of all variables in the file if a single variable
+ name match cannot be found.
+ """
+ try:
+ f = Nio.open_file(filename)
+ except:
+ #print 'PyNio had an issue opening the filename (%s) you provided' % filename
+ print "NIOError:", sys.exc_info()[0]
+ raise
+
+ variableKeys = f.variables.keys()
+ f.close()
+ variableKeys = [variable.lower() for variable in variableKeys]
+ variableMatch = VARIABLE_NAMES[variableType]
+
+ commonVariables = list(set(variableKeys).intersection(variableMatch))
+
+ if len(commonVariables) == 1:
+ return str(commonVariables[0])
+
+ else:
+ return variableKeys
+
+def getVariableRange(filename, variableName):
+ """
+ Function to return the min and max values of the given variable in
+ the supplied filename.
+
+ Input::
+ filename - absolute path to a file
+ variableName - variable whose min and max values should be returned
+
+ Output::
+ variableRange - tuple of order (variableMin, variableMax)
+ """
+ try:
+ f = Nio.open_file(filename)
+ except:
+ #print 'PyNio had an issue opening the filename (%s) you provided' % filename
+ print "NIOError:", sys.exc_info()[0]
+ raise
+
+ varArray = f.variables[variableName][:]
+ return (varArray.min(), varArray.max())
+
+
+def read_data_from_file_list(filelist, myvar, timeVarName, latVarName, lonVarName):
+ '''
+ Read in data from a list of model files into a single data structure
+
+ Input:
+ filelist - list of filenames (including path)
+ myvar - string containing name of variable to load in (as it appears in file)
+ Output:
+ lat, lon - 2D array of latitude and longitude values
+ timestore - list of times
+ t2store - numpy array containing data from all files
+
+ NB. originally written specific for WRF netCDF output files
+ modified to make more general (Feb 2011)
+
+ Peter Lean July 2010
+ '''
+
+ filelist.sort()
+ filename = filelist[0]
+ # Crash nicely if 'filelist' is zero length
+ """TODO: Throw Error instead via try Except"""
+ if len(filelist) == 0:
+ print 'Error: no files have been passed to read_data_from_file_list()'
+ sys.exit()
+
+ # Open the first file in the list to:
+ # i) read in lats, lons
+ # ii) find out how many timesteps in the file
+ # (assume same ntimes in each file in list)
+ # -allows you to create an empty array to store variable data for all times
+ tmp = Nio.open_file(filename)
+ latsraw = tmp.variables[latVarName][:]
+ lonsraw = tmp.variables[lonVarName][:]
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+
+ """TODO: Guard against case where latsraw and lonsraw are not the same dim?"""
+
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
+
+ timesraw = tmp.variables[timeVarName]
+ ntimes = len(timesraw)
+
+ print 'Lats and lons read in for first file in filelist', ntimes
+
+ # Create a single empty masked array to store model data from all files
+ t2store = ma.zeros((ntimes * len(filelist), len(lat[:, 0]), len(lon[0, :])))
+ timestore = ma.zeros((ntimes * len(filelist)))
+
+ # Now load in the data for real
+ # NB. no need to reload in the latitudes and longitudes -assume invariant
+ i = 0
+ timesaccu = 0 # a counter for number of times stored so far in t2store
+ # NB. this method allows for missing times in data files
+ # as no assumption made that same number of times in each file...
+
+
+ for ifile in filelist:
+
+ #print 'Loading data from file: ',filelist[i]
+ f = Nio.open_file(ifile)
+ t2raw = f.variables[myvar][:]
+ timesraw = f.variables[timeVarName]
+ time = timesraw[:]
+ ntimes = len(time)
+ print 'file= ', i, 'ntimes= ', ntimes, filelist[i]
+ print '\nt2raw shape: ', t2raw.shape
+
+ # Flatten dimensions which needn't exist, i.e. level
+ # e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
+ # Code requires data to have dimensions, (time,lat,lon)
+ # i.e. remove level dimensions
+ # Remove 1d axis from the t2raw array
+ # Example: t2raw.shape == (365, 180, 360 1) <maps to (time, lat, lon, height)>
+ # After the squeeze you will be left with (365, 180, 360) instead
+ t2tmp = t2raw.squeeze()
+ # Nb. if this happens to be data for a single time only, then we just flattened it by accident
+ # lets put it back...
+ if t2tmp.ndim == 2:
+ t2tmp = np.expand_dims(t2tmp, 0)
+
+ print "\n timesaccu is: ", timesaccu, time,timesaccu + np.arange(ntimes)
+
+ t2store[timesaccu + np.arange(ntimes), :, :] = t2tmp[:, :, :]
+ timestore[timesaccu + np.arange(ntimes)] = time
+ timesaccu = timesaccu + ntimes
+ print "\n ***timesaccu is: ", timesaccu
+ f.close()
+ i += 1
+
+ print '\nData read in successfully with dimensions: ', t2store.shape, timesaccu
+
+ # TODO: search for duplicated entries (same time) and remove duplicates.
+ # Check to see if number of unique times == number of times, if so then no problem
+
+ if(len(np.unique(timestore)) != len(np.where(timestore != 0)[0].view())):
+ print '\nWARNING: Possible duplicated times'
+
+ # Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
+ timestore, _ = process.getModelTimes(filename, timeVarName)
+
+ data_dict = {}
+ data_dict['lats'] = lat
+ data_dict['lons'] = lon
+ data_dict['times'] = timestore
+ data_dict['data'] = t2store
+ #return lat, lon, timestore, t2store
+ return data_dict
+
+def select_var_from_file(myfile, fmt='not set'):
+ '''
+ Routine to act as user interface to allow users to select variable of interest from a file.
+
+ Input:
+ myfile - filename
+ fmt - (optional) specify fileformat for PyNIO if filename suffix is non-standard
+
+ Output:
+ myvar - variable name in file
+
+ Peter Lean September 2010
+ '''
+
+ print fmt
+
+ if fmt == 'not set':
+ f = Nio.open_file(myfile)
+
+ if fmt != 'not set':
+ f = Nio.open_file(myfile, format=fmt)
+
+ keylist = f.variables.keys()
+
+ i = 0
+ for v in keylist:
+ print '[', i, '] ', f.variables[v].long_name, ' (', v, ')'
+ i += 1
+
+ user_selection = raw_input('Please select variable : [0 -' + str(i - 1) + '] ')
+
+ myvar = keylist[int(user_selection)]
+
+ return myvar
+
+def select_var_from_wrf_file(myfile):
+ '''
+ Routine to act as user interface to allow users to select variable of interest from a wrf netCDF file.
+
+ Input:
+ myfile - filename
+
+ Output:
+ mywrfvar - variable name in wrf file
+
+ Peter Lean September 2010
+ '''
+
+ f = Nio.open_file(myfile, format='nc')
+
+ keylist = f.variables.keys()
+
+ i = 0
+ for v in keylist:
+ try:
+ print '[', i, '] ', f.variables[v].description, ' (', v, ')'
+ except:
+ print ''
+
+ i += 1
+
+ user_selection = raw_input('Please select WRF variable : [0 -' + str(i - 1) + '] ')
+
+ mywrfvar = keylist[int(user_selection)]
+
+ return mywrfvar
+
+def read_lolaT_from_file(filename, latVarName, lonVarName, timeVarName, file_type):
+ """
+ Function that will return lat, lon, and time arrays
+
+ Input::
+ filename - the file to inspect
+ latVarName - name of the Latitude Variable
+ lonVarName - name of the Longitude Variable
+ timeVarName - name of the Time Variable
+ fileType = type of file we are trying to parse
+
+ Output::
+ lat - MESH GRID of Latitude values with shape (nx, ny)
+ lon - MESH GRID of Longitude values with shape (nx, ny)
+ timestore - Python list of Datetime objects
+
+ MESHGRID docs: http://docs.scipy.org/doc/numpy/reference/generated/numpy.meshgrid.html
+
+ """
+
+ tmp = Nio.open_file(filename, format=file_type)
+ lonsraw = tmp.variables[lonVarName][:]
+ latsraw = tmp.variables[latVarName][:]
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw
+ lat = latsraw
+ timestore, _ = process.getModelTimes(filename, timeVarName)
+ print ' read_lolaT_from_file: Lats, lons and times read in for the model domain'
+ return lat, lon, timestore
+
+def read_data_from_one_file(ifile, myvar, timeVarName, lat, file_type):
+ ##################################################################################
+ # Read in data from one file at a time
+ # Input: filelist - list of filenames (including path)
+ # myvar - string containing name of variable to load in (as it appears in file)
+ # Output: lat, lon - 2D array of latitude and longitude values
+ # times - list of times
+ # t2store - numpy array containing data from all files
+ # Modified from read_data_from_file_list to read data from multiple models into a 4-D array
+ # 1. The code now processes model data that completely covers the 20-yr period. Thus,
+ # all model data must have the same time levels (ntimes). Unlike in the oroginal, ntimes
+ # is fixed here.
+ # 2. Because one of the model data exceeds 240 mos (243 mos), the model data must be
+ # truncated to the 240 mons using the ntimes determined from the first file.
+ ##################################################################################
+ f = Nio.open_file(ifile)
+ try:
+ varUnit = f.variables[myvar].units.upper()
+ except:
+ varUnit = raw_input('Enter the model variable unit: \n> ').upper()
+ t2raw = f.variables[myvar][:]
+ t2tmp = t2raw.squeeze()
+ if t2tmp.ndim == 2:
+ t2tmp = np.expand_dims(t2tmp, 0)
+ t2tmp = t2tmp
+ f.close()
+ print ' success read_data_from_one_file: VarName=', myvar, ' Shape(Full)= ', t2tmp.shape, ' Unit= ', varUnit
+ timestore = process.decode_model_timesK(ifile, timeVarName, file_type)
+ return timestore, t2tmp, varUnit
+
+def findTimeVariable(filename):
+ """
+ Function to find what the time variable is called in a model file.
+ Input::
+ filename - file to crack open and check for a time variable
+ Output::
+ timeName - name of the input file's time variable
+ variableNameList - list of variable names from the input filename
+ """
+ try:
+ f = Nio.open_file(filename, mode='r')
+ except:
+ print("Unable to open '%s' to try and read the Time variable" % filename)
+ raise
+
+ variableNameList = f.variables.keys()
+ # convert all variable names into lower case
+ varNameListLowerCase = [x.lower() for x in variableNameList]
+
+ # Use "set" types for finding common variable name from in the file and from the list of possibilities
+ possibleTimeNames = set(['time', 'times', 'date', 'dates', 'julian'])
+
+ # Use the sets to find the intersection where variable names are in possibleNames
+ timeNameSet = possibleTimeNames.intersection(varNameListLowerCase)
+
+ if len(timeNameSet) == 0:
+ print("Unable to autodetect the Time Variable Name in the '%s'" % filename)
+ timeName = misc.askUserForVariableName(variableNameList, targetName ="Time")
+
+ else:
+ timeName = timeNameSet.pop()
+
+ return timeName, variableNameList
+
+
+def findLatLonVarFromFile(filename):
+ """
+ Function to find what the latitude and longitude variables are called in a model file.
+
+ Input::
+ -filename
+ Output::
+ -latVarName
+ -lonVarName
+ -latMin
+ -latMax
+ -lonMin
+ -lonMax
+ """
+ try:
+ f = Nio.open_file(filename, mode='r')
+ except:
+ print("Unable to open '%s' to try and read the Latitude and Longitude variables" % filename)
+ raise
+
+ variableNameList = f.variables.keys()
+ # convert all variable names into lower case
+ varNameListLowerCase = [x.lower() for x in variableNameList]
+
+ # Use "set" types for finding common variable name from in the file and from the list of possibilities
+ possibleLatNames = set(['latitude', 'lat', 'lats', 'latitudes'])
+ possibleLonNames = set(['longitude', 'lon', 'lons', 'longitudes'])
+
+ # Use the sets to find the intersection where variable names are in possibleNames
+ latNameSet = possibleLatNames.intersection(varNameListLowerCase)
+ lonNameSet = possibleLonNames.intersection(varNameListLowerCase)
+
+ if len(latNameSet) == 0 or len(lonNameSet) == 0:
+ print("Unable to autodetect Latitude and/or Longitude values in the file")
+ latName = misc.askUserForVariableName(variableNameList, targetName ="Latitude")
+ lonName = misc.askUserForVariableName(variableNameList, targetName ="Longitude")
+
+ else:
+ latName = latNameSet.pop()
+ lonName = lonNameSet.pop()
+
+ lats = np.array(f.variables[latName][:])
+ lons = np.array(f.variables[lonName][:])
+
+ latMin = lats.min()
+ latMax = lats.max()
+
+ # Convert the lons from 0:360 into -180:180
+ lons[lons > 180] = lons[lons > 180] - 360.
+ lonMin = lons.min()
+ lonMax = lons.max()
+
+ return latName, lonName, latMin, latMax, lonMin, lonMax
+
+
+def read_data_from_file_list_K(filelist, myvar, timeVarName, latVarName, lonVarName, file_type):
+ ##################################################################################
+ # Read in data from a list of model files into a single data structure
+ # Input: filelist - list of filenames (including path)
+ # myvar - string containing name of variable to load in (as it appears in file)
+ # Output: lat, lon - 2D array of latitude and longitude values
+ # times - list of times
+ # t2store - numpy array containing data from all files
+ # Modified from read_data_from_file_list to read data from multiple models into a 4-D array
+ # 1. The code now processes model data that completely covers the 20-yr period. Thus,
+ # all model data must have the same time levels (ntimes). Unlike in the oroginal, ntimes
+ # is fixed here.
+ # 2. Because one of the model data exceeds 240 mos (243 mos), the model data must be
+ # truncated to the 240 mons using the ntimes determined from the first file.
+ ##################################################################################
+ filelist.sort()
+ nfiles = len(filelist)
+ # Crash nicely if 'filelist' is zero length
+ if nfiles == 0:
+ print 'Error: no files have been passed to read_data_from_file_list(): Exit'
+ sys.exit()
+
+ # Open the first file in the list to:
+ # i) read in lats, lons
+ # ii) find out how many timesteps in the file (assume same ntimes in each file in list)
+ # -allows you to create an empty array to store variable data for all times
+ tmp = Nio.open_file(filelist[0], format=file_type)
+ latsraw = tmp.variables[latVarName][:]
+ lonsraw = tmp.variables[lonVarName][:]
+ lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
+ if(latsraw.ndim == 1):
+ lon, lat = np.meshgrid(lonsraw, latsraw)
+ if(latsraw.ndim == 2):
+ lon = lonsraw; lat = latsraw
+
+ timesraw = tmp.variables[timeVarName]
+ ntimes = len(timesraw); nygrd = len(lat[:, 0]); nxgrd = len(lon[0, :])
+
+ print 'Lats and lons read in for first file in filelist'
+
+ # Create a single empty masked array to store model data from all files
+ #t2store = ma.zeros((ntimes*nfiles,nygrd,nxgrd))
+ t2store = ma.zeros((nfiles, ntimes, nygrd, nxgrd))
+ #timestore=ma.zeros((ntimes*nfiles))
+
+ ## Now load in the data for real
+ ## NB. no need to reload in the latitudes and longitudes -assume invariant
+ #timesaccu=0 # a counter for number of times stored so far in t2store
+ # NB. this method allows for missing times in data files
+ # as no assumption made that same number of times in each file...
+
+ i = 0
+ for ifile in filelist:
+ #print 'Loading data from file: ',filelist[i]
+ f = Nio.open_file(ifile)
+ t2raw = f.variables[myvar][:]
+ timesraw = f.variables[timeVarName]
+ #time = timesraw[0:ntimes]
+ ntimes = len(timesraw)
+ #ntimes=len(time)
+ print 'file= ',i,' ntimes= ',ntimes,filelist[i]
+ ## Flatten dimensions which needn't exist, i.e. level
+ ## e.g. if for single level then often data have 4 dimensions, when 3 dimensions will do.
+ ## Code requires data to have dimensions, (time,lat,lon)
+ ## i.e. remove level dimensions
+ t2tmp = t2raw.squeeze()
+ ## Nb. if data happen to be for a single time, we flattened it by accident; lets put it back...
+ if t2tmp.ndim == 2:
+ t2tmp = np.expand_dims(t2tmp, 0)
+ #t2store[timesaccu+np.arange(ntimes),:,:]=t2tmp[0:ntimes,:,:]
+ t2store[i, 0:(ntimes-1), :, :] = t2tmp[0:(ntimes-1), :, :]
+ #timestore[timesaccu+np.arange(ntimes)]=time
+ #timesaccu=timesaccu+ntimes
+ f.close()
+ i += 1
+
+ print 'Data read in successfully with dimensions: ', t2store.shape
+
+ # Decode model times into python datetime objects. Note: timestore becomes a list (no more an array) here
+ ifile = filelist[0]
+ timestore, _ = process.getModelTimes(ifile, timeVarName)
+
+ return lat, lon, timestore, t2store
+
+def find_latlon_ranges(filelist, lat_var_name, lon_var_name):
+ # Function to return the latitude and longitude ranges of the data in a file,
+ # given the identifying variable names.
+ #
+ # Input:
+ # filelist - list of filenames (data is read in from first file only)
+ # lat_var_name - variable name of the 'latitude' variable
+ # lon_var_name - variable name of the 'longitude' variable
+ #
+ # Output:
+ # latMin, latMax, lonMin, lonMax - self explanatory
+ #
+ # Peter Lean March 2011
+
+ filename = filelist[0]
+
+ try:
+ f = Nio.open_file(filename)
+
+ lats = f.variables[lat_var_name][:]
+ latMin = lats.min()
+ latMax = lats.max()
+
+ lons = f.variables[lon_var_name][:]
+ lons[lons > 180] = lons[lons > 180] - 360.
+ lonMin = lons.min()
+ lonMax = lons.max()
+
+ return latMin, latMax, lonMin, lonMax
+
+ except:
+ print 'Error: there was a problem with finding the latitude and longitude ranges in the file'
+ print ' Please check that you specified the filename, and variable names correctly.'
+
+ sys.exit()
+
+def writeBN_lola(fileName, lons, lats):
+ # write a binary data file that include longitude (1-d) and latitude (1-d) values
+
+ F = fortranfile.FortranFile(fileName, mode='w')
+ ngrdY = lons.shape[0]; ngrdX = lons.shape[1]
+ tmpDat = ma.zeros(ngrdX); tmpDat[:] = lons[0, :]; F.writeReals(tmpDat)
+ tmpDat = ma.zeros(ngrdY); tmpDat[:] = lats[:, 0]; F.writeReals(tmpDat)
+ # release temporary arrays
+ tmpDat = 0
+ F.close()
+
+def writeBNdata(fileName, numOBSs, numMDLs, nT, ngrdX, ngrdY, numSubRgn, obsData, mdlData, obsRgnAvg, mdlRgnAvg):
+ #(fileName,maskOption,numOBSs,numMDLs,nT,ngrdX,ngrdY,numSubRgn,obsData,mdlData,obsRgnAvg,mdlRgnAvg):
+ # write spatially- and regionally regridded data into a binary data file
+ missing = -1.e26
+ F = fortranfile.FortranFile(fileName, mode='w')
+ # construct a data array to replace mask flag with a missing value (missing=-1.e12) for printing
+ data = ma.zeros((nT, ngrdY, ngrdX))
+ for m in np.arange(numOBSs):
+ data[:, :, :] = obsData[m, :, :, :]; msk = data.mask
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ for i in np.arange(ngrdX):
+ if msk[n, j, i]: data[n, j, i] = missing
+
+ # write observed data. allowed to write only one row at a time
+ tmpDat = ma.zeros(ngrdX)
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ tmpDat[:] = data[n, j, :]
+ F.writeReals(tmpDat)
+
+ # write model data (dep. on the number of models).
+ for m in np.arange(numMDLs):
+ data[:, :, :] = mdlData[m, :, :, :]
+ msk = data.mask
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ for i in np.arange(ngrdX):
+ if msk[n, j, i]:
+ data[n, j, i] = missing
+
+ for n in np.arange(nT):
+ for j in np.arange(ngrdY):
+ tmpDat[:] = data[n, j, :]
+ F.writeReals(tmpDat)
+
+ data = 0 # release the array allocated for data
+ # write data in subregions
+ if numSubRgn > 0:
+ print 'Also included are the time series of the means over ', numSubRgn, ' areas from obs and model data'
+ tmpDat = ma.zeros(nT); print numSubRgn
+ for m in np.arange(numOBSs):
+ for n in np.arange(numSubRgn):
+ tmpDat[:] = obsRgnAvg[m, n, :]
+ F.writeReals(tmpDat)
+ for m in np.arange(numMDLs):
+ for n in np.arange(numSubRgn):
+ tmpDat[:] = mdlRgnAvg[m, n, :]
+ F.writeReals(tmpDat)
+ tmpDat = 0 # release the array allocated for tmpDat
+ F.close()
+
+def writeNCfile(fileName, numSubRgn, lons, lats, obsData, mdlData, obsRgnAvg, mdlRgnAvg, obsList, mdlList, subRegions):
+ # write an output file of variables up to 3 dimensions
+ # fileName: the name of the output data file
+ # numSubRgn : the number of subregions
+ # lons[ngrdX]: longitude
+ # lats[ngrdY]: latitudes
+ # obsData[nT,ngrdY,ngrdX]: the obs time series of the entire model domain
+ # mdlData[numMDLs,nT,ngrdY,ngrdX]: the mdltime series of the entire model domain
+ # obsRgnAvg[numSubRgn,nT]: the obs time series for the all subregions
+ # mdlRgnAvg[numMDLs,numSubRgn,nT]: the mdl time series for the all subregions
+ dimO = obsData.shape[0] # the number of obs data
+ dimM = mdlData.shape[0] # the number of mdl data
+ dimT = mdlData.shape[1] # the number of time levels
+ dimY = mdlData.shape[2] # y-dimension
+ dimX = mdlData.shape[3] # x-dimension
+ dimR = obsRgnAvg.shape[1] # the number of subregions
+ f = Nio.open_file(fileName, mode='w', format='nc')
+ print mdlRgnAvg.shape, dimM, dimR, dimT
+ #create global attributes
+ f.globalAttName = ''
+ # create dimensions
+ print 'Creating Dimensions within the NetCDF Object...'
+ f.create_dimension('unity', 1)
+ f.create_dimension('time', dimT)
+ f.create_dimension('west_east', dimX)
+ f.create_dimension('south_north', dimY)
+ f.create_dimension('obs', dimO)
+ f.create_dimension('models', dimM)
+
+ # create the variable (real*4) to be written in the file
+ print 'Creating Variables...'
+ f.create_variable('lon', 'd', ('south_north', 'west_east'))
+ f.create_variable('lat', 'd', ('south_north', 'west_east'))
+ f.create_variable('oDat', 'd', ('obs', 'time', 'south_north', 'west_east'))
+ f.create_variable('mDat', 'd', ('models', 'time', 'south_north', 'west_east'))
+
+ if subRegions:
+ f.create_dimension('regions', dimR)
+ f.create_variable('oRgn', 'd', ('obs', 'regions', 'time'))
+ f.create_variable('mRgn', 'd', ('models', 'regions', 'time'))
+ f.variables['oRgn'].varAttName = 'Observation time series: Subregions'
+ f.variables['mRgn'].varAttName = 'Model time series: Subregions'
+
+ loadDataIntoNetCDF(f, obsList, obsData, 'Observation')
+ print 'Loaded the Observations into the NetCDF'
+
+ loadDataIntoNetCDF(f, mdlList, mdlData, 'Model')
+
+ # create attributes and units for the variable
+ print 'Creating Attributes and Units...'
+ f.variables['lon'].varAttName = 'Longitudes'
+ f.variables['lon'].varUnit = 'degrees East'
+ f.variables['lat'].varAttName = 'Latitudes'
+ f.variables['lat'].varUnit = 'degrees North'
+ f.variables['oDat'].varAttName = 'Observation time series: entire domain'
+ f.variables['mDat'].varAttName = 'Model time series: entire domain'
+
+ # assign the values to the variable and write it
+ f.variables['lon'][:, :] = lons
+ f.variables['lat'][:, :] = lats
+ if subRegions:
+ f.variables['oRgn'][:, :, :] = obsRgnAvg
+ f.variables['mRgn'][:, :, :] = mdlRgnAvg
+
+ f.close()
+
+def loadDataIntoNetCDF(fileObject, datasets, dataArray, dataType):
+ """
+ Input::
+ fileObject - PyNIO file object data will be loaded into
+ datasets - List of dataset names
+ dataArray - Multi-dimensional array of data to be loaded into the NetCDF file
+ dataType - String with value of either 'Model' or 'Observation'
+ Output::
+ No return value. PyNIO file object is updated in place
+ """
+ datasetCount = 0
+ for dataset in datasets:
+ if dataType.lower() == 'observation':
+ datasetName = dataset.replace(' ','')
+ elif dataType.lower() == 'model':
+ datasetName = path.splitext(path.basename(dataset))[0]
+ print "Creating variable %s" % datasetName
+ fileObject.create_variable(datasetName, 'd', ('time', 'south_north', 'west_east'))
+ fileObject.variables[datasetName].varAttName = 'Obseration time series: entire domain'
+ print 'Loading values into %s' % datasetName
+ fileObject.variables[datasetName].assign_value(dataArray[datasetCount,:,:,:])
+ datasetCount += 1
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/mainProg.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProg.py b/mccsearch/code/mainProg.py
new file mode 100644
index 0000000..37e1cc8
--- /dev/null
+++ b/mccsearch/code/mainProg.py
@@ -0,0 +1,142 @@
+'''
+# running the program
+# Kim Whitehall Nov 2012
+# Last updated: 15 May 2013
+'''
+
+#import rcmes
+import sys
+import networkx as nx
+import mccSearch
+#import cloudclusters
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+import subprocess
+
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ subprocess.call('export DISPLAY=:0.0', shell=True)
+
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/thesis/presentation/caseStudies/BFStudy"
+ #mainDirStr= "/home/whitehal/mccsearch/caseStudy2"
+ #directories for the original data
+ #TRMMdirName ="/home/whitehal/summer2006TRMM"
+ #CEoriDirName = "/home/whitehal/17Jul30Sep/mergNETCDF"
+ TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
+ CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
+ #allMCSsDict ={{}}
+
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # sys.exit()
+
+ # # mccSearch.plotAccTRMM(TRMMCEdirName)
+ # sys.exit()
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # sys.exit()
+
+ #using merg data ***********
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ print 'timeList', timeList
+ print 'TRMMdirName ', TRMMdirName
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName isnt entered for what ever reason
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
+ # sys.exit()
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ # for eachList in MCCList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print "MCS List has been acquired ", len(MCSList)
+ # for eachList in MCSList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING TRMM ----------"
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+ #MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ print ("-"*80)
+ #print "MCCTimes is: ", MCCTimes
+ #mccSearch.plotAccTRMM(MCCList)
+ mccSearch.displayPrecip(MCCList)
+ mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ mccSearch.displaySize(MCCList)
+ mccSearch.displayPrecip(MCCList)
+ mccSearch.plotHistograms(MCCList)
+ print "creating the MCC userfile ", mccSearch.createTextFile(MCCList,1)
+ print "creating the MCS userfile ", mccSearch.createTextFile(MCSList,2)
+
+ # print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
+ # print "Histogram is ", hist, bin_edges
+ # plt.bar(bin_edges[:-1], hist, width = 10)
+ # plt.xlim(min(bin_edges), max(bin_edges))
+ # plt.show()
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistograms(MCCList)
+ #sys.exit()
+ #
+ print ("-"*80)
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print "********** findPrecipRate found"
+ # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print ("-"*80)
+ # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
+ # print ("-"*80)
+ # # sys.exit()
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/mainProgGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgGoyens.py b/mccsearch/code/mainProgGoyens.py
new file mode 100644
index 0000000..ad54ab1
--- /dev/null
+++ b/mccsearch/code/mainProgGoyens.py
@@ -0,0 +1,126 @@
+'''
+# running the program
+# Kim Whitehall Nov 2012
+# Last updated: 15 May 2013
+'''
+
+#import rcmes
+import sys
+import networkx as nx
+import mccSearchGoyens
+#import cloudclusters
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccSearchGoyens/caseStudy1"
+ #directories for the original data
+ TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
+ CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
+ #allMCSsDict ={{}}
+
+ # mccSearchGoyens.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # sys.exit()
+
+ # # mccSearchGoyens.plotAccTRMM(TRMMCEdirName)
+ # sys.exit()
+
+ #create main directory and file structure for storing intel
+ mccSearchGoyens.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # #mccSearchGoyens.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearchGoyens.postProcessingNetCDF(2)
+ # sys.exit()
+
+ #using merg data ***********
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearchGoyens.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName isnt entered for what ever reason
+ # CEGraph = mccSearchGoyens.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearchGoyens.findPrecipRate(TRMMdirName)
+ # sys.exit()
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearchGoyens.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearchGoyens.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ # for eachList in MCCList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
+ print "MCS List has been acquired ", len(MCSList)
+ # for eachList in MCSList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearchGoyens.thisDict(eachNode)['nodeMCSIdentifier']
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING TRMM ----------"
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+ #MCCTimes, tdelta = mccSearchGoyens.temporalAndAreaInfoMetric(MCCList)
+ print ("-"*80)
+ #print "MCCTimes is: ", MCCTimes
+ print "creating the MCS userfile ", mccSearchGoyens.createTextFile(MCCList)
+
+ # print "number of MCCs is: ", mccSearchGoyens.numberOfFeatures(MCCList)
+ # print "longest duration is: ", mccSearchGoyens.longestDuration(MCCTimes), "hrs"
+ # print "shortest duration is: ", mccSearchGoyens.shortestDuration(MCCTimes), "hrs"
+ # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
+ # print "Average duration is: ", mccSearchGoyens.averageDuration(MCCTimes), "hrs"
+ # print "Average size is: ", mccSearchGoyens.averageFeatureSize(MCCList), "km^2"
+
+ # #print "Average duration is: ", mccSearchGoyens.convert_timedelta(mccSearchGoyens.averageMCCLength(MCCTimes))
+ # hist, bin_edges = mccSearchGoyens.commonFeatureSize(allMCSsList)
+ # print "Histogram is ", hist, bin_edges
+ # plt.bar(bin_edges[:-1], hist, width = 10)
+ # plt.xlim(min(bin_edges), max(bin_edges))
+ # plt.show()
+
+ #some plots that work
+ # mccSearchGoyens.plotAccTRMM(MCCList)
+ # mccSearchGoyens.displayPrecip(MCCList)
+ # mccSearchGoyens.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearchGoyens.displaySize(MCCList)
+ # mccSearchGoyens.displayPrecip(MCCList)
+ # mccSearchGoyens.plotHistograms(MCCList)
+ #sys.exit()
+ #
+ print ("-"*80)
+
+ # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
+ # print "********** findPrecipRate found"
+ # allCETRMMList = mccSearchGoyens.findPrecipRate(CEdirName,TRMMdirName)
+ # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
+
+ # allCETRMMList = mccSearchGoyens.precipTotals(MCCList) #allMCSsList)
+ # print ("-"*80)
+ # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
+ # print ("-"*80)
+ # # sys.exit()
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/climate/blob/5f3a8521/mccsearch/code/mainProgWorks.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mainProgWorks.py b/mccsearch/code/mainProgWorks.py
new file mode 100644
index 0000000..f6eb55d
--- /dev/null
+++ b/mccsearch/code/mainProgWorks.py
@@ -0,0 +1,124 @@
+'''
+# running the program
+# Kim Whitehall Nov 2012
+# Last updated: 15 May 2013
+'''
+
+#import rcmes
+import sys
+import networkx as nx
+import mccSearch
+#import cloudclusters
+import numpy as np
+import numpy.ma as ma
+import files
+import matplotlib.pyplot as plt
+
+def main():
+ CEGraph = nx.DiGraph()
+ prunedGraph = nx.DiGraph()
+ MCCList =[]
+ MCSList=[]
+ MCSMCCNodesList =[]
+ allMCSsList =[]
+ allCETRMMList =[]
+ mainDirStr= "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
+ #directories for the original data
+ TRMMdirName = "/Users/kimwhitehall/Documents/HU/research/DATA/TRMM" #compSciPaper/case2/TRMM"
+ CEoriDirName = "/Users/kimwhitehall/Documents/HU/research/DATA/mergNETCDF"
+ #allMCSsDict ={{}}
+
+ # mccSearch.preprocessingMERG("/Users/kimwhitehall/Documents/HU/research/DATA")
+ # sys.exit()
+
+ # # mccSearch.plotAccTRMM(TRMMCEdirName)
+ # sys.exit()
+
+ #create main directory and file structure for storing intel
+ mccSearch.createMainDirectory(mainDirStr)
+ TRMMCEdirName = mainDirStr+'/TRMMnetcdfCEs'
+ CEdirName = mainDirStr+'/MERGnetcdfCEs'
+
+ # #mccSearch.postProcessingNetCDF(3,CEoriDirName)
+ # mccSearch.postProcessingNetCDF(2)
+ # sys.exit()
+
+ #using merg data ***********
+ print "\n -------------- Read MERG Data ----------"
+ mergImgs, timeList = mccSearch.readMergData(CEoriDirName)
+ print ("-"*80)
+
+ print 'in main', len(mergImgs)
+ print "\n -------------- TESTING findCloudElements ----------"
+ CEGraph = mccSearch.findCloudElements(mergImgs,timeList,TRMMdirName)
+ #if the TRMMdirName isnt entered for what ever reason
+ # CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
+ # allCETRMMList=mccSearch.findPrecipRate(TRMMdirName)
+ # sys.exit()
+ print ("-"*80)
+ print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
+ print ("-"*80)
+ print "\n -------------- TESTING findCloudClusters ----------"
+ prunedGraph = mccSearch.findCloudClusters(CEGraph)
+ print ("-"*80)
+ print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING findMCCs ----------"
+ MCCList,MCSList = mccSearch.findMCC(prunedGraph)
+ print ("-"*80)
+ print "MCC List has been acquired ", len(MCCList)
+ # for eachList in MCCList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print "MCS List has been acquired ", len(MCSList)
+ # for eachList in MCSList:
+ # print eachList
+ # for eachNode in eachList:
+ # print eachNode, mccSearch.thisDict(eachNode)['nodeMCSIdentifier']
+ print ("-"*80)
+ #sys.exit()
+ print "\n -------------- TESTING TRMM ----------"
+ #now ready to perform various calculations/metrics
+ print "\n -------------- TESTING METRICS ----------"
+ MCCTimes, tdelta = mccSearch.temporalAndAreaInfoMetric(MCCList)
+ print ("-"*80)
+ print "MCCTimes is: ", MCCTimes
+ print ("-"*80)
+ print "number of MCCs is: ", mccSearch.numberOfFeatures(MCCList)
+ print "longest duration is: ", mccSearch.longestDuration(MCCTimes), "hrs"
+ print "shortest duration is: ", mccSearch.shortestDuration(MCCTimes), "hrs"
+ #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ print "Average duration is: ", mccSearch.averageDuration(MCCTimes), "hrs"
+ print "Average size is: ", mccSearch.averageFeatureSize(MCCList), "km^2"
+
+ # #print "Average duration is: ", mccSearch.convert_timedelta(mccSearch.averageMCCLength(MCCTimes))
+ # hist, bin_edges = mccSearch.commonFeatureSize(allMCSsList)
+ # print "Histogram is ", hist, bin_edges
+ # plt.bar(bin_edges[:-1], hist, width = 10)
+ # plt.xlim(min(bin_edges), max(bin_edges))
+ # plt.show()
+
+ #some plots that work
+ # mccSearch.plotAccTRMM(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotAccuInTimeRange('2009-09-01_00:00:00', '2009-09-01_23:00:00')
+ # mccSearch.displaySize(MCCList)
+ # mccSearch.displayPrecip(MCCList)
+ # mccSearch.plotHistograms(MCCList)
+ #sys.exit()
+ #
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print "********** findPrecipRate found"
+ # allCETRMMList = mccSearch.findPrecipRate(CEdirName,TRMMdirName)
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+
+ # allCETRMMList = mccSearch.precipTotals(MCCList) #allMCSsList)
+ # print ("-"*80)
+ # print "allCETRMMList List has been aquired ", len(allCETRMMList), allCETRMMList
+ # print ("-"*80)
+ # # sys.exit()
+
+main()
\ No newline at end of file
[42/50] [abbrv] Fix formatting, remove unused imports,
update files.py and process.py to most recent OCW code
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/code/process.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/process.py b/mccsearch/code/process.py
index 5a362ec..7367ee7 100644
--- a/mccsearch/code/process.py
+++ b/mccsearch/code/process.py
@@ -1,4 +1,20 @@
-"""Collection of functions that process numpy arrays of varying shapes.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Collection of functions that process numpy arrays of varying shapes.
Functions can range from subsetting to regridding"""
# Python Standard Libary Imports
@@ -10,38 +26,38 @@ import sys
import time
# 3rd Party Imports
-import Nio
+import netCDF4
import numpy as np
import numpy.ma as ma
-from scipy.ndimage import map_coordinates
+
def extract_subregion_from_data_array(data, lats, lons, latmin, latmax, lonmin, lonmax):
'''
Extract a sub-region from a data array.
-
+
Example::
**e.g.** the user may load a global model file, but only want to examine data over North America
This function extracts a sub-domain from the original data.
The defined sub-region must be a regular lat/lon bounding box,
but the model data may be on a non-regular grid (e.g. rotated, or Guassian grid layout).
- Data are kept on the original model grid and a rectangular (in model-space) region
+ Data are kept on the original model grid and a rectangular (in model-space) region
is extracted which contains the rectangular (in lat/lon space) user supplied region.
-
+
INPUT::
data - 3d masked data array
lats - 2d array of latitudes corresponding to data array
lons - 2d array of longitudes corresponding to data array
latmin, latmax, lonmin, lonmax - bounding box of required region to extract
-
+
OUTPUT::
data2 - subset of original data array containing only data from required subregion
lats2 - subset of original latitude data array
lons2 - subset of original longitude data array
'''
-
-
+
+
# Mask model data array to find grid points inside the supplied bounding box
whlat = (lats > latmin) & (lats < latmax)
@@ -67,37 +83,37 @@ def extract_subregion_from_data_array(data, lats, lons, latmin, latmax, lonmin,
def calc_area_mean(data, lats, lons, mymask='not set'):
'''
Calculate Area Average of data in a masked array
-
+
INPUT::
data: a masked array of data (NB. only data from one time expected to be passed at once)
lats: 2d array of regularly gridded latitudes
lons: 2d array of regularly gridded longitudes
mymask: (optional) defines spatial region to do averaging over
-
+
OUTPUT::
area_mean: a value for the mean inside the area
'''
-
-
+
+
# If mask not passed in, then set maks to cover whole data domain
if(mymask == 'not set'):
- mymask = np.empty(data.shape)
- mymask[:] = False # NB. mask means (don't show), so False everywhere means use everything.
-
+ mymask = np.empty(data.shape)
+ mymask[:] = False # NB. mask means (don't show), so False everywhere means use everything.
+
# Dimension check on lats, lons
# Sometimes these arrays are 3d, sometimes 2d, sometimes 1d
# This bit of code just converts to the required 2d array shape
if(len(lats.shape) == 3):
- lats = lats[0, :, :]
+ lats = lats[0, :, :]
if(len(lons.shape) == 3):
- lons = lons[0, :, :]
+ lons = lons[0, :, :]
if(np.logical_and(len(lats.shape) == 1, len(lons.shape) == 1)):
- lons, lats = np.meshgrid(lons, lats)
-
+ lons, lats = np.meshgrid(lons, lats)
+
# Calculate grid length (assuming regular lat/lon grid)
dlat = lats[1, 0] - lats[0, 0]
dlon = lons[0, 1] - lons[0, 0]
@@ -108,28 +124,28 @@ def calc_area_mean(data, lats, lons, mymask='not set'):
# Create a new masked array covering just user selected area (defined in mymask)
# NB. this preserves missing data points in the observations data
subdata = ma.masked_array(data, mask=mymask)
-
+
if(myweights.shape != subdata.shape):
- myweights.resize(subdata.shape)
- myweights[1:, :] = myweights[0, :]
+ myweights.resize(subdata.shape)
+ myweights[1:, :] = myweights[0, :]
# Calculate weighted mean using ma.average (which takes weights)
area_mean = ma.average(subdata, weights=myweights)
-
+
return area_mean
def calc_area_in_grid_box(latitude, dlat, dlon):
'''
Calculate area of regular lat-lon grid box
-
+
INPUT::
latitude: latitude of grid box (degrees)
dlat: grid length in latitude direction (degrees)
dlon: grid length in longitude direction (degrees)
-
+
OUTPUT::
A: area of the grid box
-
+
'''
R = 6371000 # radius of Earth in metres
@@ -141,422 +157,286 @@ def calc_area_in_grid_box(latitude, dlat, dlon):
return A
-def do_regrid(q, lat, lon, lat2, lon2, order=1, mdi= -999999999):
+def do_regrid(q, lat, lon, lat2, lon2, order=1, mdi=-999999999):
+ """
+ This function has been moved to the ocw/dataset_processor module
+ """
+ from ocw import dataset_processor
+ q2 = dataset_processor._rcmes_spatial_regrid(q, lat, lon, lat2, lon2, order=1)
+
+ return q2
+
+def create_mask_using_threshold(masked_array, threshold=0.5):
+ """
+ .. deprecated:: 0.3-incubating
+ Use :func:`ocw.dataset_processor._rcmes_create_mask_using_threshold` instead.
+ """
+ from ocw import dataset_processor
+ mask = dataset_processor._rcmes_create_mask_using_threshold(masked_array, threshold)
+
+ return mask
+
+def calc_average_on_new_time_unit(data, dateList, unit='monthly'):
'''
- Perform regridding from one set of lat,lon values onto a new set (lat2,lon2)
-
+ Routine to calculate averages on longer time units than the data exists on.
+
+ Example::
+ If the data is 6-hourly, calculate daily, or monthly means.
+
Input::
- q - the variable to be regridded
- lat,lon - original co-ordinates corresponding to q values
- lat2,lon2 - new set of latitudes and longitudes that you want to regrid q onto
- order - (optional) interpolation order 1=bi-linear, 3=cubic spline
- mdi - (optional) fill value for missing data (used in creation of masked array)
-
- Output::
- q2 - q regridded onto the new set of lat2,lon2
-
+ data - data values
+ dateList - list of python datetime structures corresponding to data values
+ unit - string describing time unit to average onto.
+ *Valid values are: 'monthly', 'daily', 'pentad','annual','decadal'*
+
+ Output:
+ meanstorem - numpy masked array of data values meaned over required time period
+ newTimesList - a list of python datetime objects representing the data in the new averagin period
+ *NB.* currently set to beginning of averaging period,
+ **i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.**
'''
- nlat = q.shape[0]
- nlon = q.shape[1]
-
- nlat2 = lat2.shape[0]
- nlon2 = lon2.shape[1]
-
- # To make our lives easier down the road, let's
- # turn these into arrays of x & y coords
- loni = lon2.ravel()
- lati = lat2.ravel()
-
- loni = loni.copy() # NB. it won't run unless you do this...
- lati = lati.copy()
-
- # Now, we'll set points outside the boundaries to lie along an edge
- loni[loni > lon.max()] = lon.max()
- loni[loni < lon.min()] = lon.min()
-
- # To deal with the "hard" break, we'll have to treat y differently,
- # so we're just setting the min here...
- lati[lati > lat.max()] = lat.max()
- lati[lati < lat.min()] = lat.min()
-
-
- # We need to convert these to (float) indicies
- # (xi should range from 0 to (nx - 1), etc)
- loni = (nlon - 1) * (loni - lon.min()) / (lon.max() - lon.min())
-
- # Deal with the "hard" break in the y-direction
- lati = (nlat - 1) * (lati - lat.min()) / (lat.max() - lat.min())
-
- # Notes on dealing with MDI when regridding data.
- # Method adopted here:
- # Use bilinear interpolation of data by default (but user can specify other order using order=... in call)
- # Perform bilinear interpolation of data, and of mask.
- # To be conservative, new grid point which contained some missing data on the old grid is set to missing data.
- # -this is achieved by looking for any non-zero interpolated mask values.
- # To avoid issues with bilinear interpolation producing strong gradients leading into the MDI,
- # set values at MDI points to mean data value so little gradient visible = not ideal, but acceptable for now.
-
- # Set values in MDI so that similar to surroundings so don't produce large gradients when interpolating
- # Preserve MDI mask, by only changing data part of masked array object.
- for shift in (-1, 1):
- for axis in (0, 1):
- q_shifted = np.roll(q, shift=shift, axis=axis)
- idx = ~q_shifted.mask * q.mask
- q.data[idx] = q_shifted[idx]
-
- # Now we actually interpolate
- # map_coordinates does cubic interpolation by default,
- # use "order=1" to preform bilinear interpolation instead...
- q2 = map_coordinates(q, [lati, loni], order=order)
- q2 = q2.reshape([nlat2, nlon2])
-
- # Set values to missing data outside of original domain
- q2 = ma.masked_array(q2, mask=np.logical_or(np.logical_or(lat2 >= lat.max(),
- lat2 <= lat.min()),
- np.logical_or(lon2 <= lon.min(),
- lon2 >= lon.max())))
-
- # Make second map using nearest neighbour interpolation -use this to determine locations with MDI and mask these
- qmdi = np.zeros_like(q)
- qmdi[q.mask == True] = 1.
- qmdi[q.mask == False] = 0.
- qmdi_r = map_coordinates(qmdi, [lati, loni], order=order)
- qmdi_r = qmdi_r.reshape([nlat2, nlon2])
- mdimask = (qmdi_r != 0.0)
-
- # Combine missing data mask, with outside domain mask define above.
- q2.mask = np.logical_or(mdimask, q2.mask)
+ # First catch unknown values of time unit passed in by user
+ acceptable = (unit == 'full') | (unit == 'annual') | (unit == 'monthly') | (unit == 'daily') | (unit == 'pentad')
- return q2
+ if not acceptable:
+ print 'Error: unknown unit type selected for time averaging'
+ print ' Please check your code.'
+ return
-def create_mask_using_threshold(masked_array, threshold=0.5):
- '''
- Routine to create a mask, depending on the proportion of times with missing data.
- For each pixel, calculate proportion of times that are missing data,
- **if** the proportion is above a specified threshold value,
- then mark the pixel as missing data.
-
- Input::
- masked_array - a numpy masked array of data (assumes time on axis 0, and space on axes 1 and 2).
- threshold - (optional) threshold proportion above which a pixel is marked as 'missing data'.
- NB. default threshold = 50%
-
- Output::
- mymask - a numpy array describing the mask. NB. not a masked array, just the mask itself.
+ # Calculate arrays of years (2007,2007),
+ # yearsmonths (200701,200702),
+ # or yearmonthdays (20070101,20070102)
+ # -depending on user selected averaging period.
- '''
-
+ # Year list
+ if unit == 'annual':
+ print 'Calculating annual mean data'
+ timeunits = []
- # try, except used as some model files don't have a full mask, but a single bool
- # the except catches this situation and deals with it appropriately.
- try:
- nT = masked_array.mask.shape[0]
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year))
- # For each pixel, count how many times are masked.
- nMasked = masked_array.mask[:, :, :].sum(axis=0)
+ timeunits = np.array(timeunits, dtype=int)
- # Define new mask as when a pixel has over a defined threshold ratio of masked data
- # e.g. if the threshold is 75%, and there are 10 times,
- # then a pixel will be masked if more than 5 times are masked.
- mymask = nMasked > (nT * threshold)
+ # YearMonth format list
+ if unit == 'monthly':
+ print 'Calculating monthly mean data'
+ timeunits = []
- except:
- mymask = np.zeros_like(masked_array.data[0, :, :])
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
- return mymask
+ timeunits = np.array(timeunits, dtype=int)
-def calc_average_on_new_time_unit(data, dateList, unit='monthly'):
- '''
- Routine to calculate averages on longer time units than the data exists on.
-
- Example::
- If the data is 6-hourly, calculate daily, or monthly means.
-
- Input::
- data - data values
- dateList - list of python datetime structures corresponding to data values
- unit - string describing time unit to average onto.
- *Valid values are: 'monthly', 'daily', 'pentad','annual','decadal'*
-
- Output:
- meanstorem - numpy masked array of data values meaned over required time period
- newTimesList - a list of python datetime objects representing the data in the new averagin period
- *NB.* currently set to beginning of averaging period,
- **i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.**
- '''
+ # YearMonthDay format list
+ if unit == 'daily':
+ print 'Calculating daily mean data'
+ timeunits = []
- # First catch unknown values of time unit passed in by user
- acceptable = (unit == 'full') | (unit == 'annual') | (unit == 'monthly') | (unit == 'daily') | (unit == 'pentad')
+ for i in np.arange(len(dateList)):
+ timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + \
+ str("%02d" % dateList[i].day))
- if not acceptable:
- print 'Error: unknown unit type selected for time averaging'
- print ' Please check your code.'
- return
+ timeunits = np.array(timeunits, dtype=int)
- # Calculate arrays of years (2007,2007),
- # yearsmonths (200701,200702),
- # or yearmonthdays (20070101,20070102)
- # -depending on user selected averaging period.
- # Year list
- if unit == 'annual':
- print 'Calculating annual mean data'
- timeunits = []
+ # TODO: add pentad setting using Julian days?
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year))
+ # Full list: a special case
+ if unit == 'full':
+ print 'Calculating means data over full time range'
+ timeunits = []
- timeunits = np.array(timeunits, dtype=int)
-
- # YearMonth format list
- if unit == 'monthly':
- print 'Calculating monthly mean data'
- timeunits = []
+ for i in np.arange(len(dateList)):
+ timeunits.append(999) # i.e. we just want the same value for all times.
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
+ timeunits = np.array(timeunits, dtype=int)
- timeunits = np.array(timeunits, dtype=int)
- # YearMonthDay format list
- if unit == 'daily':
- print 'Calculating daily mean data'
- timeunits = []
-
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + \
- str("%02d" % dateList[i].day))
- timeunits = np.array(timeunits, dtype=int)
+ # empty list to store new times
+ newTimesList = []
+ # Decide whether or not you need to do any time averaging.
+ # i.e. if data are already on required time unit then just pass data through and
+ # calculate and return representative datetimes.
+ processing_required = True
+ if len(timeunits) == (len(np.unique(timeunits))):
+ processing_required = False
- # TODO: add pentad setting using Julian days?
+ # 1D data arrays, i.e. time series
+ if data.ndim == 1:
+ # Create array to store the resulting data
+ meanstore = np.zeros(len(np.unique(timeunits)))
+ # Calculate the means across each unique time unit
+ i = 0
+ for myunit in np.unique(timeunits):
+ if processing_required:
+ datam = ma.masked_array(data, timeunits != myunit)
+ meanstore[i] = ma.average(datam)
- # Full list: a special case
- if unit == 'full':
- print 'Calculating means data over full time range'
- timeunits = []
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(myunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
- for i in np.arange(len(dateList)):
- timeunits.append(999) # i.e. we just want the same value for all times.
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+ i = i + 1
- timeunits = np.array(timeunits, dtype=int)
+ # 3D data arrays
+ if data.ndim == 3:
+ #datamask = create_mask_using_threshold(data,threshold=0.75)
+ # Create array to store the resulting data
+ meanstore = np.zeros([len(np.unique(timeunits)), data.shape[1], data.shape[2]])
- # empty list to store new times
- newTimesList = []
+ # Calculate the means across each unique time unit
+ i = 0
+ datamask_store = []
+ for myunit in np.unique(timeunits):
+ if processing_required:
- # Decide whether or not you need to do any time averaging.
- # i.e. if data are already on required time unit then just pass data through and
- # calculate and return representative datetimes.
- processing_required = True
- if len(timeunits) == (len(np.unique(timeunits))):
- processing_required = False
+ mask = np.zeros_like(data)
+ mask[timeunits != myunit, :, :] = 1.0
- # 1D data arrays, i.e. time series
- if data.ndim == 1:
- # Create array to store the resulting data
- meanstore = np.zeros(len(np.unique(timeunits)))
-
- # Calculate the means across each unique time unit
- i = 0
- for myunit in np.unique(timeunits):
- if processing_required:
- datam = ma.masked_array(data, timeunits != myunit)
- meanstore[i] = ma.average(datam)
-
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit) == 4: # YYYY
- yyyy = int(myunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit) == 6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit) == 8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit) == 3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1] - dateList[0]
- halfway = dateList[0] + (dt / 2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
-
- newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
- i = i + 1
-
- # 3D data arrays
- if data.ndim == 3:
-
- #datamask = create_mask_using_threshold(data,threshold=0.75)
-
- # Create array to store the resulting data
- meanstore = np.zeros([len(np.unique(timeunits)), data.shape[1], data.shape[2]])
-
- # Calculate the means across each unique time unit
- i = 0
- datamask_store = []
- for myunit in np.unique(timeunits):
- if processing_required:
-
- mask = np.zeros_like(data)
- mask[timeunits != myunit, :, :] = 1.0
-
- # Calculate missing data mask within each time unit...
- datamask_at_this_timeunit = np.zeros_like(data)
- datamask_at_this_timeunit[:] = create_mask_using_threshold(data[timeunits == myunit, :, :], threshold=0.75)
- # Store results for masking later
- datamask_store.append(datamask_at_this_timeunit[0])
-
- # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
- datam = ma.masked_array(data, np.logical_or(mask, datamask_at_this_timeunit))
- meanstore[i, :, :] = ma.average(datam, axis=0)
-
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit) == 4: # YYYY
- yyyy = int(smyunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit) == 6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit) == 8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit) == 3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1] - dateList[0]
- halfway = dateList[0] + (dt / 2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
-
- newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
-
- i += 1
-
- if not processing_required:
- meanstorem = data
-
- if processing_required:
- # Create masked array (using missing data mask defined above)
- datamask_store = np.array(datamask_store)
- meanstorem = ma.masked_array(meanstore, datamask_store)
-
- return meanstorem, newTimesList
+ # Calculate missing data mask within each time unit...
+ datamask_at_this_timeunit = np.zeros_like(data)
+ datamask_at_this_timeunit[:] = create_mask_using_threshold(data[timeunits == myunit, :, :], threshold=0.75)
+ # Store results for masking later
+ datamask_store.append(datamask_at_this_timeunit[0])
+
+ # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
+ datam = ma.masked_array(data, np.logical_or(mask, datamask_at_this_timeunit))
+ meanstore[i, :, :] = ma.average(datam, axis=0)
+
+ # construct new times list
+ smyunit = str(myunit)
+ if len(smyunit) == 4: # YYYY
+ yyyy = int(smyunit[0:4])
+ mm = 1
+ dd = 1
+ if len(smyunit) == 6: # YYYYMM
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = 1
+ if len(smyunit) == 8: # YYYYMMDD
+ yyyy = int(smyunit[0:4])
+ mm = int(smyunit[4:6])
+ dd = int(smyunit[6:8])
+ if len(smyunit) == 3: # Full time range
+ # Need to set an appropriate time representing the mid-point of the entire time span
+ dt = dateList[-1] - dateList[0]
+ halfway = dateList[0] + (dt / 2)
+ yyyy = int(halfway.year)
+ mm = int(halfway.month)
+ dd = int(halfway.day)
+
+ newTimesList.append(datetime.datetime(yyyy, mm, dd, 0, 0, 0, 0))
+
+ i += 1
+
+ if not processing_required:
+ meanstorem = data
+
+ if processing_required:
+ # Create masked array (using missing data mask defined above)
+ datamask_store = np.array(datamask_store)
+ meanstorem = ma.masked_array(meanstore, datamask_store)
+
+ return meanstorem, newTimesList
def calc_running_accum_from_period_accum(data):
'''
Routine to calculate running total accumulations from individual period accumulations.
::
-
+
e.g. 0,0,1,0,0,2,2,1,0,0
-> 0,0,1,1,1,3,5,6,6,6
-
+
Input::
data: numpy array with time in the first axis
-
+
Output::
running_acc: running accumulations
-
+
'''
-
-
+
+
running_acc = np.zeros_like(data)
-
+
if(len(data.shape) == 1):
running_acc[0] = data[0]
-
+
if(len(data.shape) > 1):
running_acc[0, :] = data[0, :]
-
+
for i in np.arange(data.shape[0] - 1):
if(len(data.shape) == 1):
running_acc[i + 1] = running_acc[i] + data[i + 1]
-
+
if(len(data.shape) > 1):
running_acc[i + 1, :] = running_acc[i, :] + data[i + 1, :]
-
+
return running_acc
def ignore_boundaries(data, rim=10):
'''
Routine to mask the lateral boundary regions of model data to ignore them from calculations.
-
+
Input::
data - a masked array of model data
rim - (optional) number of grid points to ignore
-
+
Output::
data - data array with boundary region masked
-
+
'''
-
+
nx = data.shape[1]
ny = data.shape[0]
-
+
rimmask = np.zeros_like(data)
for j in np.arange(rim):
rimmask[j, 0:nx - 1] = 1.0
-
+
for j in ny - 1 - np.arange(rim):
rimmask[j, 0:nx - 1] = 1.0
-
+
for i in np.arange(rim):
rimmask[0:ny - 1, i] = 1.0
-
+
for i in nx - 1 - np.arange(rim):
rimmask[0:ny - 1, i] = 1.0
-
+
data = ma.masked_array(data, mask=rimmask)
-
+
return data
def normalizeDatetimes(datetimes, timestep):
- """
- Input::
- datetimes - list of datetime objects that need to be normalized
- timestep - string of value ('daily' | 'monthly')
- Output::
- normalDatetimes - list of datetime objects that have been normalized
-
- Normalization Rules::
- Daily data will be forced to an hour value of 00:00:00
- Monthly data will be forced to the first of the month at midnight
- """
- normalDatetimes = []
- if timestep.lower() == 'monthly':
- for inputDatetime in datetimes:
- if inputDatetime.day != 1:
- # Clean the inputDatetime
- inputDatetimeString = inputDatetime.strftime('%Y%m%d')
- normalInputDatetimeString = inputDatetimeString[:6] + '01'
- inputDatetime = datetime.datetime.strptime(normalInputDatetimeString, '%Y%m%d')
-
- normalDatetimes.append(inputDatetime)
-
- elif timestep.lower() == 'daily':
- for inputDatetime in datetimes:
- if inputDatetime.hour != 0 or inputDatetime.minute != 0 or inputDatetime.second != 0:
- datetimeString = inputDatetime.strftime('%Y%m%d%H%M%S')
- normalDatetimeString = datetimeString[:8] + '000000'
- inputDatetime = datetime.datetime.strptime(normalDatetimeString, '%Y%m%d%H%M%S')
-
- normalDatetimes.append(inputDatetime)
-
- return normalDatetimes
+ """ This function has been moved to the ocw/dataset_processor module """
+ from ocw import dataset_processor as dsp
+ return dsp._rcmes_normalize_datetimes(datetimes, timestep)
def getModelTimes(modelFile, timeVarName):
'''
@@ -573,21 +453,19 @@ def getModelTimes(modelFile, timeVarName):
modelTimeStep - 'hourly','daily','monthly','annual'
'''
- f = Nio.open_file(modelFile)
+ f = netCDF4.Dataset(modelFile, mode='r')
xtimes = f.variables[timeVarName]
- timeFormat = xtimes.attributes['units']
-
+ timeFormat = xtimes.units.encode()
+
# search to check if 'since' appears in units
try:
- sinceLoc = re.search('since', timeFormat).end()
-
+ sinceLoc = re.search('since', timeFormat).end()
- #KDW the below block generates and error. But the print statement, indicates that sinceLoc found something
except AttributeError:
- print 'Error decoding model times: time variable attributes do not contain "since"'
- raise
+ print 'Error decoding model times: time variable attributes do not contain "since"'
+ raise
- units = ''
+ units = None
TIME_UNITS = ('minutes', 'hours', 'days', 'months', 'years')
# search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
for unit in TIME_UNITS:
@@ -597,17 +475,14 @@ def getModelTimes(modelFile, timeVarName):
# cut out base time (the bit following 'since')
base_time_string = string.lstrip(timeFormat[sinceLoc:])
-
# decode base time
base_time = decodeTimeFromString(base_time_string)
-
times = []
-
-
for xtime in xtimes[:]:
-
- # Cast time as an int ***KDW remove this so fractional xtime can be read from MERG
+
+ # Cast time as an int
+ #TODO: KDW this may cause problems for data that is hourly with more than one timestep in it
xtime = int(xtime)
if units == 'minutes':
@@ -631,23 +506,24 @@ def getModelTimes(modelFile, timeVarName):
elif units == 'years':
dt = datetime.timedelta(years=xtime)
new_time = base_time + dt
-
- #print "xtime is:", xtime, "dt is: ", dt
-
-
times.append(new_time)
-
+
try:
- timeStepLength = int(xtimes[1] - xtimes[0] + 1.e-12)
+ if len(xtimes) == 1:
+ timeStepLength = 0
+ else:
+ timeStepLength = int(xtimes[1] - xtimes[0] + 1.e-12)
+
modelTimeStep = getModelTimeStep(units, timeStepLength)
-
- #KDW if timeStepLength is zero do not normalize times as this would create an empty list
+
+ #if timeStepLength is zero do not normalize times as this would create an empty list for MERG (hourly) data
if timeStepLength != 0:
- times = normalizeDatetimes(times, modelTimeStep)
+ times = normalizeDatetimes(times, modelTimeStep)
except:
raise
+
return times, modelTimeStep
def getModelTimeStep(units, stepSize):
@@ -661,22 +537,22 @@ def getModelTimeStep(units, stepSize):
# 28 days through 31 days
elif 40320 <= stepSize <= 44640:
modelTimeStep = 'monthly'
- # 365 days through 366 days
+ # 365 days through 366 days
elif 525600 <= stepSize <= 527040:
- modelTimeStep = 'annual'
+ modelTimeStep = 'annual'
else:
raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
elif units == 'hours':
- #need a check for fractional hrs and only one hr i.e. stepSize=0
+ #need a check for fractional hrs and only one hr i.e. stepSize=0 e.g. with MERG data
if stepSize == 0 or stepSize == 1:
modelTimeStep = 'hourly'
elif stepSize == 24:
modelTimeStep = 'daily'
elif 672 <= stepSize <= 744:
- modelTimeStep = 'monthly'
+ modelTimeStep = 'monthly'
elif 8760 <= stepSize <= 8784:
- modelTimeStep = 'annual'
+ modelTimeStep = 'annual'
else:
raise Exception('model data time step interval exceeds the max time interval (annual)', units, stepSize)
@@ -717,7 +593,7 @@ def decodeTimeFromString(time_string):
::
**Input:** time_string - a string that represents a date/time
-
+
**Output:** mytime - a python datetime object
'''
@@ -774,9 +650,9 @@ def decodeTimeFromString(time_string):
except ValueError:
pass
-#KDW added
+
try:
- mytime = time.strptime(time_string, '%Y-%m-%d')
+ mytime = time.strptime(time_string, '%Y-%m-%d %H')
mytime = datetime.datetime(*mytime[0:6])
return mytime
@@ -784,210 +660,286 @@ def decodeTimeFromString(time_string):
pass
try:
- mytime = time.strptime(time_string, '%Y-%m-%d %H')
+ mytime = time.strptime(time_string, '%Y-%m-%d')
mytime = datetime.datetime(*mytime[0:6])
return mytime
except ValueError:
pass
-
print 'Error decoding time string: string does not match a predefined time format'
return 0
def regrid_wrapper(regrid_choice, obsdata, obslats, obslons, wrfdata, wrflats, wrflons):
- '''
- Wrapper routine for regridding.
-
- Either regrids model to obs grid, or obs to model grid, depending on user choice
-
- Inputs::
- regrid_choice - [0] = Regrid obs data onto model grid or
- [1] = Regrid model data onto obs grid
-
- obsdata,wrfdata - data arrays
- obslats,obslons - observation lat,lon arrays
- wrflats,wrflons - model lat,lon arrays
-
- Output::
- rdata,rdata2 - regridded data
- lats,lons - latitudes and longitudes of regridded data
-
- '''
+ '''
+ Wrapper routine for regridding.
+
+ Either regrids model to obs grid, or obs to model grid, depending on user choice
+
+ Inputs::
+ regrid_choice - [0] = Regrid obs data onto model grid or
+ [1] = Regrid model data onto obs grid
+
+ obsdata,wrfdata - data arrays
+ obslats,obslons - observation lat,lon arrays
+ wrflats,wrflons - model lat,lon arrays
+
+ Output::
+ rdata,rdata2 - regridded data
+ lats,lons - latitudes and longitudes of regridded data
- # Regrid obs data onto model grid
- if(regrid_choice == '0'):
+ '''
+
+ # Regrid obs data onto model grid
+ if(regrid_choice == '0'):
+
+ ndims = len(obsdata.shape)
+ if(ndims == 3):
+ newshape = [obsdata.shape[0], wrfdata.shape[1], wrfdata.shape[2]]
+ nT = obsdata.shape[0]
- ndims = len(obsdata.shape)
- if(ndims == 3):
- newshape = [obsdata.shape[0], wrfdata.shape[1], wrfdata.shape[2]]
- nT = obsdata.shape[0]
+ if(ndims == 2):
+ newshape = [wrfdata.shape[0], wrfdata.shape[1]]
+ nT = 0
- if(ndims == 2):
- newshape = [wrfdata.shape[0], wrfdata.shape[1]]
- nT = 0
+ rdata = wrfdata
+ lats, lons = wrflats, wrflons
- rdata = wrfdata
- lats, lons = wrflats, wrflons
+ # Create a new masked array with the required dimensions
+ tmp = np.zeros(newshape)
+ rdata2 = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
- # Create a new masked array with the required dimensions
- tmp = np.zeros(newshape)
- rdata2 = ma.MaskedArray(tmp, mask=(tmp == 0))
- tmp = 0 # free up some memory
+ rdata2[:] = 0.0
- rdata2[:] = 0.0
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata2[t, :, :] = do_regrid(obsdata[t, :, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
- if(nT > 0):
- for t in np.arange(nT):
- rdata2[t, :, :] = do_regrid(obsdata[t, :, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+ if(nT == 0):
+ rdata2[:, :] = do_regrid(obsdata[:, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
- if(nT == 0):
- rdata2[:, :] = do_regrid(obsdata[:, :], obslats[:, :], obslons[:, :], wrflats[:, :], wrflons[:, :])
+ # Regrid model data onto obs grid
+ if(regrid_choice == '1'):
+ ndims = len(wrfdata.shape)
+ if(ndims == 3):
+ newshape = [wrfdata.shape[0], obsdata.shape[1], obsdata.shape[2]]
+ nT = obsdata.shape[0]
- # Regrid model data onto obs grid
- if(regrid_choice == '1'):
- ndims = len(wrfdata.shape)
- if(ndims == 3):
- newshape = [wrfdata.shape[0], obsdata.shape[1], obsdata.shape[2]]
- nT = obsdata.shape[0]
+ if(ndims == 2):
+ newshape = [obsdata.shape[0], obsdata.shape[1]]
+ nT = 0
- if(ndims == 2):
- newshape = [obsdata.shape[0], obsdata.shape[1]]
- nT = 0
+ rdata2 = obsdata
+ lats, lons = obslats, obslons
- rdata2 = obsdata
- lats, lons = obslats, obslons
+ tmp = np.zeros(newshape)
+ rdata = ma.MaskedArray(tmp, mask=(tmp == 0))
+ tmp = 0 # free up some memory
- tmp = np.zeros(newshape)
- rdata = ma.MaskedArray(tmp, mask=(tmp == 0))
- tmp = 0 # free up some memory
+ rdata[:] = 0.0
- rdata[:] = 0.0
-
- if(nT > 0):
- for t in np.arange(nT):
- rdata[t, :, :] = do_regrid(wrfdata[t, :, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+ if(nT > 0):
+ for t in np.arange(nT):
+ rdata[t, :, :] = do_regrid(wrfdata[t, :, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
- if(nT == 0):
- rdata[:, :] = do_regrid(wrfdata[:, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
+ if(nT == 0):
+ rdata[:, :] = do_regrid(wrfdata[:, :], wrflats[:, :], wrflons[:, :], obslats[:, :], obslons[:, :])
-
- return rdata, rdata2, lats, lons
+
+ return rdata, rdata2, lats, lons
def extract_sub_time_selection(allTimes, subTimes, data):
'''
Routine to extract a sub-selection of times from a data array.
Example::
- Suppose a data array has 30 time values for daily data for a whole month,
+ Suppose a data array has 30 time values for daily data for a whole month,
but you only want the data from the 5th-15th of the month.
-
+
Input::
allTimes - list of datetimes describing what times the data in the data array correspond to
subTimes - the times you want to extract data from
data - the data array
-
+
Output::
subdata - subselection of data array
-
+
'''
# Create new array to store the subselection
subdata = np.zeros([len(subTimes), data.shape[1], data.shape[2]])
-
+
# Loop over all Times and when it is a member of the required subselection, copy the data across
i = 0 # counter for allTimes
subi = 0 # counter for subTimes
for t in allTimes:
- if(np.setmember1d(allTimes, subTimes)):
+ if(np.in1d(allTimes, subTimes)):
subdata[subi, :, :] = data[i, :, :]
subi += 1
i += 1
-
+
return subdata
def calc_average_on_new_time_unit_K(data, dateList, unit):
- '''
- Routine to calculate averages on longer time units than the data exists on.
- e.g. if the data is 6-hourly, calculate daily, or monthly means.
-
- Input:
- data - data values
- dateList - list of python datetime structures corresponding to data values
- unit - string describing time unit to average onto
- e.g. 'monthly', 'daily', 'pentad','annual','decadal'
- Output:
- meanstorem - numpy masked array of data values meaned over required time period
- newTimesList - a list of python datetime objects representing the data in the new averagin period
- NB. currently set to beginning of averaging period,
- i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
- '''
+ """
+ This function has been moved to the ocw/dataset_processor module
+ """
+ from ocw import dataset_processor
+ temporally_regridded_data = dataset_processor._rcmes_calc_average_on_new_time_unit_K(data, dateList, unit)
+ return temporally_regridded_data
+
+def decode_model_timesK(ifile,timeVarName,file_type):
+ #################################################################################################
+ # Convert model times ('hours since 1900...', 'days since ...') into a python datetime structure
+ # Input:
+ # filelist - list of model files
+ # timeVarName - name of the time variable in the model files
+ # Output:
+ # times - list of python datetime objects describing model data times
+ # Peter Lean February 2011
+ #################################################################################################
+ f = netCDF4.Dataset(ifile,mode='r',format=file_type)
+ xtimes = f.variables[timeVarName]
+ timeFormat = xtimes.units.encode()
+ #timeFormat = "days since 1979-01-01 00:00:00"
+ # search to check if 'since' appears in units
+ try:
+ sinceLoc = re.search('since',timeFormat).end()
+ except:
+ print 'Error decoding model times: time var attributes do not contain "since": Exit'
+ sys.exit()
+ # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
+ # TODO: Swap out this section for a set of if...elseif statements
+ units = ''
+ try:
+ _ = re.search('minutes',timeFormat).end()
+ units = 'minutes'
+ except:
+ pass
+ try:
+ _ = re.search('hours',timeFormat).end()
+ units = 'hours'
+ except:
+ pass
+ try:
+ _ = re.search('days',timeFormat).end()
+ units = 'days'
+ except:
+ pass
+ try:
+ _ = re.search('months',timeFormat).end()
+ units = 'months'
+ except:
+ pass
+ try:
+ _ = re.search('years',timeFormat).end()
+ units = 'years'
+ except:
+ pass
+ # cut out base time (the bit following 'since')
+ base_time_string = string.lstrip(timeFormat[sinceLoc:])
+ # decode base time
+ # 9/25/2012: datetime.timedelta has problem with the argument because xtimes is NioVariable.
+ # Soln (J. Kim): use a temp variable ttmp in the for loop, then convert it into an integer xtime.
+ base_time = decodeTimeFromString(base_time_string)
+ times=[]
+ for ttmp in xtimes[:]:
+ xtime = int(ttmp)
+ if(units=='minutes'):
+ dt = datetime.timedelta(minutes=xtime); new_time = base_time + dt
+ if(units=='hours'):
+ dt = datetime.timedelta(hours=xtime); new_time = base_time + dt
+ if(units=='days'):
+ dt = datetime.timedelta(days=xtime); new_time = base_time + dt
+ if(units=='months'): # NB. adding months in python is complicated as month length varies and hence ambigous.
+ # Perform date arithmatic manually
+ # Assumption: the base_date will usually be the first of the month
+ # NB. this method will fail if the base time is on the 29th or higher day of month
+ # -as can't have, e.g. Feb 31st.
+ new_month = int(base_time.month + xtime % 12)
+ new_year = int(math.floor(base_time.year + xtime / 12.))
+ #print type(new_year),type(new_month),type(base_time.day),type(base_time.hour),type(base_time.second)
+ new_time = datetime.datetime(new_year,new_month,base_time.day,base_time.hour,base_time.second,0)
+ if(units=='years'):
+ dt = datetime.timedelta(years=xtime); new_time = base_time + dt
+ times.append(new_time)
+ return times
+
+
+def regrid_in_time(data,dateList,unit):
+ #################################################################################################
+ # Routine to calculate averages on longer time units than the data exists on.
+ # e.g. if the data is 6-hourly, calculate daily, or monthly means.
+ # Input:
+ # data - data values
+ # dateList - list of python datetime structures corresponding to data values
+ # unit - string describing time unit to average onto
+ # e.g. 'monthly', 'daily', 'pentad','annual','decadal'
+ # Output:
+ # meanstorem - numpy masked array of data values meaned over required time period
+ # newTimesList - a list of python datetime objects representing the data in the new averagin period
+ # NB. currently set to beginning of averaging period,
+ # i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
+ # ..............................
+ # Jinwon Kim, Sept 30, 2012
+ # Created from calc_average_on_new_time_unit_K, Peter's original, with the modification below:
+ # Instead of masked array used by Peter, use wh to defined data within the averaging range.
+ #################################################################################################
+
+ print '*** Begin calc_average_on_new_time_unit_KK ***'
# Check if the user-selected temporal grid is valid. If not, EXIT
acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
if not acceptable:
- print 'Error: unknown unit type selected for time averaging: EXIT'
- return -1,-1,-1,-1
-
- # Calculate arrays of: annual timeseries: year (2007,2007),
- # monthly time series: year-month (200701,200702),
- # daily timeseries: year-month-day (20070101,20070102)
- # depending on user-selected averaging period.
+ print 'Error: unknown unit type selected for time averaging: EXIT'; return -1,-1,-1,-1
- # Year list
- if unit=='annual':
+ # Calculate time arrays of: annual (yyyy, [2007]), monthly (yyyymm, [200701,200702]), daily (yyyymmdd, [20070101,20070102])
+ # "%02d" is similar to i2.2 in Fortran
+ if unit=='annual': # YYYY
timeunits = []
for i in np.arange(len(dateList)):
timeunits.append(str(dateList[i].year))
- timeunits = np.array(timeunits, dtype=int)
-
- # YearMonth format list
- if unit=='monthly':
+ elif unit=='monthly': # YYYYMM
timeunits = []
for i in np.arange(len(dateList)):
timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
- timeunits = np.array(timeunits,dtype=int)
-
- # YearMonthDay format list
- if unit=='daily':
+ elif unit=='daily': # YYYYMMDD
timeunits = []
for i in np.arange(len(dateList)):
timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
- timeunits = np.array(timeunits,dtype=int)
-
- # TODO: add pentad setting using Julian days?
-
- # Full list: a special case
- if unit == 'full':
+ elif unit=='full': # Full list: a special case
comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
timeunits = []
for i in np.arange(len(dateList)):
- timeunits.append(999) # i.e. we just want the same value for all times.
- timeunits = np.array(timeunits, dtype=int)
+ timeunits.append(999) # i.e. we just want the same value for all times.
+ timeunits = np.array(timeunits,dtype=int)
+ print 'timeRegridOption= ',unit#'; output timeunits= ',timeunits
+ #print 'timeRegridOption= ',unit,'; output timeunits= ',timeunits
- # empty list to store new times
+ # empty list to store time levels after temporal regridding
newTimesList = []
# Decide whether or not you need to do any time averaging.
- # i.e. if data are already on required time unit then just pass data through and
- # calculate and return representative datetimes.
+ # i.e. if data are already on required time unit then just pass data through and calculate and return representative datetimes.
processing_required = True
if len(timeunits)==(len(np.unique(timeunits))):
processing_required = False
+ print 'processing_required= ',processing_required,': input time steps= ',len(timeunits),': regridded output time steps = ',len(np.unique(timeunits))
+ #print 'np.unique(timeunits): ',np.unique(timeunits)
+ print 'data.ndim= ',data.ndim
- # 1D data arrays, i.e. time series
- if data.ndim==1:
- # Create array to store the resulting data
+ if data.ndim==1: # 1D data arrays, i.e. 1D time series
+ # Create array to store the temporally regridded data
meanstore = np.zeros(len(np.unique(timeunits)))
-
# Calculate the means across each unique time unit
i=0
for myunit in np.unique(timeunits):
if processing_required:
- datam=ma.masked_array(data,timeunits!=myunit)
+ wh = timeunits==myunit
+ datam = data[wh]
meanstore[i] = ma.average(datam)
-
- # construct new times list
+ # construct new times list
smyunit = str(myunit)
if len(smyunit)==4: # YYYY
yyyy = int(myunit[0:4])
@@ -1008,32 +960,21 @@ def calc_average_on_new_time_unit_K(data, dateList, unit):
yyyy = int(halfway.year)
mm = int(halfway.month)
dd = int(halfway.day)
-
newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
i = i+1
- # 3D data arrays
- if data.ndim==3:
- # datamask = create_mask_using_threshold(data,threshold=0.75)
+ elif data.ndim==3: # 3D data arrays, i.e. 2D time series
# Create array to store the resulting data
meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
-
# Calculate the means across each unique time unit
i=0
datamask_store = []
for myunit in np.unique(timeunits):
if processing_required:
- mask = np.zeros_like(data)
- mask[timeunits!=myunit,:,:] = 1.0
- # Calculate missing data mask within each time unit...
- datamask_at_this_timeunit = np.zeros_like(data)
- datamask_at_this_timeunit[:]= create_mask_using_threshold(data[timeunits==myunit,:,:],threshold=0.75)
- # Store results for masking later
- datamask_store.append(datamask_at_this_timeunit[0])
- # Calculate means for each pixel in this time unit, ignoring missing data (using masked array).
- datam = ma.masked_array(data,np.logical_or(mask,datamask_at_this_timeunit))
+ wh = timeunits==myunit
+ datam = data[wh,:,:]
meanstore[i,:,:] = ma.average(datam,axis=0)
- # construct new times list
+ # construct new times list
smyunit = str(myunit)
if len(smyunit)==4: # YYYY
yyyy = int(smyunit[0:4])
@@ -1059,226 +1000,8 @@ def calc_average_on_new_time_unit_K(data, dateList, unit):
if not processing_required:
meanstorem = data
+ elif processing_required:
+ meanstorem = meanstore
- if processing_required:
- # Create masked array (using missing data mask defined above)
- datamask_store = np.array(datamask_store)
- meanstorem = ma.masked_array(meanstore, datamask_store)
-
+ print '*** End calc_average_on_new_time_unit_KK ***'
return meanstorem,newTimesList
-
-def decode_model_timesK(ifile,timeVarName,file_type):
- #################################################################################################
- # Convert model times ('hours since 1900...', 'days since ...') into a python datetime structure
- # Input:
- # filelist - list of model files
- # timeVarName - name of the time variable in the model files
- # Output:
- # times - list of python datetime objects describing model data times
- # Peter Lean February 2011
- #################################################################################################
- f = Nio.open_file(ifile,mode='r',options=None,format=file_type)
- xtimes = f.variables[timeVarName]
- timeFormat = xtimes.attributes['units']
- #timeFormat = "days since 1979-01-01 00:00:00"
- # search to check if 'since' appears in units
- try:
- sinceLoc = re.search('since',timeFormat).end()
- except:
- print 'Error decoding model times: time var attributes do not contain "since": Exit'
- sys.exit()
- # search for 'seconds','minutes','hours', 'days', 'months', 'years' so know units
- # TODO: Swap out this section for a set of if...elseif statements
- units = ''
- try:
- _ = re.search('minutes',timeFormat).end()
- units = 'minutes'
- except:
- pass
- try:
- _ = re.search('hours',timeFormat).end()
- units = 'hours'
- except:
- pass
- try:
- _ = re.search('days',timeFormat).end()
- units = 'days'
- except:
- pass
- try:
- _ = re.search('months',timeFormat).end()
- units = 'months'
- except:
- pass
- try:
- _ = re.search('years',timeFormat).end()
- units = 'years'
- except:
- pass
- # cut out base time (the bit following 'since')
- base_time_string = string.lstrip(timeFormat[sinceLoc:])
- # decode base time
- # 9/25/2012: datetime.timedelta has problem with the argument because xtimes is NioVariable.
- # Soln (J. Kim): use a temp variable ttmp in the for loop, then convert it into an integer xtime.
- base_time = decodeTimeFromString(base_time_string)
- times=[]
- for ttmp in xtimes[:]:
- xtime = int(ttmp)
- if(units=='minutes'):
- dt = datetime.timedelta(minutes=xtime); new_time = base_time + dt
- if(units=='hours'):
- dt = datetime.timedelta(hours=xtime); new_time = base_time + dt
- if(units=='days'):
- dt = datetime.timedelta(days=xtime); new_time = base_time + dt
- if(units=='months'): # NB. adding months in python is complicated as month length varies and hence ambigous.
- # Perform date arithmatic manually
- # Assumption: the base_date will usually be the first of the month
- # NB. this method will fail if the base time is on the 29th or higher day of month
- # -as can't have, e.g. Feb 31st.
- new_month = int(base_time.month + xtime % 12)
- new_year = int(math.floor(base_time.year + xtime / 12.))
- #print type(new_year),type(new_month),type(base_time.day),type(base_time.hour),type(base_time.second)
- new_time = datetime.datetime(new_year,new_month,base_time.day,base_time.hour,base_time.second,0)
- if(units=='years'):
- dt = datetime.timedelta(years=xtime); new_time = base_time + dt
- times.append(new_time)
- return times
-
-
-def regrid_in_time(data,dateList,unit):
- #################################################################################################
- # Routine to calculate averages on longer time units than the data exists on.
- # e.g. if the data is 6-hourly, calculate daily, or monthly means.
- # Input:
- # data - data values
- # dateList - list of python datetime structures corresponding to data values
- # unit - string describing time unit to average onto
- # e.g. 'monthly', 'daily', 'pentad','annual','decadal'
- # Output:
- # meanstorem - numpy masked array of data values meaned over required time period
- # newTimesList - a list of python datetime objects representing the data in the new averagin period
- # NB. currently set to beginning of averaging period,
- # i.e. mean Jan 1st - Jan 31st -> represented as Jan 1st, 00Z.
- # ..............................
- # Jinwon Kim, Sept 30, 2012
- # Created from calc_average_on_new_time_unit_K, Peter's original, with the modification below:
- # Instead of masked array used by Peter, use wh to defined data within the averaging range.
- #################################################################################################
-
- print '*** Begin calc_average_on_new_time_unit_KK ***'
- # Check if the user-selected temporal grid is valid. If not, EXIT
- acceptable = (unit=='full')|(unit=='annual')|(unit=='monthly')|(unit=='daily')|(unit=='pentad')
- if not acceptable:
- print 'Error: unknown unit type selected for time averaging: EXIT'; return -1,-1,-1,-1
-
- # Calculate time arrays of: annual (yyyy, [2007]), monthly (yyyymm, [200701,200702]), daily (yyyymmdd, [20070101,20070102])
- # "%02d" is similar to i2.2 in Fortran
- if unit=='annual': # YYYY
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year))
- elif unit=='monthly': # YYYYMM
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month))
- elif unit=='daily': # YYYYMMDD
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(str(dateList[i].year) + str("%02d" % dateList[i].month) + str("%02d" % dateList[i].day))
- elif unit=='full': # Full list: a special case
- comment='Calculating means data over the entire time range: i.e., annual-mean climatology'
- timeunits = []
- for i in np.arange(len(dateList)):
- timeunits.append(999) # i.e. we just want the same value for all times.
- timeunits = np.array(timeunits,dtype=int)
- print 'timeRegridOption= ',unit#'; output timeunits= ',timeunits
- #print 'timeRegridOption= ',unit,'; output timeunits= ',timeunits
-
- # empty list to store time levels after temporal regridding
- newTimesList = []
-
- # Decide whether or not you need to do any time averaging.
- # i.e. if data are already on required time unit then just pass data through and calculate and return representative datetimes.
- processing_required = True
- if len(timeunits)==(len(np.unique(timeunits))):
- processing_required = False
- print 'processing_required= ',processing_required,': input time steps= ',len(timeunits),': regridded output time steps = ',len(np.unique(timeunits))
- #print 'np.unique(timeunits): ',np.unique(timeunits)
- print 'data.ndim= ',data.ndim
-
- if data.ndim==1: # 1D data arrays, i.e. 1D time series
- # Create array to store the temporally regridded data
- meanstore = np.zeros(len(np.unique(timeunits)))
- # Calculate the means across each unique time unit
- i=0
- for myunit in np.unique(timeunits):
- if processing_required:
- wh = timeunits==myunit
- datam = data[wh]
- meanstore[i] = ma.average(datam)
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit)==4: # YYYY
- yyyy = int(myunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit)==6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit)==8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit)==3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1]-dateList[0]
- halfway = dateList[0]+(dt/2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
- newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
- i = i+1
-
- elif data.ndim==3: # 3D data arrays, i.e. 2D time series
- # Create array to store the resulting data
- meanstore = np.zeros([len(np.unique(timeunits)),data.shape[1],data.shape[2]])
- # Calculate the means across each unique time unit
- i=0
- datamask_store = []
- for myunit in np.unique(timeunits):
- if processing_required:
- wh = timeunits==myunit
- datam = data[wh,:,:]
- meanstore[i,:,:] = ma.average(datam,axis=0)
- # construct new times list
- smyunit = str(myunit)
- if len(smyunit)==4: # YYYY
- yyyy = int(smyunit[0:4])
- mm = 1
- dd = 1
- if len(smyunit)==6: # YYYYMM
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = 1
- if len(smyunit)==8: # YYYYMMDD
- yyyy = int(smyunit[0:4])
- mm = int(smyunit[4:6])
- dd = int(smyunit[6:8])
- if len(smyunit)==3: # Full time range
- # Need to set an appropriate time representing the mid-point of the entire time span
- dt = dateList[-1]-dateList[0]
- halfway = dateList[0]+(dt/2)
- yyyy = int(halfway.year)
- mm = int(halfway.month)
- dd = int(halfway.day)
- newTimesList.append(datetime.datetime(yyyy,mm,dd,0,0,0,0))
- i += 1
-
- if not processing_required:
- meanstorem = data
- elif processing_required:
- meanstorem = meanstore
-
- print '*** End calc_average_on_new_time_unit_KK ***'
- return meanstorem,newTimesList
http://git-wip-us.apache.org/repos/asf/climate/blob/dc091596/mccsearch/docs/mccsearch.md
----------------------------------------------------------------------
diff --git a/mccsearch/docs/mccsearch.md b/mccsearch/docs/mccsearch.md
index 466e027..6b3ab1a 100644
--- a/mccsearch/docs/mccsearch.md
+++ b/mccsearch/docs/mccsearch.md
@@ -9,14 +9,14 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
##What is needed?
* Python 2.7.4 (We used Anaconda 1.5.1 64 bit libraries, which installed most of the dependencies.) Other module dependences:
- * Nio
- * netCDF
- * sciPy
- * NumPy
- * Networkx
- * matplotlib
- * GrADS (We used OpenGrADS grads2 Version 2.0.1.oga.1)
- * LATS4D
+ * Nio - https://www.pyngl.ucar.edu/Nio.shtml
+ * netCDF - http://www.unidata.ucar.edu/software/netcdf/
+ * sciPy - http://www.scipy.org/scipylib/index.html
+ * NumPy - http://www.scipy.org/scipylib/download.html
+ * Networkx - https://networkx.github.io/download.html
+ * matplotlib - http://matplotlib.org/downloads.html
+ * GrADS (We used OpenGrADS grads2 Version 2.0.1.oga.1) - http://sourceforge.net/projects/opengrads/files/
+ * LATS4D - http://sourceforge.net/projects/opengrads/files/
##Download the source code and store in a folder
* mccSearch.py contains all the function needed
@@ -24,7 +24,7 @@ The data is read from netCDF files into arrays with the dimensions time, latitud
* process.py contains some needed functions (from older version of Apache OCW)
* file.py contains some needed functions (from older version of Apache OCW)
* mainProg.py contains a sample of the general workflow of the order the modules should be called. There are three main inputs you will have to supply:
- * mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be store
+ * mainDirStr : this is the directory where you wish all the output files –images, textfiles, clipped netCDF files- to be stored
* TRMMdirName : this is the directory where the original TRMM data in netCDF format is stored
* CEoriDirName : this is the directory where the original MERG data in netCDF format is stored
* Store the GrADsScripts folder (and contents) in a separate folder in the same main directory as the source code folder.
[37/50] [abbrv] remove unnecessary files
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/climate/blob/c1e22a93/mccsearch/code/mccSearchGoyens.py
----------------------------------------------------------------------
diff --git a/mccsearch/code/mccSearchGoyens.py b/mccsearch/code/mccSearchGoyens.py
deleted file mode 100644
index 0f49a10..0000000
--- a/mccsearch/code/mccSearchGoyens.py
+++ /dev/null
@@ -1,3606 +0,0 @@
-'''
-# All the functions for the MCC search algorithm
-# RCMES data in format (t,lat,lon), value
-# Kim Whitehall Sep 2012
-# Last updated: 23Jan 2014, 6 Dec 2013, 21 May 2013, 4 Jun 2013
-# June 13 - called this merg.py and edited out comments. merg.py is where all "cleaned" functions are to go
-# June 27 - including the first part of graphing, according to the center_of_mass
-# July 1 - going back to brute force graph approach, but weigh the edges according to closeness, up to within 20_perc of the forecasted
-#
-
-Mimumum MCS is 3 hours
-'''
-
-import datetime
-from datetime import timedelta, datetime
-import calendar
-import fileinput
-import glob
-import itertools
-import json
-import math
-import Nio
-from netCDF4 import Dataset, num2date, date2num
-import numpy as np
-import numpy.ma as ma
-import os
-import pickle
-import re
-from scipy import ndimage
-import string
-import subprocess
-import sys
-import time
-
-import networkx as nx
-import matplotlib.pyplot as plt
-import matplotlib.dates as mdates
-from matplotlib.dates import DateFormatter,HourLocator
-from matplotlib import cm
-import matplotlib.cm as cm
-import matplotlib.colors as mcolors
-from matplotlib.ticker import FuncFormatter, FormatStrFormatter
-
-#existing modules in services
-import files
-import process
-#----------------------- GLOBAL VARIABLES --------------------------
-# --------------------- User defined variables ---------------------
-#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
-#the first point closest the the value (for the min) from the MERG data is used, etc.
-LATMIN = '5.0' #'5.0' #'12.0' #'-40' #'12.0' #'10.0' #'-40.0' #'-5.0' #min latitude; -ve values in the SH e.g. 5S = -5
-LATMAX = '20.0' #'20.0' #'17.0' #'-20.0' #'17.0' #'20.0' #'30.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
-LONMIN = '0.0' #'-8.0' #'10.0' #'-8.0' #'-5.0' #'10.0' #'-40.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
-LONMAX = '30.0' #'8.0' #'40.0' #'8.0' #'40.0' #'5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
-XRES = 4.0 #x direction spatial resolution in km
-YRES = 4.0 #y direction spatial resolution in km
-TRES = 1 #temporal resolution in hrs
-T_BB_MAX = 243 #241 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
-T_BB_MIN = 218 #221 #cooler temp for the center of the system
-STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
- #have same rank of the matrix it is being compared against
-CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
-MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
-MIN_CE_SPEED = 45.0 #the slowest speed a CE can move btwn F for the resolved data in kmhr^-1.here is hrly, therefore val
-MAX_CE_SPEED = 70.0 #the fastest speed a CE can move btwn F for the resolved data in kmhr^-1.
-LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
-LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
-DIRECTION = 45.0 #general direction of the wind flow in degrees
-AREA_MIN = 3500.0 #2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
-MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
-
-#---Assuming using the MCC function, these will have to be changed
-ECCENTRICITY_THRESHOLD_MAX = 0.75 #tending to 1 is a circle e.g. hurricane,
-ECCENTRICITY_THRESHOLD_MIN = 0.25 #0.65 #tending to 0 is a linear e.g. squall line
-OUTER_CLOUD_SHIELD_AREA = 30000.0 #80000.0 #100000.0 #km^2
-INNER_CLOUD_SHIELD_AREA = 3000.0#30000.0 #50000.0 #km^2
-OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
-INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
-MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, African MCCs is 6hrs)
-MAXIMUM_DURATION = 100 #24 #max number of framce the MCC can last for (assuming hrly frames, African MCCs last at most 15hrs)
-
-#MAINDIRECTORY = "/Users/kimwhitehall/Documents/HU/research/mccsearch/caseStudy1"
-#------------------- End user defined Variables -------------------
-edgeWeight = [1,2,3] #weights for the graph edges
-#graph object fo the CEs meeting the criteria
-CLOUD_ELEMENT_GRAPH = nx.DiGraph()
-#graph meeting the CC criteria
-PRUNED_GRAPH = nx.DiGraph()
-#------------------------ End GLOBAL VARS -------------------------
-#************************ Begin Functions *************************
-#******************************************************************
-def readMergData(dirname):
- '''
- Purpose::
- Read MERG data into RCMES format
-
- Input::
- Directory to the MERG files in NETCDF format
-
- Output::
- A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
- criteria for each frame
- **remove below**
- A dictionary of all the MERG data from the files in the directory given.
- The dictionary contains, a string representing the time (a datetime string), a
- tuple (lat,lon,value) representing only the data that meets the temperature requirements
- i.e. T_BB_MAX
-
- Assumptions::
- The MERG data has been converted to NETCDF using LATS4D
- The data has the same lat/lon format
- '''
-
- global LAT
- global LON
-
- # these strings are specific to the MERG data
- mergVarName = 'ch4'
- mergTimeVarName = 'time'
- mergLatVarName = 'latitude'
- mergLonVarName = 'longitude'
-
- filelistInstructions = dirname + '/*'
- filelist = glob.glob(filelistInstructions)
-
-
- #sat_img is the array that will contain all the masked frames
- mergImgs = []
- #timelist of python time strings
- timelist = []
- time2store = None
- tempMaskedValueNp =[]
-
-
- filelist.sort()
- nfiles = len(filelist)
-
- # Crash nicely if there are no netcdf files
- if nfiles == 0:
- print 'Error: no files in this directory! Exiting elegantly'
- sys.exit()
- else:
- # Open the first file in the list to read in lats, lons and generate the grid for comparison
- tmp = Nio.open_file(filelist[0], format='nc')
- #TODO: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
-
- #clip the lat/lon grid according to user input
- #http://www.pyngl.ucar.edu/NioExtendedSelection.shtml
- latsraw = tmp.variables[mergLatVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX].astype('f2')
- lonsraw = tmp.variables[mergLonVarName][mergLonVarName+"|"+LONMIN+":"+LONMAX].astype('f2')
- lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
-
- LON, LAT = np.meshgrid(lonsraw, latsraw)
- #clean up
- latsraw =[]
- lonsraw = []
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
- #print 'Lats and lons read in for first file in filelist',nygrd,nxgrd
- tmp.close
-
- for files in filelist:
- try:
- thisFile = Nio.open_file(files, format='nc')
- #TODO: thisFile = netCDF4.Dataset(files)
-
- #clip the dataset according to user lat,lon coordinates
- #mask the data and fill with zeros for later
- tempRaw = thisFile.variables[mergVarName][mergLatVarName+"|"+LATMIN+":"+LATMAX \
- +" " +mergLonVarName+"|"+LONMIN+":"+LONMAX ].astype('int16')
-
- tempMask = ma.masked_array(tempRaw, mask=(tempRaw > T_BB_MAX), fill_value=0)
-
- #get the actual values that the mask returned
- tempMaskedValue = ma.zeros((tempRaw.shape)).astype('int16')
- for index, value in maenumerate(tempMask):
- time_index, lat_index, lon_index = index
- tempMaskedValue[time_index,lat_index, lon_index]=value
-
- timesRaw = thisFile.variables[mergTimeVarName]
- #convert this time to a python datastring
- time2store, _ = process.getModelTimes(files, mergTimeVarName)
- #extend instead of append because getModelTimes returns a list already and we don't
- #want a list of list
- timelist.extend(time2store)
- #to get the actual masked data only
- #http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html#accessing-only-the-valid-entries
- mergImgs.extend(tempMaskedValue)
- thisFile.close
- thisFile = None
-
- except:
- print "bad file! ", file
-
- mergImgs = ma.array(mergImgs)
-
- return mergImgs, timelist
-#******************************************************************
-def findCloudElements(mergImgs,timelist,TRMMdirName=None):
- '''
- Purpose::
- Determines the contiguous boxes for a given time of the satellite imaages i.e. each frame
- It recursively calls another function to determine if the boxes are touching
-
- Input::
- 2 variables
- sat_img: masked numpy array in (time,lat,lon),T_bb representing the satellite data. This is masked based on the
- maximum acceptable temperature, T_BB_MAX
- timelist: a list of python datatimes
- TRMMdirName (optional): string representing the path where to find the TRMM datafiles
-
- Output::
- 1 variable
- cloudEements: list of dictionary of cloudElements which have met the temperature, area and shape requirements
- The dictionary is cloudElements={"frame": datetime object,
- "cloudElementNum": integer,
- "CEuniqueID": string,
- "cloudElementLatLon": 2D array of lat_lon}
- "cloudElementCenter": 1D array [0] is lat, [1]is lon of the center_of_mass}
- "cloudElementEccentricity: float"
-
- Assumptions::
- Assumes we are dealing with MERG data which is 4kmx4km resolved, thus the smallest value
- required according to Vila et al. (2008) is 2400km^2
- therefore, 2400/16 = 150 contiguous squares
- '''
-
- frame = ma.empty((1,mergImgs.shape[1],mergImgs.shape[2]))
- CEcounter = 0
- frameCEcounter = 0
- frameNum = 0
- cloudElementEpsilon = 0.0
- cloudElementDict = {}
- cloudElementCenter = [] #list with two elements [lat,lon] for the center of a CE
- prevFrameCEs = [] #list for CEs in previous frame
- currFrameCEs = [] #list for CEs in current frame
- cloudElementLat = [] #list for a particular CE's lat values
- cloudElementLon = [] #list for a particular CE's lon values
- cloudElementLatLons = [] #list for a particular CE's (lat,lon) values
-
- prevLatValue = 0.0
- prevLonValue = 0.0
- TIR_min = 0.0
- TIR_max = 0.0
- temporalRes = 3 # TRMM data is 3 hourly
- precipTotal = 0.0
- CETRMMList =[]
- precip =[]
- TRMMCloudElementLatLons =[]
-
- #edgeWeight = [1,2]
- minCELatLimit = 0.0
- minCELonLimit = 0.0
- maxCELatLimit = 0.0
- maxCELonLimit = 0.0
-
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
-
- #openfile for storing ALL cloudElement information
- cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
-
-
- #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
- cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
- #cloudElementsTextFile = open('/Users/kimwhitehall/Documents/HU/research/mccsearch/cloudElementsTextFile.txt','w')
-
- #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
- for t in xrange(mergImgs.shape[0]):
- #-------------------------------------------------
- # #textfile name for saving the data for arcgis
- # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt'
- # cloudElementsTextFile = open(thisFileName,'w')
- #-------------------------------------------------
-
- #determine contiguous locations with temeperature below the warmest temp i.e. cloudElements in each frame
- frame, CEcounter = ndimage.measurements.label(mergImgs[t,:,:], structure=STRUCTURING_ELEMENT)
- frameCEcounter=0
- frameNum += 1
-
- #for each of the areas identified, check to determine if it a valid CE via an area and T requirement
- for count in xrange(CEcounter):
- #[0] is time dimension. Determine the actual values from the data
- #loc is a masked array
- loc = ndimage.find_objects(frame==(count+1))[0]
- cloudElement = mergImgs[t,:,:][loc]
- labels, lcounter = ndimage.label(cloudElement)
-
- #determine the true lats and lons for this particular CE
- cloudElementLat = LAT[loc[0],0]
- cloudElementLon = LON[0,loc[1]]
-
- #determine number of boxes in this cloudelement
- numOfBoxes = np.count_nonzero(cloudElement)
- cloudElementArea = numOfBoxes*XRES*YRES
-
- #If the area is greater than the area required, then start considering as CE
- #TODO: if the area is smaller than the suggested area, check if it meets a convective fraction requirement
- #if it does, still consider as CE
-
- if cloudElementArea >= AREA_MIN or (cloudElementArea < AREA_MIN and ((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels)))) < CONVECTIVE_FRACTION ):
- #do a second pass on the identified area
- # if lcounter > 1:
- # #i.e.there is more than one item in the area identified
- # for lcount in xrange(lcounter):
- # print " in lcount for", lcount
- # loc = ndimage.find_objects(labels ==(lcount+1))[0]
- # lcloudElement = cloudElement[:,:][loc]
- # llabels, _ = ndimage.label(lcloudElement)
- # plt.imshow(llabels)
- # plt.show()
-
-
- #get some time information and labeling info
- frameTime = str(timelist[t])
- frameCEcounter +=1
- CEuniqueID = 'F'+str(frameNum)+'CE'+str(frameCEcounter)
-
- #-------------------------------------------------
- #textfile name for accesing CE data using MATLAB code
- # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.txt'
- # cloudElementsTextFile = open(thisFileName,'w')
- #-------------------------------------------------
-
- # ------ NETCDF File stuff for brightness temp stuff ------------------------------------
- thisFileName = MAINDIRECTORY +'/MERGnetcdfCEs/cloudElements'+ (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
- currNetCDFCEData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFCEData.description = 'Cloud Element '+CEuniqueID + ' temperature data'
- currNetCDFCEData.calendar = 'standard'
- currNetCDFCEData.conventions = 'COARDS'
- # dimensions
- currNetCDFCEData.createDimension('time', None)
- currNetCDFCEData.createDimension('lat', len(LAT[:,0]))
- currNetCDFCEData.createDimension('lon', len(LON[0,:]))
- # variables
- tempDims = ('time','lat', 'lon',)
- times = currNetCDFCEData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitudes = currNetCDFCEData.createVariable('latitude', 'f8', ('lat',))
- longitudes = currNetCDFCEData.createVariable('longitude', 'f8', ('lon',))
- brightnesstemp = currNetCDFCEData.createVariable('brightnesstemp', 'i16',tempDims )
- brightnesstemp.units = 'Kelvin'
- # NETCDF data
- dates=[timelist[t]+timedelta(hours=0)]
- times[:] = date2num(dates,units=times.units)
- longitudes[:] = LON[0,:]
- longitudes.units = "degrees_east"
- longitudes.long_name = "Longitude"
-
- latitudes[:] = LAT[:,0]
- latitudes.units = "degrees_north"
- latitudes.long_name ="Latitude"
-
- #generate array of zeros for brightness temperature
- brightnesstemp1 = ma.zeros((1,len(latitudes), len(longitudes))).astype('int16')
- #-----------End most of NETCDF file stuff ------------------------------------
-
- #if other dataset (TRMM) assumed to be a precipitation dataset was entered
- if TRMMdirName:
- #------------------TRMM stuff -------------------------------------------------
- fileDate = ((str(timelist[t])).replace(" ", "")[:-8]).replace("-","")
- fileHr1 = (str(timelist[t])).replace(" ", "")[-8:-6]
-
- if int(fileHr1) % temporalRes == 0:
- fileHr = fileHr1
- else:
- fileHr = (int(fileHr1)/temporalRes) * temporalRes
- if fileHr < 10:
- fileHr = '0'+str(fileHr)
- else:
- str(fileHr)
-
- #open TRMM file for the resolution info and to create the appropriate sized grid
- TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+str(fileHr)+".7A.nc"
-
- TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
- precipRate = TRMMData.variables['pcp'][:,:,:]
- latsrawTRMMData = TRMMData.variables['latitude'][:]
- lonsrawTRMMData = TRMMData.variables['longitude'][:]
- lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
- LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
-
- nygrdTRMM = len(LATTRMM[:,0]); nxgrdTRMM = len(LONTRMM[0,:])
- precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
- #---------regrid the TRMM data to the MERG dataset ----------------------------------
- #regrid using the do_regrid stuff from the Apache OCW
- regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
- regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
- #----------------------------------------------------------------------------------
-
- # #get the lat/lon info from cloudElement
- #get the lat/lon info from the file
- latCEStart = LAT[0][0]
- latCEEnd = LAT[-1][0]
- lonCEStart = LON[0][0]
- lonCEEnd = LON[0][-1]
-
- #get the lat/lon info for TRMM data (different resolution)
- latStartT = find_nearest(latsrawTRMMData, latCEStart)
- latEndT = find_nearest(latsrawTRMMData, latCEEnd)
- lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
- lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
- latStartIndex = np.where(latsrawTRMMData == latStartT)
- latEndIndex = np.where(latsrawTRMMData == latEndT)
- lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
- lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
-
- #get the relevant TRMM info
- CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
- TRMMData.close()
-
- # ------ NETCDF File info for writing TRMM CE rainfall ------------------------------------
- thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/TRMM' + (str(timelist[t])).replace(" ", "_") + CEuniqueID +'.nc'
- currNetCDFTRMMData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFTRMMData.description = 'Cloud Element '+CEuniqueID + ' precipitation data'
- currNetCDFTRMMData.calendar = 'standard'
- currNetCDFTRMMData.conventions = 'COARDS'
- # dimensions
- currNetCDFTRMMData.createDimension('time', None)
- currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
- currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
-
- # variables
- TRMMprecip = ('time','lat', 'lon',)
- times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
- longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
- rainFallacc.units = 'mm'
-
- longitude[:] = LON[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
-
- latitude[:] = LAT[:,0]
- latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
-
- finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
- #-----------End most of NETCDF file stuff ------------------------------------
-
- #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
- for index,value in np.ndenumerate(cloudElement):
- if value != 0 :
- lat_index,lon_index = index
- lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value)
-
- #generate the comma separated file for GIS
- cloudElementLatLons.append(lat_lon_tuple)
-
- #temp data for CE NETCDF file
- brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value
-
- if TRMMdirName:
- finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])]
- CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]]))
-
-
- brightnesstemp[:] = brightnesstemp1[:]
- currNetCDFCEData.close()
-
- if TRMMdirName:
-
- #print finalCETRMMvalues
- #calculate the total precip associated with the feature
- for index, value in np.ndenumerate(finalCETRMMvalues):
- precipTotal += value
- precip.append(value)
-
- rainFallacc[:] = finalCETRMMvalues[:]
- currNetCDFTRMMData.close()
- TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
- TRMMArea = TRMMnumOfBoxes*XRES*YRES
- try:
- maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
- minCEprecipRate = np.min(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
- except:
- pass
-
- #sort cloudElementLatLons by lats
- cloudElementLatLons.sort(key=lambda tup: tup[0])
-
- #determine if the cloud element the shape
- cloudElementEpsilon = eccentricity (cloudElement)
- cloudElementsUserFile.write("\n\nTime is: %s" %(str(timelist[t])))
- cloudElementsUserFile.write("\nCEuniqueID is: %s" %CEuniqueID)
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
-
- #latCenter and lonCenter are given according to the particular array defining this CE
- #so you need to convert this value to the overall domain truth
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsUserFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementCenter.append(latCenter)
- cloudElementCenter.append(lonCenter)
- cloudElementsUserFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsUserFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
- cloudElementsUserFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
- cloudElementsUserFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
- cloudElementsUserFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
- #populate the dictionary
- if TRMMdirName:
- cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min, 'cloudElementPrecipTotal':precipTotal,'cloudElementLatLonTRMM':CETRMMList, 'TRMMArea': TRMMArea}
- else:
- cloudElementDict = {'uniqueID': CEuniqueID, 'cloudElementTime': timelist[t],'cloudElementLatLon': cloudElementLatLons, 'cloudElementCenter':cloudElementCenter, 'cloudElementArea':cloudElementArea, 'cloudElementEccentricity':cloudElementEpsilon, 'cloudElementTmax':TIR_max, 'cloudElementTmin': TIR_min,}
-
- #current frame list of CEs
- currFrameCEs.append(cloudElementDict)
-
- #draw the graph node
- CLOUD_ELEMENT_GRAPH.add_node(CEuniqueID, cloudElementDict)
-
- if frameNum != 1:
- for cloudElementDict in prevFrameCEs:
- thisCElen = len(cloudElementLatLons)
- percentageOverlap, areaOverlap = cloudElementOverlap(cloudElementLatLons, cloudElementDict['cloudElementLatLon'])
-
- #change weights to integers because the built in shortest path chokes on floating pts
- #according to Goyens et al, two CEs are considered related if there is atleast 95% overlap between them for consecutive imgs a max of 2 hrs apart
- if percentageOverlap >= 0.95:
- CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[0])
-
- elif percentageOverlap >= 0.90 and percentageOverlap < 0.95 :
- CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[1])
-
- elif areaOverlap >= MIN_OVERLAP:
- CLOUD_ELEMENT_GRAPH.add_edge(cloudElementDict['uniqueID'], CEuniqueID, weight=edgeWeight[2])
-
- else:
- #TODO: remove this else as we only wish for the CE details
- #ensure only the non-zero elements are considered
- #store intel in allCE file
- labels, _ = ndimage.label(cloudElement)
- cloudElementsFile.write("\n-----------------------------------------------")
- cloudElementsFile.write("\n\nTime is: %s" %(str(timelist[t])))
- # cloudElementLat = LAT[loc[0],0]
- # cloudElementLon = LON[0,loc[1]]
-
- #populate cloudElementLatLons by unpacking the original values from loc
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
- # as cloudElement is masked
- for index,value in np.ndenumerate(cloudElement):
- if value != 0 :
- lat_index,lon_index = index
- lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index])
- cloudElementLatLons.append(lat_lon_tuple)
-
- cloudElementsFile.write("\nLocation of rejected CE (lat,lon) points are: %s" %cloudElementLatLons)
- #latCenter and lonCenter are given according to the particular array defining this CE
- #so you need to convert this value to the overall domain truth
- latCenter, lonCenter = ndimage.measurements.center_of_mass(cloudElement, labels=labels)
- latCenter = cloudElementLat[round(latCenter)]
- lonCenter = cloudElementLon[round(lonCenter)]
- cloudElementsFile.write("\nCenter (lat,lon) is: %.2f\t%.2f" %(latCenter, lonCenter))
- cloudElementsFile.write("\nNumber of boxes are: %d" %numOfBoxes)
- cloudElementsFile.write("\nArea is: %.4f km^2" %(cloudElementArea))
- cloudElementsFile.write("\nAverage brightness temperature is: %.4f K" %ndimage.mean(cloudElement, labels=labels))
- cloudElementsFile.write("\nMin brightness temperature is: %.4f K" %ndimage.minimum(cloudElement, labels=labels))
- cloudElementsFile.write("\nMax brightness temperature is: %.4f K" %ndimage.maximum(cloudElement, labels=labels))
- cloudElementsFile.write("\nBrightness temperature variance is: %.4f K" %ndimage.variance(cloudElement, labels=labels))
- cloudElementsFile.write("\nConvective fraction is: %.4f " %(((ndimage.minimum(cloudElement, labels=labels))/float((ndimage.maximum(cloudElement, labels=labels))))*100.0))
- cloudElementsFile.write("\nEccentricity is: %.4f " %(cloudElementEpsilon))
- cloudElementsFile.write("\n-----------------------------------------------")
-
- #reset list for the next CE
- nodeExist = False
- cloudElementCenter=[]
- cloudElement = []
- cloudElementCenter = []
- cloudElementLat=[]
- cloudElementLon =[]
- cloudElementLatLons =[]
- brightnesstemp1 =[]
- brightnesstemp =[]
- finalCETRMMvalues =[]
- CEprecipRate =[]
- CETRMMList =[]
- precipTotal = 0.0
- precip=[]
- TRMMCloudElementLatLons=[]
-
- #reset for the next time
- prevFrameCEs =[]
- prevFrameCEs = currFrameCEs
- currFrameCEs =[]
-
- cloudElementsFile.close
- cloudElementsUserFile.close
- #if using ARCGIS data store code, uncomment this file close line
- #cloudElementsTextFile.close
-
- #clean up graph - remove parent and childless nodes
- outAndInDeg = CLOUD_ELEMENT_GRAPH.degree_iter()
- toRemove = [node[0] for node in outAndInDeg if node[1]<1]
- CLOUD_ELEMENT_GRAPH.remove_nodes_from(toRemove)
-
- print "number of nodes are: ", CLOUD_ELEMENT_GRAPH.number_of_nodes()
- print "number of edges are: ", CLOUD_ELEMENT_GRAPH.number_of_edges()
- print ("*"*80)
-
- #hierachial graph output
- #graphTitle = "Cloud Elements observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Elements observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z" # Niamey, Niger"
- drawGraph(CLOUD_ELEMENT_GRAPH, graphTitle, edgeWeight)
-
- return CLOUD_ELEMENT_GRAPH
-#******************************************************************
-def findPrecipRate(TRMMdirName):
- '''
- TODO: needs fixing
- Purpose:: Determines the precipitation rates for MCSs found
-
- Input:: TRMMdirName: a string representing the directory for the original TRMM netCDF files
-
- Output:: a list of dictionary of the TRMM data
- NB: also creates netCDF with TRMM data for each CE (for post processing) index
- in MAINDIRECTORY/TRMMnetcdfCEs
-
- Assumptions:: Assumes that findCloudElements was run without the TRMMdirName value
-
- '''
- allCEnodesTRMMdata =[]
- TRMMdataDict={}
- precipTotal = 0.0
-
- os.chdir((MAINDIRECTORY+'/MERGnetcdfCEs/'))
- imgFilename = ''
- temporalRes = 3 #3 hours for TRMM
-
- #sort files
- files = filter(os.path.isfile, glob.glob("*.nc"))
- files.sort(key=lambda x: os.path.getmtime(x))
-
- for afile in files:
- #try:
- print "in for ", afile
- fullFname = os.path.splitext(afile)[0]
- noFrameExtension = (fullFname.replace("_","")).split('F')[0]
- CEuniqueID = 'F' +(fullFname.replace("_","")).split('F')[1]
- fileDateTimeChar = (noFrameExtension.replace(":","")).split('s')[1]
- fileDateTime = fileDateTimeChar.replace("-","")
- fileDate = fileDateTime[:-6]
- fileHr=fileDateTime[-6:-4]
-
- cloudElementData = Dataset(afile,'r', format='NETCDF4')
- brightnesstemp = cloudElementData.variables['brightnesstemp'][:]
- latsrawCloudElements = cloudElementData.variables['latitude'][:]
- lonsrawCloudElements = cloudElementData.variables['longitude'][:]
-
- # print "noFrameExtension", noFrameExtension
- # print "fileDateTime ", fileDateTime
- # print "fileDateTimeChar ", fileDateTimeChar
- # print "fullFname ", fullFname
- # print "time is: ", (fullFname.replace("_"," ").split('F')[0]).split('s')[1]
- # print "fileDate and fileHr ", fileDate, fileHr
- # print "CEuniqueID ", CEuniqueID
- # sys.exit()
- if int(fileHr) % 3 == 0:
- TRMMfileName = TRMMdirName+'/3B42.'+ fileDate + "."+fileHr+".7A.nc"
- #print "TRMMfileName is ", TRMMfileName
- TRMMData = Dataset(TRMMfileName,'r', format='NETCDF4')
- precipRate = TRMMData.variables['pcp'][:,:,:]
- latsrawTRMMData = TRMMData.variables['latitude'][:]
- lonsrawTRMMData = TRMMData.variables['longitude'][:]
- lonsrawTRMMData[lonsrawTRMMData > 180] = lonsrawTRMMData[lonsrawTRMMData>180] - 360.
- LONTRMM, LATTRMM = np.meshgrid(lonsrawTRMMData, latsrawTRMMData)
-
- #nygrdTRMM = len(LATTRMM[:,0]); nxgrd = len(LONTRMM[0,:])
- nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
-
- precipRateMasked = ma.masked_array(precipRate, mask=(precipRate < 0.0))
- #---------regrid the TRMM data to the MERG dataset ----------------------------------
- #regrid using the do_regrid stuff from the Apache OCW
- regriddedTRMM = ma.zeros((0, nygrd, nxgrd))
- regriddedTRMM = process.do_regrid(precipRateMasked[0,:,:], LATTRMM, LONTRMM, LAT, LON, order=1, mdi= -999999999)
- #----------------------------------------------------------------------------------
-
- # #get the lat/lon info from cloudElement
- latCEStart = LAT[0][0]
- latCEEnd = LAT[-1][0]
- lonCEStart = LON[0][0]
- lonCEEnd = LON[0][-1]
-
- #get the lat/lon info for TRMM data (different resolution)
- latStartT = find_nearest(latsrawTRMMData, latCEStart)
- latEndT = find_nearest(latsrawTRMMData, latCEEnd)
- lonStartT = find_nearest(lonsrawTRMMData, lonCEStart)
- lonEndT = find_nearest(lonsrawTRMMData, lonCEEnd)
- latStartIndex = np.where(latsrawTRMMData == latStartT)
- latEndIndex = np.where(latsrawTRMMData == latEndT)
- lonStartIndex = np.where(lonsrawTRMMData == lonStartT)
- lonEndIndex = np.where(lonsrawTRMMData == lonEndT)
-
- #get the relevant TRMM info
- CEprecipRate = precipRate[:,(latStartIndex[0][0]-1):latEndIndex[0][0],(lonStartIndex[0][0]-1):lonEndIndex[0][0]]
- TRMMData.close()
-
-
-
- # ------ NETCDF File stuff ------------------------------------
- thisFileName = MAINDIRECTORY+'/TRMMnetcdfCEs/'+ fileDateTime + CEuniqueID +'.nc'
- # print "thisFileName ", thisFileName
- # sys.exit()
- currNetCDFData = Dataset(thisFileName, 'w', format='NETCDF4')
- currNetCDFData.description = 'Cloud Element '+CEuniqueID + ' rainfall data'
- currNetCDFData.calendar = 'standard'
- currNetCDFData.conventions = 'COARDS'
- # dimensions
- currNetCDFData.createDimension('time', None)
- currNetCDFData.createDimension('lat', CEprecipRate.shape[1])
- currNetCDFData.createDimension('lon', CEprecipRate.shape[2])
- # variables
- TRMMprecip = ('time','lat', 'lon',)
- times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
- longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
- rainFallacc.units = 'mm'
-
- longitude[:] = LON[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
-
- latitude[:] = LAT[:,0]
- latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
-
- finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
- #-----------End most of NETCDF file stuff ------------------------------------
- #finalCETRMMvalues = ma.zeros((CEprecipRate.shape))
- for index,value in np.ndenumerate(brightnesstemp):
- #print "index, value ", index, value
- time_index, lat_index, lon_index = index
- currTimeValue = 0
- if value > 0:
- finalCETRMMvalues[0,int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==brightnesstemp[time_index,lat_index])[0]),int(np.where(LON[0,:]==brightnesstemp[time_index,lon_index])[0])]
-
-
- # #print "lat_index and value ", lat_index, latsrawCloudElements[lat_index]
- # currLatvalue = find_nearest(latsrawTRMMData, latsrawCloudElements[lat_index])
- # currLonValue = find_nearest(lonsrawTRMMData ,lonsrawCloudElements[lon_index])
- # currLatIndex = np.where(latsrawTRMMData==currLatvalue)[0][0]-latStartIndex[0][0]
- # currLonIndex = np.where(lonsrawTRMMData==currLonValue)[0][0]- lonStartIndex[0][0]
- # #print "currLatIndex , currLonIndex ", currLatIndex , currLonIndex
- # finalCETRMMvalues[time_index,currLatIndex , currLonIndex] = value * TRES*1.0 #because the rainfall TRMM is mm/hr
- # # if currLatvalue != prevLatValue and currLonValue != prevLonValue:
- # # precipTotal = value*temporalRes*1.0
- # # prevLatValue = currLatvalue
- # # prevLonValue = currLonValue
-
- TRMMnumOfBoxes = np.count_nonzero(finalCETRMMvalues)
- TRMMArea = TRMMnumOfBoxes*XRES*YRES
- rainFallacc[:] = finalCETRMMvalues
- currNetCDFData.close()
- for index, value in np.ndenumerate(finalCETRMMvalues):
- #print "index, value ", index, value
- precipTotal += value
-
- # #add info to the dictionary and to the list
- # TRMMdataDict = {'uniqueID': CEuniqueID, 'cloudElementTime': (fullFname.replace("_"," ").split('F')[0]).split('s')[1],'cloudElementLatLon': finalCETRMMvalues, 'cloudElementPrecipTotal':precipTotal}
- # allCEnodesTRMMdata.append(TRMMdataDict)
- # print "precipTotal is ", precipTotal
- #add info to CLOUDELEMENTSGRAPH
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
- if eachdict[1]['uniqueID'] == CEuniqueID:
- if not 'cloudElementPrecipTotal' in eachdict[1].keys():
- eachdict[1]['cloudElementPrecipTotal'] = precipTotal
- if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
- eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
- if not 'TRMMArea' in eachdict[1].keys():
- eachdict[1]['TRMMArea'] = TRMMArea
- #clean up
- precipTotal = 0.0
- latsrawTRMMData =[]
- lonsrawTRMMData = []
- latsrawCloudElements=[]
- lonsrawCloudElements=[]
- finalCETRMMvalues =[]
- CEprecipRate =[]
- brightnesstemp =[]
- TRMMdataDict ={}
-
- return allCEnodesTRMMdata
-#******************************************************************
-def findCloudClusters(CEGraph):
- '''
- Purpose:: Determines the cloud clusters properties from the subgraphs in
- the graph i.e. prunes the graph according to the minimum depth
-
- Input:: 1 graph - CEGraph: a directed graph of the CEs with weighted edges
- according the area overlap between nodes (CEs) of consectuive frames
-
- Output:: a dictionary of all possible cloudClusters
-
- '''
-
- seenNode = []
- allMCSLists =[]
- pathDictList =[]
- pathList=[]
-
- cloudClustersFile = open((MAINDIRECTORY+'/textFiles/cloudClusters.txt'),'wb')
-
- for eachNode in CEGraph:
- #check if the node has been seen before
- if eachNode not in dict(enumerate(zip(*seenNode))):
- #look for all trees associated with node as the root
- thisPathDistanceAndLength = nx.single_source_dijkstra(CEGraph, eachNode)
- #determine the actual shortestPath and minimum depth/length
- maxDepthAndMinPath = findMaxDepthAndMinPath(thisPathDistanceAndLength)
- if maxDepthAndMinPath:
- maxPathLength = maxDepthAndMinPath[0]
- shortestPath = maxDepthAndMinPath[1]
-
- #add nodes and paths to PRUNED_GRAPH
- for i in xrange(len(shortestPath)):
- if PRUNED_GRAPH.has_node(shortestPath[i]) is False:
- PRUNED_GRAPH.add_node(shortestPath[i])
-
- #add edge if necessary
- if i < (len(shortestPath)-1) and PRUNED_GRAPH.has_edge(shortestPath[i], shortestPath[i+1]) is False:
- prunedGraphEdgeweight = CEGraph.get_edge_data(shortestPath[i], shortestPath[i+1])['weight']
- PRUNED_GRAPH.add_edge(shortestPath[i], shortestPath[i+1], weight=prunedGraphEdgeweight)
-
- #note information in a file for consideration later i.e. checking to see if it works
- cloudClustersFile.write("\nSubtree pathlength is %d and path is %s" %(maxPathLength, shortestPath))
- #update seenNode info
- seenNode.append(shortestPath)
-
- print "pruned graph"
- print "number of nodes are: ", PRUNED_GRAPH.number_of_nodes()
- print "number of edges are: ", PRUNED_GRAPH.number_of_edges()
- print ("*"*80)
-
- #graphTitle = "Cloud Clusters observed over Niamey, Niger 11 Sep 2006 00Z - 12 Sep 2006 12Z"
- graphTitle = "Cloud Clusters observed over Burkina Faso 31 Aug 2009 00Z - 01 Sep 2008 23Z"
- drawGraph(PRUNED_GRAPH, graphTitle, edgeWeight)
- cloudClustersFile.close
-
- return PRUNED_GRAPH
-#******************************************************************
-def findMCC (prunedGraph):
- '''
- Purpose:: Determines if subtree is a MCC according to Laurent et al 1998 criteria
-
- Input:: a Networkx Graph - prunedGraph
-
- Output:: finalMCCList: a list of list of tuples representing a MCC
-
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
-
- '''
- MCCList = []
- MCSList = []
- definiteMCC = []
- definiteMCS = []
- eachList =[]
- eachMCCList =[]
- maturing = False
- decaying = False
- fNode = ''
- lNode = ''
- removeList =[]
- imgCount = 0
- imgTitle =''
-
- maxShieldNode = ''
- orderedPath =[]
- treeTraversalList =[]
- definiteMCCFlag = False
- unDirGraph = nx.Graph()
- aSubGraph = nx.DiGraph()
- definiteMCSFlag = False
-
-
- #connected_components is not available for DiGraph, so generate graph as undirected
- unDirGraph = PRUNED_GRAPH.to_undirected()
- subGraph = nx.connected_component_subgraphs(unDirGraph)
-
- #for each path in the subgraphs determined
- for path in subGraph:
- #definite is a subTree provided the duration is longer than 3 hours
-
- if len(path.nodes()) > MIN_MCS_DURATION:
- orderedPath = path.nodes()
- orderedPath.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
- #definiteMCS.append(orderedPath)
-
- #build back DiGraph for checking purposes/paper purposes
- aSubGraph.add_nodes_from(path.nodes())
- for eachNode in path.nodes():
- if prunedGraph.predecessors(eachNode):
- for node in prunedGraph.predecessors(eachNode):
- aSubGraph.add_edge(node,eachNode,weight=edgeWeight[0])
-
- if prunedGraph.successors(eachNode):
- for node in prunedGraph.successors(eachNode):
- aSubGraph.add_edge(eachNode,node,weight=edgeWeight[0])
- imgTitle = 'CC'+str(imgCount+1)
- drawGraph(aSubGraph, imgTitle, edgeWeight) #for eachNode in path:
- imgCount +=1
- #----------end build back ---------------------------------------------
-
- mergeList, splitList = hasMergesOrSplits(path)
- #add node behavior regarding neutral, merge, split or both
- for node in path:
- if node in mergeList and node in splitList:
- addNodeBehaviorIdentifier(node,'B')
- elif node in mergeList and not node in splitList:
- addNodeBehaviorIdentifier(node,'M')
- elif node in splitList and not node in mergeList:
- addNodeBehaviorIdentifier(node,'S')
- else:
- addNodeBehaviorIdentifier(node,'N')
-
-
- #Do the first part of checking for the MCC feature
- #find the path
- treeTraversalList = traverseTree(aSubGraph, orderedPath[0],[],[])
- print "treeTraversalList is ", treeTraversalList
- #check the nodes to determine if a MCC on just the area criteria (consecutive nodes meeting the area and temp requirements)
- MCCList = checkedNodesMCC(prunedGraph, treeTraversalList)
- for aDict in MCCList:
- for eachNode in aDict["fullMCSMCC"]:
- addNodeMCSIdentifier(eachNode[0],eachNode[1])
-
- #do check for if MCCs overlap
- if MCCList:
- if len(MCCList) > 1:
- for count in range(len(MCCList)): #for eachDict in MCCList:
- #if there are more than two lists
- if count >= 1:
- #and the first node in this list
- eachList = list(x[0] for x in MCCList[count]["possMCCList"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
- if eachList:
- fNode = eachList[0]
- #get the lastNode in the previous possMCC list
- eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
- if eachList:
- lNode = eachList[-1]
- if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
- for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
- if aNode in eachList and aNode == lNode:
- #if edge_data is equal or less than to the exisitng edge in the tree append one to the other
- if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']:
- MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"])
- MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"])
- MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"]
- MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
- MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
- MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"]
- removeList.append(count)
- #update the MCCList
- if removeList:
- for i in removeList:
- del MCCList[i]
- removeList =[]
-
- #check if the nodes also meet the duration criteria and the shape crieria
- for eachDict in MCCList:
- #order the fullMCSMCC list, then run maximum extent and eccentricity criteria
- if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION:
- eachList = list(x[0] for x in eachDict["fullMCSMCC"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
- eachMCCList = list(x[0] for x in eachDict["possMCCList"])
- eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
-
- #update the nodemcsidentifer behavior
- #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
- #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
- #ensure that everything between is listed as 'M'
- for eachNode in eachList[:(eachList.index(eachMCCList[0]))]:
- addNodeMCSIdentifier(eachNode,'I')
-
- addNodeMCSIdentifier(eachMCCList[0],'M')
-
- for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]:
- addNodeMCSIdentifier(eachNode, 'D')
-
- #update definiteMCS list
- for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]:
- addNodeMCSIdentifier(eachNode, 'D')
-
- #run maximum extent and eccentricity criteria
- maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList)
- #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag
- if definiteMCCFlag == True:
- definiteMCC.append(eachList)
-
-
- definiteMCS.append(orderedPath)
-
- #reset for next subGraph
- aSubGraph.clear()
- orderedPath=[]
- MCCList =[]
- MCSList =[]
- definiteMCSFlag = False
-
- return definiteMCC, definiteMCS
-#******************************************************************
-def traverseTree(subGraph,node, queue, checkedNodes=None):
- '''
- Purpose:: To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
-
- Input:: subGraph: a Networkx DiGraph representing a CC
- lengthOfsubGraph: an integer representing the length of the subgraph
- node: a string representing the node currently being checked
- queue: a list of strings representing a list of nodes in a stack functionality
- i.e. First-In-FirstOut (FIFO) for sorting the information from each visited node
- checkedNodes: a list of strings representing the list of the nodes in the traversal
-
- Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal
-
- Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images
-
- '''
- if len(checkedNodes) == len(subGraph):
- return checkedNodes
-
- if not checkedNodes:
- queue =[]
- checkedNodes.append(node)
-
- #check one level infront first...if something does exisit, stick it at the front of the stack
- upOneLevel = subGraph.predecessors(node)
- downOneLevel = subGraph.successors(node)
- for parent in upOneLevel:
- if parent not in checkedNodes and parent not in queue:
- for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
- queue.insert(0,child)
- #downCheckFlag = True
- queue.insert(0,parent)
-
- for child in downOneLevel:
- if child not in checkedNodes and child not in queue:
- if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
- queue.insert(0,child)
- else:
- queue.append(child)
-
- #print "^^^^^stack ", stack
- for eachNode in queue:
- if eachNode not in checkedNodes:
- #print "if eachNode ", checkedNodes, eachNode, stack
- checkedNodes.append(eachNode)
- return traverseTree(subGraph, eachNode, queue, checkedNodes)
-
- return checkedNodes
-#******************************************************************
-def checkedNodesMCC (prunedGraph, nodeList):
- '''
- Purpose :: Determine if this path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
-
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- nodeList: list of strings (CE ID) from the traversal
-
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
- dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
- '''
-
- CounterCriteriaAFlag = False
- CounterCriteriaBFlag = False
- INITIATIONFLAG = False
- MATURITYFLAG = False
- DECAYFLAG = False
- thisdict = {} #will have the same items as the cloudElementDict
- cloudElementArea = 0.0
- epsilon = 0.0
- frameNum =0
- oldNode =''
- potentialMCCList =[]
- durationAandB = 0
- CounterCriteriaA = 0
- CountercriteriaB = 0
-
- #check for if the list contains only one string/node
- if type(nodeList) is str:
- oldNode=nodeList
- nodeList =[]
- nodeList.append(oldNode)
-
- for node in nodeList:
- thisdict = thisDict(node)
- CounterCriteriaAFlag = False
- CounterCriteriaBFlag = False
- existingFrameFlag = False
-
- if thisdict['cloudElementArea'] >= OUTER_CLOUD_SHIELD_AREA:
- #print "OUTER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- CounterCriteriaAFlag = True
- INITIATIONFLAG = True
- MATURITYFLAG = False
- #check if criteriaB is meet
- cloudElementArea,criteriaB = checkCriteriaB(thisdict['cloudElementLatLon'])
-
- #if Criteria A and B have been met, then the MCC is initiated, i.e. store node as potentialMCC
- if cloudElementArea >= INNER_CLOUD_SHIELD_AREA:
- #print "INNER_CLOUD_SHIELD_AREA met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- CounterCriteriaBFlag = True
- #append this information on to the dictionary
- addInfothisDict(node, cloudElementArea, criteriaB)
- INITIATIONFLAG = False
- MATURITYFLAG = True
- stage = 'M'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- else:
- #criteria B failed
- CounterCriteriaBFlag = False
- if INITIATIONFLAG == True:
- stage = 'I'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
-
- elif (INITIATIONFLAG == False and MATURITYFLAG == True) or DECAYFLAG==True:
- DECAYFLAG = True
- MATURITYFLAG = False
- stage = 'D'
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- else:
- #criteria A failed
- CounterCriteriaAFlag = False
- CounterCriteriaBFlag = False
- #print "!!OUTER_CLOUD_SHIELD_AREA NOT met by: ", node, INITIATIONFLAG, MATURITYFLAG, DECAYFLAG
- #add as a CE before or after the main feature
- if INITIATIONFLAG == True or (INITIATIONFLAG == False and MATURITYFLAG == True):
- stage ="I"
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- elif (INITIATIONFLAG == False and MATURITYFLAG == False) or DECAYFLAG == True:
- stage = "D"
- DECAYFLAG = True
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
- elif (INITIATIONFLAG == False and MATURITYFLAG == False and DECAYFLAG == False):
- stage ="I"
- potentialMCCList = updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag)
-
- return potentialMCCList
-#******************************************************************
-def updateMCCList(prunedGraph, potentialMCCList,node,stage, CounterCriteriaAFlag, CounterCriteriaBFlag):
- '''
- Purpose :: Utility function to determine if a path is (or is part of) a MCC and provides
- preliminary information regarding the stages of the feature
-
- Input:: prunedGraph: a Networkx Graph representing all the cloud clusters
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- node: a string representing the cloud element currently being assessed
- CounterCriteriaAFlag: a boolean value indicating whether the node meets the MCC criteria A according to Laurent et al
- CounterCriteriaBFlag: a boolean value indicating whether the node meets the MCC criteria B according to Laurent et al
-
- Output:: potentialMCCList: list of dictionaries representing all possible MCC within the path
- dictionary = {"possMCCList":[(node,'I')], "fullMCSMCC":[(node,'I')], "CounterCriteriaA": CounterCriteriaA, "durationAandB": durationAandB}
-
- '''
- existingFrameFlag = False
- existingMCSFrameFlag = False
- predecessorsFlag = False
- predecessorsMCSFlag = False
- successorsFlag = False
- successorsMCSFlag = False
- frameNum = 0
-
- frameNum = int((node.split('CE')[0]).split('F')[1])
- if potentialMCCList==[]:
- #list empty
- stage = 'I'
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
- elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
-
- else:
- #list not empty
- predecessorsFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,1)
-
- if predecessorsFlag == True:
-
- for eachNode in potentialMCCList[index]["possMCCList"]:# MCCDict["possMCCList"]:
- if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum :
- existingFrameFlag = True
-
- #this MUST come after the check for the existing frame
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- stage = 'M'
- potentialMCCList[index]["possMCCList"].append((node,stage))
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
-
-
- if existingFrameFlag == False:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- stage ='M'
- potentialMCCList[index]["CounterCriteriaA"]+= 1
- potentialMCCList[index]["durationAandB"]+=1
- if frameNum > potentialMCCList[index]["frameNum"]:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
- #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list
- if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0:
- stage = 'I'
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- return potentialMCCList
- elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- return potentialMCCList
-
- #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
- if existingFrameFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- if CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
-
- if predecessorsFlag == False:
- successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
-
- if successorsFlag == True:
- for eachNode in potentialMCCList[index]["possMCCList"]: #MCCDict["possMCCList"]:
- if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
- existingFrameFlag = True
-
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
- stage = 'M'
- potentialMCCList[index]["possMCCList"].append((node,stage))
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
-
- if existingFrameFlag == False:
- if stage == 'M':
- stage = 'D'
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- potentialMCCList[index]["CounterCriteriaA"]+= 1
- potentialMCCList[index]["durationAandB"]+=1
- elif CounterCriteriaAFlag == True:
- potentialMCCList[index]["CounterCriteriaA"] += 1
- elif CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
- #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
- else:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- if CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
-
- #if this node isn't connected to exisiting MCCs check if it is connected to exisiting MCSs ...
- if predecessorsFlag == False and successorsFlag == False:
- stage = 'I'
- predecessorsMCSFlag, index = isThereALink(prunedGraph, 1,node,potentialMCCList,2)
- if predecessorsMCSFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
- potentialMCCList[index]["possMCCList"].append((node,'M'))
- potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
- potentialMCCList[index]["durationAandB"] += 1
- if frameNum > potentialMCCList[index]["frameNum"]:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
- if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- elif CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
- else:
- successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
- if successorsMCSFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
- potentialMCCList[index]["possMCCList"].append((node,'M'))
- potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
- potentialMCCList[index]["durationAandB"] += 1
- if frameNum > potentialMCCList[index]["frameNum"]:
- potentialMCCList[index]["frameNum"] = frameNum
- potentialMCCList[index]["highestMCCnode"] = node
- return potentialMCCList
-
-
- if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
- return potentialMCCList
- elif CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
-
- #if this node isn't connected to existing MCCs or MCSs, create a new one ...
- if predecessorsFlag == False and predecessorsMCSFlag == False and successorsFlag == False and successorsMCSFlag == False:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag ==True:
- potentialMCCList.append({"possMCCList":[(node,stage)], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 1, "highestMCCnode":node, "frameNum":frameNum})
- elif CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
- elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
-
- return potentialMCCList
-#******************************************************************
-def isThereALink(prunedGraph, upOrDown,node,potentialMCCList,whichList):
- '''
- Purpose: Utility script for updateMCCList mostly because there is no Pythonic way to break out of nested loops
-
- Input:: prunedGraph:a Networkx Graph representing all the cloud clusters
- upOrDown: an integer representing 1- to do predecesor check and 2 - to do successor checkedNodesMCC
- node: a string representing the cloud element currently being assessed
- potentialMCCList: a list of dictionaries representing the possible MCCs within a path
- whichList: an integer representing which list ot check in the dictionary; 1- possMCCList, 2- fullMCSMCC
-
- Output:: thisFlag: a boolean representing whether the list passed has in the parent or child of the node
- index: an integer representing the location in the potentialMCCList where thisFlag occurs
-
- '''
- thisFlag = False
- index = -1
- checkList =""
- if whichList == 1:
- checkList = "possMCCList"
- elif whichList ==2:
- checkList = "fullMCSMCC"
-
- #check parents
- if upOrDown == 1:
- for aNode in prunedGraph.predecessors(node):
- #reset the index counter for this node search through potentialMCCList
- index = -1
- for MCCDict in potentialMCCList:
- index += 1
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
- thisFlag = True
- #get out of looping so as to avoid the flag being written over when another node in the predecesor list is checked
- return thisFlag, index
-
- #check children
- if upOrDown == 2:
- for aNode in prunedGraph.successors(node):
- #reset the index counter for this node search through potentialMCCList
- index = -1
- for MCCDict in potentialMCCList:
- index += 1
-
- if aNode in list(x[0] for x in MCCDict[checkList]): #[0]:
- thisFlag = True
- return thisFlag, index
-
- return thisFlag, index
-#******************************************************************
-def maxExtentAndEccentricity(eachList):
- '''
- Purpose:: perform the final check for MCC based on maximum extent and eccentricity criteria
-
- Input:: eachList: a list of strings representing the node of the possible MCCs within a path
-
- Output:: maxShieldNode: a string representing the node with the maximum maxShieldNode
- definiteMCCFlag: a boolean indicating that the MCC has met all requirements
- '''
- maxShieldNode =''
- maxShieldArea = 0.0
- maxShieldEccentricity = 0.0
- definiteMCCFlag = False
-
- if eachList:
- for eachNode in eachList:
- if (thisDict(eachNode)['nodeMCSIdentifier'] == 'M' or thisDict(eachNode)['nodeMCSIdentifier'] == 'D') and thisDict(eachNode)['cloudElementArea'] > maxShieldArea:
- maxShieldNode = eachNode
- maxShieldArea = thisDict(eachNode)['cloudElementArea']
-
- maxShieldEccentricity = thisDict(maxShieldNode)['cloudElementEccentricity']
- if thisDict(maxShieldNode)['cloudElementEccentricity'] >= ECCENTRICITY_THRESHOLD_MIN and thisDict(maxShieldNode)['cloudElementEccentricity'] <= ECCENTRICITY_THRESHOLD_MAX :
- #criteria met
- definiteMCCFlag = True
-
- return maxShieldNode, definiteMCCFlag
-#******************************************************************
-def findMaxDepthAndMinPath (thisPathDistanceAndLength):
- '''
- Purpose:: To determine the maximum depth and min path for the headnode
-
- Input:: tuple of dictionaries representing the shortest distance and paths for a node in the tree as returned by nx.single_source_dijkstra
- thisPathDistanceAndLength({distance}, {path})
- {distance} = nodeAsString, valueAsInt, {path} = nodeAsString, pathAsList
-
- Output:: tuple of the max pathLength and min pathDistance as a tuple (like what was input)
- minDistanceAndMaxPath = ({distance},{path})
- '''
- maxPathLength = 0
- minPath = 0
-
- #maxPathLength for the node in question
- maxPathLength = max(len (values) for values in thisPathDistanceAndLength[1].values())
-
- #if the duration is shorter then the min MCS length, then don't store!
- if maxPathLength < MIN_MCS_DURATION: #MINIMUM_DURATION :
- minDistanceAndMaxPath = ()
-
- #else find the min path and max depth
- else:
- #max path distance for the node in question
- minPath = max(values for values in thisPathDistanceAndLength[0].values())
-
- #check to determine the shortest path from the longest paths returned
- for pathDistance, path in itertools.izip(thisPathDistanceAndLength[0].values(), thisPathDistanceAndLength[1].values()):
- pathLength = len(path)
- #if pathLength is the same as the maxPathLength, then look the pathDistance to determine if the min
- if pathLength == maxPathLength :
- if pathDistance <= minPath:
- minPath = pathLength
- #store details if absolute minPath and deepest... so need to search this stored info in tuple myTup = {dict1, dict2}
- minDistanceAndMaxPath = (pathDistance, path)
- return minDistanceAndMaxPath
-#******************************************************************
-def thisDict (thisNode):
- '''
- Purpose:: return dictionary from graph if node exist in tree
-
- Input:: String - thisNode
-
- Output :: Dictionary - eachdict[1] associated with thisNode from the Graph
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- return eachdict[1]
-#******************************************************************
-def checkCriteriaB (thisCloudElementLatLon):
- '''
- Purpose:: Determine if criteria B is met for a CEGraph
-
- Input:: 2d array of (lat,lon) variable from the node dictionary being currently considered
-
- Output :: float - cloudElementArea, masked array of values meeting the criteria - criteriaB
-
- '''
- cloudElementCriteriaBLatLon=[]
-
- frame, CEcounter = ndimage.measurements.label(thisCloudElementLatLon, structure=STRUCTURING_ELEMENT)
- frameCEcounter=0
- #determine min and max values in lat and lon, then use this to generate teh array from LAT,LON meshgrid
- minLat = min(x[0] for x in thisCloudElementLatLon)
- maxLat = max(x[0]for x in thisCloudElementLatLon)
- minLon = min(x[1]for x in thisCloudElementLatLon)
- maxLon = max(x[1]for x in thisCloudElementLatLon)
-
- minLatIndex = np.argmax(LAT[:,0] == minLat)
- maxLatIndex = np.argmax(LAT[:,0]== maxLat)
- minLonIndex = np.argmax(LON[0,:] == minLon)
- maxLonIndex = np.argmax(LON[0,:] == maxLon)
-
- criteriaBframe = ma.zeros(((abs(maxLatIndex - minLatIndex)+1), (abs(maxLonIndex - minLonIndex)+1)))
-
- for x in thisCloudElementLatLon:
- #to store the values of the subset in the new array, remove the minLatIndex and minLonindex from the
- #index given in the original array to get the indices for the new array
- #criteriaBframe[(np.argmax(LAT[:,0] == x[0])),(np.argmax(LON[0,:] == x[1]))] = x[2]
- criteriaBframe[(np.argmax(LAT[:,0] == x[0]) - minLatIndex),(np.argmax(LON[0,:] == x[1]) - minLonIndex)] = x[2]
-
- #print criteriaBframe
- #keep only those values < TBB_MAX
- tempMask = ma.masked_array(criteriaBframe, mask=(criteriaBframe >= INNER_CLOUD_SHIELD_TEMPERATURE), fill_value = 0)
-
- #get the actual values that the mask returned
- criteriaB = ma.zeros((criteriaBframe.shape)).astype('int16')
-
- for index, value in maenumerate(tempMask):
- lat_index, lon_index = index
- criteriaB[lat_index, lon_index]=value
-
- for count in xrange(CEcounter):
- #[0] is time dimension. Determine the actual values from the data
- #loc is a masked array
- #***** returns elements down then across thus (6,4) is 6 arrays deep of size 4
-
- loc = ndimage.find_objects(criteriaB)[0]
- cloudElementCriteriaB =criteriaB[loc]
-
- for index,value in np.ndenumerate(cloudElementCriteriaB):
- if value !=0:
- t,lat,lon = index
- #add back on the minLatIndex and minLonIndex to find the true lat, lon values
- lat_lon_tuple = (LAT[(lat),0], LON[0,(lon)],value)
- cloudElementCriteriaBLatLon.append(lat_lon_tuple)
-
- cloudElementArea = np.count_nonzero(cloudElementCriteriaB)*XRES*YRES
-
- return cloudElementArea, cloudElementCriteriaBLatLon
-#******************************************************************
-def hasMergesOrSplits (nodeList):
- '''
- Purpose:: Determine if nodes within a path defined from shortest_path splittingNodeDict
- Input:: nodeList - list of nodes from a path
- Output:: two list_vars_in_file
- splitList list of all the nodes in the path that split
- mergeList list of all the nodes in the path that merged
- '''
- mergeList=[]
- splitList=[]
-
- for node,numParents in PRUNED_GRAPH.in_degree(nodeList).items():
- if numParents > 1:
- mergeList.append(node)
-
- for node, numChildren in PRUNED_GRAPH.out_degree(nodeList).items():
- if numChildren > 1:
- splitList.append(node)
- #sort
- splitList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
- mergeList.sort(key=lambda item:(len(item.split('C')[0]), item.split('C')[0]))
-
- return mergeList,splitList
-#******************************************************************
-def allAncestors(path, aNode):
- '''
- Purpose:: Utility script to provide the path leading up to a nodeList
-
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for parents
-
- Output:: list of strings - path: the list of the nodes connected to aNode through its parents
- integer - numOfChildren: the number of parents of the node passed
- '''
-
- numOfParents = PRUNED_GRAPH.in_degree(aNode)
- try:
- if PRUNED_GRAPH.predecessors(aNode) and numOfParents <= 1:
- path = path + PRUNED_GRAPH.predecessors(aNode)
- thisNode = PRUNED_GRAPH.predecessors(aNode)[0]
- return allAncestors(path,thisNode)
- else:
- path = path+aNode
- return path, numOfParents
- except:
- return path, numOfParents
-#******************************************************************
-def allDescendants(path, aNode):
- '''
- Purpose:: Utility script to provide the path leading up to a nodeList
-
- Input:: list of strings - path: the nodes in the path
- string - aNode: a string representing a node to be checked for children
-
- Output:: list of strings - path: the list of the nodes connected to aNode through its children
- integer - numOfChildren: the number of children of the node passed
- '''
-
- numOfChildren = PRUNED_GRAPH.out_degree(aNode)
- try:
- if PRUNED_GRAPH.successors(aNode) and numOfChildren <= 1:
- path = path + PRUNED_GRAPH.successors(aNode)
- thisNode = PRUNED_GRAPH.successors(aNode)[0]
- return allDescendants(path,thisNode)
- else:
- path = path + aNode
- #i.e. PRUNED_GRAPH.predecessors(aNode) is empty
- return path, numOfChildren
- except:
- #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
- return path, numOfChildren
-#******************************************************************
-def addInfothisDict (thisNode, cloudElementArea,criteriaB):
- '''
- Purpose:: update original dictionary node with information
-
- Input:: String - thisNode
- float - cloudElementArea,
- masked array of floats meeting the criteria - criteriaB
-
- Output ::
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- eachdict[1]['CriteriaBArea'] = cloudElementArea
- eachdict[1]['CriteriaBLatLon'] = criteriaB
- return
-#******************************************************************
-def addNodeBehaviorIdentifier (thisNode, nodeBehaviorIdentifier):
- '''
- Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
-
- Input:: String - thisNode
- String - nodeBehaviorIdentifier = S- split, M- merge, B- both split and merge, N- neither split or merge
-
- Output :: None
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- if not 'nodeBehaviorIdentifier' in eachdict[1].keys():
- eachdict[1]['nodeBehaviorIdentifier'] = nodeBehaviorIdentifier
- return
-#******************************************************************
-def addNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
- '''
- Purpose:: add an identifier to the node dictionary to indicate splitting, merging or neither node
-
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
-
- Output :: None
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- if not 'nodeMCSIdentifier' in eachdict[1].keys():
- eachdict[1]['nodeMCSIdentifier'] = nodeMCSIdentifier
- return
-#******************************************************************
-def updateNodeMCSIdentifier (thisNode, nodeMCSIdentifier):
- '''
- Purpose:: update an identifier to the node dictionary to indicate splitting, merging or neither node
-
- Input:: String - thisNode
- String - nodeMCSIdentifier = 'I' for Initiation, 'M' for Maturity, 'D' for Decay
-
-
- Output :: None
-
- '''
- for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
- if eachdict[1]['uniqueID'] == thisNode:
- eachdict[1]['nodeMCSIdentifier'] = nodeBehaviorIdentifier
-
- return
-#******************************************************************
-def eccentricity (cloudElementLatLon):
- '''
- Purpose::
- Determines the eccentricity (shape) of contiguous boxes
- Values tending to 1 are more circular by definition, whereas
- values tending to 0 are more linear
-
- Input::
- 1 variable
- cloudElementLatLon: 3D array in (time,lat,lon),T_bb contiguous squares
-
- Output::
- 1 variable
- epsilon: a float representing the eccentricity of the matrix passed
-
- '''
-
- epsilon = 0.0
-
- #loop over all lons and determine longest (non-zero) col
- #loop over all lats and determine longest (non-zero) row
- for latLon in cloudElementLatLon:
- #assign a matrix to determine the legit values
-
- nonEmptyLons = sum(sum(cloudElementLatLon)>0)
- nonEmptyLats = sum(sum(cloudElementLatLon.transpose())>0)
-
- lonEigenvalues = 1.0 * nonEmptyLats / (nonEmptyLons+0.001) #for long oval on y axis
- latEigenvalues = 1.0 * nonEmptyLons / (nonEmptyLats +0.001) #for long oval on x-axs
- epsilon = min(latEigenvalues,lonEigenvalues)
-
- return epsilon
-#******************************************************************
-def cloudElementOverlap (currentCELatLons, previousCELatLons):
- '''
- Purpose::
- Determines the percentage overlap between two list of lat-lons passed
-
- Input::
- 2 sorted list of tuples
- currentCELatLons - the list of tuples for the current CE
- previousCELatLons - the list of tuples for the other CE being considered
-
- Output::
- 2 variables
- percentageOverlap - a float representing the number of overlapping lat_lon tuples
- areaOverlap - a floating-point number representing the area overlapping
-
- '''
-
- latlonprev =[]
- latloncurr = []
- count = 0
- percentageOverlap = 0.0
- areaOverlap = 0.0
-
- #remove the temperature from the tuples for currentCELatLons and previousCELatLons then check for overlap
- latlonprev = [(x[0],x[1]) for x in previousCELatLons]
- latloncurr = [(x[0],x[1]) for x in currentCELatLons]
-
- #find overlap
- count = len(list(set(latloncurr)&set(latlonprev)))
-
- #find area overlap
- areaOverlap = count*XRES*YRES
-
- #find percentage
- percentageOverlap = max(((count*1.0)/(len(latloncurr)*1.0)),((count*1.0)/(len(latlonprev)*1.0)))
-
- return percentageOverlap, areaOverlap
-#******************************************************************
-def findCESpeed(node, MCSList):
- '''
- Purpose:: to determine the speed of the CEs
- uses vector displacement delta_lat/delta_lon (y/x)
-
- Input:: node: a string representing the CE
- MCSList: a list of strings representing the feature
-
- Output::
-
- '''
-
- delta_lon =0.0
- delta_lat =0.0
- CEspeed =[]
- theSpeed = 0.0
-
-
- theList = CLOUD_ELEMENT_GRAPH.successors(node)
- nodeLatLon=thisDict(node)
-
-
- for aNode in theList:
- if aNode in MCSList:
- #if aNode is part of the MCSList then determine distance
- aNodeLatLon = thisDict(aNode)
- #calculate CE speed
- #checking the lats
- nodeLatLon[0] += 90.0
- aNodeLatLon[0] += 90.0
- delta_lat = (nodeLatLon[0] - aNodeLatLon[0])*LAT_DISTANCE*1000 #convert to m
- #recall -ve ans --> northward tracking, -ve ans --> southwart tracking
-
- #checking lons lonsraw[lonsraw > 180] = lonsraw[lonsraw > 180] - 360. # convert to -180,180 if necessary
-
- if nodeLatLon[1] < 0.0:
- nodeLatLon[1] += 360.0
- if aNodeLatLon[1] <= 0.0:
- delta_lon = aNodeLatLon[1] + 360.0
-
- delta_lon = (nodeLatLon[1] - aNodeLatLon[1])*LON_DISTANCE*1000 #convert to m
- #recall +ve ans --> westward tracking, -ve ans --> eastward tracking
-
- theSpeed = abs(((delta_lat/delta_lon)/(TRES*3600))) #convert to s --> m/s
-
- CEspeed.append(theSpeed)
- print "aNode CE speed is ", aNode, ((delta_lat/delta_lon)/(TRES*3600)), theSpeed
-
- return min(CEspeed)
-#******************************************************************
-#
-# UTILITY SCRIPTS FOR MCCSEARCH.PY
-#
-#******************************************************************
-def maenumerate(mArray):
- '''
- Purpose::
- Utility script for returning the actual values from the masked array
- Taken from: http://stackoverflow.com/questions/8620798/numpy-ndenumerate-for-masked-arrays
-
- Input::
- 1 variable
- mArray - the masked array returned from the ma.array() command
-
-
- Output::
- 1 variable
- maskedValues - 3D (t,lat,lon), value of only masked values
-
- '''
-
- mask = ~mArray.mask.ravel()
- #beware yield fast, but generates a type called "generate" that does not allow for array methods
- for index, maskedValue in itertools.izip(np.ndenumerate(mArray), mask):
- if maskedValue:
- yield index
-#******************************************************************
-def createMainDirectory(mainDirStr):
- '''
- Purpose:: to create the main directory for storing information and
- the subdirectories for storing information
- Input:: mainDir: a directory for where all information generated from
- the program are to be stored
- Output:: None
-
- '''
- global MAINDIRECTORY
-
- MAINDIRECTORY = mainDirStr
- #if directory doesnt exist, creat it
- if not os.path.exists(MAINDIRECTORY):
- os.makedirs(MAINDIRECTORY)
-
- os.chdir((MAINDIRECTORY))
- #create the subdirectories
- try:
- os.makedirs('images')
- os.makedirs('textFiles')
- os.makedirs('MERGnetcdfCEs')
- os.makedirs('TRMMnetcdfCEs')
- except:
- print "Directory exists already!!!"
- #TODO: some nice way of prompting if it is ok to continue...or just leave
-
- return
-#******************************************************************
-def find_nearest(thisArray,value):
- '''
- Purpose :: to determine the value within an array closes to
- another value
-
- Input ::
- Output::
- '''
- idx = (np.abs(thisArray-value)).argmin()
- return thisArray[idx]
-#******************************************************************
-def preprocessingMERG(MERGdirname):
- '''
- Purpose::
- Utility script for unzipping and converting the merg*.Z files from Mirador to
- NETCDF format. The files end up in a folder called mergNETCDF in the directory
- where the raw MERG data is
- NOTE: VERY RAW AND DIRTY
-
- Input::
- Directory to the location of the raw MERG files, preferably zipped
-
- Output::
- none
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- 3 the files havent been unzipped
- '''
-
- os.chdir((MERGdirname+'/'))
- imgFilename = ''
-
- #Just incase the X11 server is giving problems
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- for files in glob.glob("*-pixel"):
- #for files in glob.glob("*.Z"):
- fname = os.path.splitext(files)[0]
-
- #unzip it
- bash_cmd = 'gunzip ' + files
- subprocess.call(bash_cmd, shell=True)
-
- #determine the time from the filename
- ftime = re.search('\_(.*)\_',fname).group(1)
-
- yy = ftime[0:4]
- mm = ftime[4:6]
- day = ftime[6:8]
- hr = ftime [8:10]
-
- #TODO: must be something more efficient!
-
- if mm=='01':
- mth = 'Jan'
- if mm == '02':
- mth = 'Feb'
- if mm == '03':
- mth = 'Mar'
- if mm == '04':
- mth = 'Apr'
- if mm == '05':
- mth = 'May'
- if mm == '06':
- mth = 'Jun'
- if mm == '07':
- mth = 'Jul'
- if mm == '08':
- mth = 'Aug'
- if mm == '09':
- mth = 'Sep'
- if mm == '10':
- mth = 'Oct'
- if mm == '11':
- mth = 'Nov'
- if mm == '12':
- mth = 'Dec'
-
-
- subprocess.call('rm merg.ctl', shell=True)
- subprocess.call('touch merg.ctl', shell=True)
- replaceExpDset = 'echo DSET ' + fname +' >> merg.ctl'
- replaceExpTdef = 'echo TDEF 99999 LINEAR '+hr+'z'+day+mth+yy +' 30mn' +' >> merg.ctl'
- subprocess.call(replaceExpDset, shell=True)
- subprocess.call('echo "OPTIONS yrev little_endian template" >> merg.ctl', shell=True)
- subprocess.call('echo "UNDEF 330" >> merg.ctl', shell=True)
- subprocess.call('echo "TITLE globally merged IR data" >> merg.ctl', shell=True)
- subprocess.call('echo "XDEF 9896 LINEAR 0.0182 0.036378335" >> merg.ctl', shell=True)
- subprocess.call('echo "YDEF 3298 LINEAR -59.982 0.036383683" >> merg.ctl', shell=True)
- subprocess.call('echo "ZDEF 01 LEVELS 1" >> merg.ctl', shell=True)
- subprocess.call(replaceExpTdef, shell=True)
- subprocess.call('echo "VARS 1" >> merg.ctl', shell=True)
- subprocess.call('echo "ch4 1 -1,40,1,-1 IR BT (add "75" to this value)" >> merg.ctl', shell=True)
- subprocess.call('echo "ENDVARS" >> merg.ctl', shell=True)
-
- #generate the lats4D command for GrADS
- lats4D = 'lats4d -v -q -lat '+LATMIN + ' ' +LATMAX +' -lon ' +LONMIN +' ' +LONMAX +' -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- #lats4D = 'lats4d -v -q -lat -40 -15 -lon 10 40 -time '+hr+'Z'+day+mth+yy + ' -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
- #lats4D = 'lats4d -v -q -lat -5 40 -lon -90 60 -func @+75 ' + '-i merg.ctl' + ' -o ' + fname
-
- gradscmd = 'grads -blc ' + '\'' +lats4D + '\''
- #run grads and lats4d command
- subprocess.call(gradscmd, shell=True)
- imgFilename = hr+'Z'+day+mth+yy+'.gif'
- tempMaskedImages(imgFilename)
-
- #when all the files have benn converted, mv the netcdf files
- subprocess.call('mkdir mergNETCDF', shell=True)
- subprocess.call('mv *.nc mergNETCDF', shell=True)
- #mv all images
- subprocess.call('mkdir mergImgs', shell=True)
- subprocess.call('mv *.gif mergImgs', shell=True)
- return
-#******************************************************************
-def postProcessingNetCDF(dataset, dirName = None):
- '''
-
- TODO: UPDATE TO PICK UP LIMITS FROM FILE FOR THE GRADS SCRIPTS
-
- Purpose::
- Utility script displaying the data in generated NETCDF4 files
- in GrADS
- NOTE: VERY RAW AND DIRTY
-
- Input::
- dataset: integer representing post-processed MERG (1) or TRMM data (2) or original MERG(3)
- string: Directory to the location of the raw (MERG) files, preferably zipped
-
- Output::
- images in location as specfied in the code
-
- Assumptions::
- 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
- have been installed on the system and the user can access
- 2 User can write files in location where script is being called
- '''
-
- coreDir = os.path.dirname(MAINDIRECTORY)
- #coreDir = os.path.dirname(os.path.abspath(__file__))
- ImgFilename = ''
- frameList=[]
- fileList =[]
- lines =[]
- var =''
- firstTime = True
- printLine = 0
- lineNum = 1
- #Just incase the X11 server is giving problems
- subprocess.call('export DISPLAY=:0.0', shell=True)
-
- print "coreDir is ", str(coreDir)
- prevFrameNum = 0
-
- if dataset == 1:
- var = 'ch4'
- dirName = MAINDIRECTORY+'/MERGnetcdfCEs'
- ctlTitle = 'TITLE MCC search Output Grid: Time lat lon'
- ctlLine = 'brightnesstemp=\>ch4 1 t,y,x brightnesstemperature'
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- gsFile = coreDir+"/GrADSscripts/cs2.gs"
- sologsFile = coreDir+"/GrADSscripts/mergeCE.gs"
- lineNum = 50
-
- elif dataset ==2:
- var = 'precipAcc'
- dirName = MAINDIRECTORY+'/TRMMnetcdfCEs'
- ctlTitle ='TITLE TRMM MCS accumulated precipitation search Output Grid: Time lat lon '
- ctlLine = 'precipitation_Accumulation=\>precipAcc 1 t,y,x precipAccu'
- origsFile = coreDir+"/GrADSscripts/cs3.gs"
- gsFile = coreDir+"/GrADSscripts/cs4.gs"
- sologsFile = coreDir+"/GrADSscripts/TRMMCE.gs"
- lineNum = 10
-
- elif dataset ==3:
- var = 'ch4'
- ctlTitle = 'TITLE MERG DATA'
- ctlLine = 'ch4=\>ch4 1 t,y,x brightnesstemperature'
- if dirName is None:
- print "Enter directory for original files"
- return
- else:
- origsFile = coreDir+"/GrADSscripts/cs1.gs"
- sologsFile = coreDir+"/GrADSscripts/infrared.gs"
- lineNum = 54
-
- #sort files
- os.chdir((dirName+'/'))
- try:
- os.makedirs('ctlFiles')
- except:
- print "ctl file folder created already"
-
- files = filter(os.path.isfile, glob.glob("*.nc"))
- files.sort(key=lambda x: os.path.getmtime(x))
- for eachfile in files:
- fullFname = os.path.splitext(eachfile)[0]
- fnameNoExtension = fullFname.split('.nc')[0]
- if dataset == 1 or dataset == 2:
- frameNum = int((fnameNoExtension.split('CE')[0]).split('00F')[1])
- #create the ctlFile
- ctlFile1 = dirName+'/ctlFiles/'+fnameNoExtension + '.ctl'
- #the ctl file
- subprocessCall = 'rm ' +ctlFile1
- subprocess.call(subprocessCall, shell=True)
- subprocessCall = 'touch '+ctlFile1
- subprocess.call(subprocessCall, shell=True)
- lineToWrite = 'echo DSET ' + dirName+'/'+fnameNoExtension+'.nc' +' >>' + ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo DTYPE netcdf >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo UNDEF 0 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo '+ctlTitle+' >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo XDEF 413 LINEAR -9.984375 0.036378335 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo YDEF 412 LINEAR 5.03515625 0.036378335 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo ZDEF 01 LEVELS 1 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo TDEF 99999 linear 31aug2009 1hr >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo VARS 1 >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite ='echo '+ctlLine+' >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo ENDVARS >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
- lineToWrite = 'echo >> '+ctlFile1
- subprocess.call(lineToWrite, shell=True)
-
- #create plot of just that data
- subprocessCall = 'cp '+ origsFile+' '+sologsFile
- subprocess.call(subprocessCall, shell=True)
-
- ImgFilename = fnameNoExtension + '.gif'
-
- displayCmd = '\''+'d '+ var+'\''+'\n'
- newFileCmd = '\''+'open '+ ctlFile1+'\''+'\n'
- colorbarCmd = '\''+'run cbarn'+'\''+'\n'
- printimCmd = '\''+'
<TRUNCATED>