You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by le...@apache.org on 2017/09/11 14:52:58 UTC
[4/5] climate git commit: Merge branch 'master' into CLIMATE-912
Merge branch 'master' into CLIMATE-912
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/51d9dce1
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/51d9dce1
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/51d9dce1
Branch: refs/heads/master
Commit: 51d9dce12c9c9b77e7e48f3cca054bcf5647e458
Parents: 5c86f3a 56989f5
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Wed Aug 9 11:42:08 2017 -0700
Committer: GitHub <no...@github.com>
Committed: Wed Aug 9 11:42:08 2017 -0700
----------------------------------------------------------------------
RCMES/cli_app.py | 16 +-
.../NARCCAP_examples/Fig14_and_Fig15.yaml | 4 +-
.../NARCCAP_examples/Fig16_summer.yaml | 5 +-
.../NARCCAP_examples/Fig16_winter.yaml | 4 +-
.../run_statistical_downscaling.py | 17 +-
examples/esgf_integration_example.py | 17 +-
examples/podaac_integration_example.py | 7 +-
examples/simple_model_to_model_bias.py | 14 +-
examples/taylor_diagram_example.py | 13 +-
examples/time_series_with_regions.py | 21 +-
mccsearch/code/mccSearch.py | 16 +-
mccsearch/code/mccSearchUI.py | 1 +
ocw/data_source/esgf.py | 11 +-
ocw/data_source/local.py | 4 +-
ocw/dataset_processor.py | 90 ++----
ocw/esgf/download.py | 28 +-
ocw/tests/TestGetNetcdfVariableNames.nc | Bin 0 -> 471996 bytes
ocw/tests/test_dataset_processor.py | 6 +-
ocw/tests/test_local.py | 280 +++++++++++--------
ocw/utils.py | 2 +-
test_smoke.py | 4 +-
21 files changed, 306 insertions(+), 254 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/51d9dce1/mccsearch/code/mccSearch.py
----------------------------------------------------------------------
diff --cc mccsearch/code/mccSearch.py
index 9f396f8,6f1fa26..c63b243
--- a/mccsearch/code/mccSearch.py
+++ b/mccsearch/code/mccSearch.py
@@@ -44,72 -44,61 +44,73 @@@ import ocw.plotter as plotte
#----------------------- GLOBAL VARIABLES --------------------------
# --------------------- User defined variables ---------------------
-#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
-#the first point closest the the value (for the min) from the MERG data is used, etc.
-LATMIN = '5.0' #min latitude; -ve values in the SH e.g. 5S = -5
-LATMAX = '19.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0
-LONMIN = '-5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
-LONMAX = '5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
-XRES = 4.0 #x direction spatial resolution in km
-YRES = 4.0 #y direction spatial resolution in km
-TRES = 1 #temporal resolution in hrs
-LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered
-LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered
-STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must
- #have same rank of the matrix it is being compared against
-#criteria for determining cloud elements and edges
-T_BB_MAX = 243 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
-T_BB_MIN = 218 #cooler temp for the center of the system
-CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference)
-MIN_MCS_DURATION = 3 #minimum time for a MCS to exist
-AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
-MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+# FYI the lat lon values are not necessarily inclusive of the points given. These are the limits
+# the first point closest the the value (for the min) from the MERG data
+# is used, etc.
+LATMIN = '5.0' # min latitude; -ve values in the SH e.g. 5S = -5
+LATMAX = '19.0' # max latitude; -ve values in the SH e.g. 5S = -5 20.0
+LONMIN = '-5.0' # min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30
+LONMAX = '5.0' # min longitude; -ve values in the WH e.g. 59.8W = -59.8 30
+XRES = 4.0 # x direction spatial resolution in km
+YRES = 4.0 # y direction spatial resolution in km
+TRES = 1 # temporal resolution in hrs
+LAT_DISTANCE = 111.0 # the avg distance in km for 1deg lat for the region being considered
+LON_DISTANCE = 111.0 # the avg distance in km for 1deg lon for the region being considered
+# the matrix for determining the pattern for the contiguous boxes and must
+STRUCTURING_ELEMENT = [[0, 1, 0], [1, 1, 1], [0, 1, 0]]
+# have same rank of the matrix it is being compared against
+# criteria for determining cloud elements and edges
+# warmest temp to allow (-30C to -55C according to Morel and Sensi 2002)
+T_BB_MAX = 243
+T_BB_MIN = 218 # cooler temp for the center of the system
+# the min temp/max temp that would be expected in a CE.. this is highly
+# conservative (only a 10K difference)
+CONVECTIVE_FRACTION = 0.90
+MIN_MCS_DURATION = 3 # minimum time for a MCS to exist
+# minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400
+AREA_MIN = 2400.0
+# km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992
+MIN_OVERLAP = 10000.00
#---the MCC criteria
-ECCENTRICITY_THRESHOLD_MAX = 1.0 #tending to 1 is a circle e.g. hurricane,
-ECCENTRICITY_THRESHOLD_MIN = 0.50 #tending to 0 is a linear e.g. squall line
-OUTER_CLOUD_SHIELD_AREA = 80000.0 #km^2
-INNER_CLOUD_SHIELD_AREA = 30000.0 #km^2
-OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K
-INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K
-MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, MCCs is 6hrs)
-MAXIMUM_DURATION = 24#max number of framce the MCC can last for
+ECCENTRICITY_THRESHOLD_MAX = 1.0 # tending to 1 is a circle e.g. hurricane,
+ECCENTRICITY_THRESHOLD_MIN = 0.50 # tending to 0 is a linear e.g. squall line
+OUTER_CLOUD_SHIELD_AREA = 80000.0 # km^2
+INNER_CLOUD_SHIELD_AREA = 30000.0 # km^2
+OUTER_CLOUD_SHIELD_TEMPERATURE = 233 # in K
+INNER_CLOUD_SHIELD_TEMPERATURE = 213 # in K
+# min number of frames the MCC must exist for (assuming hrly frames, MCCs
+# is 6hrs)
+MINIMUM_DURATION = 6
+MAXIMUM_DURATION = 24 # max number of framce the MCC can last for
#------------------- End user defined Variables -------------------
-edgeWeight = [1,2,3] #weights for the graph edges
-#graph object fo the CEs meeting the criteria
+edgeWeight = [1, 2, 3] # weights for the graph edges
+# graph object fo the CEs meeting the criteria
CLOUD_ELEMENT_GRAPH = nx.DiGraph()
-#graph meeting the CC criteria
+# graph meeting the CC criteria
PRUNED_GRAPH = nx.DiGraph()
#------------------------ End GLOBAL VARS -------------------------
++
#************************ Begin Functions *************************
#******************************************************************
-def readMergData(dirname, filelist = None):
+
+
+def readMergData(dirname, filelist=None):
'''
Purpose::
- Read MERG data into RCMES format
-
+ Read MERG data into RCMES format
+
Input::
- dirname: a string representing the directory to the MERG files in NETCDF format
- filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
-
+ dirname: a string representing the directory to the MERG files in NETCDF format
+ filelist (optional): a list of strings representing the filenames betweent the start and end dates provided
+
Output::
- A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
- criteria for each frame
+ A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
+ criteria for each frame
Assumptions::
- The MERG data has been converted to NETCDF using LATS4D
- The data has the same lat/lon format
+ The MERG data has been converted to NETCDF using LATS4D
+ The data has the same lat/lon format
TODO:: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0])
@@@ -123,17 -112,19 +124,16 @@@
mergTimeVarName = 'time'
mergLatVarName = 'latitude'
mergLonVarName = 'longitude'
-
-
filelistInstructions = dirname + '/*'
- if filelist == None:
+ if filelist is None:
filelist = glob.glob(filelistInstructions)
-
- #sat_img is the array that will contain all the masked frames
+ # sat_img is the array that will contain all the masked frames
mergImgs = []
- #timelist of python time strings
- timelist = []
+ # timelist of python time strings
+ timelist = []
time2store = None
- tempMaskedValueNp =[]
-
+ tempMaskedValueNp = []
filelist.sort()
nfiles = len(filelist)
@@@ -271,20 -254,16 +271,21 @@@ def findCloudElements(mergImgs, timelis
minCELonLimit = 0.0
maxCELatLimit = 0.0
maxCELonLimit = 0.0
-
+
nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :])
-
- #openfile for storing ALL cloudElement information
- cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb')
- #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case
- cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w')
-
- #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times
- for t in xrange(mergImgs.shape[0]):
+
+ # openfile for storing ALL cloudElement information
+ cloudElementsFile = open(
+ (MAINDIRECTORY + '/textFiles/cloudElements.txt'), 'wb')
+ # openfile for storing cloudElement information meeting user criteria i.e.
+ # MCCs in this case
+ cloudElementsUserFile = open(
+ (MAINDIRECTORY + '/textFiles/cloudElementsUserFile.txt'), 'w')
+
+ # NB in the TRMM files the info is hours since the time thus 00Z file has
+ # in 01, 02 and 03 times
+ for t in range(mergImgs.shape[0]):
++
#-------------------------------------------------
# #textfile name for saving the data for arcgis
# thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt'
@@@ -460,60 -409,48 +461,59 @@@
currNetCDFTRMMData.conventions = 'COARDS'
# dimensions
currNetCDFTRMMData.createDimension('time', None)
- currNetCDFTRMMData.createDimension('lat', len(LAT[:,0]))
- currNetCDFTRMMData.createDimension('lon', len(LON[0,:]))
-
+ currNetCDFTRMMData.createDimension('lat', len(LAT[:, 0]))
+ currNetCDFTRMMData.createDimension('lon', len(LON[0, :]))
+
# variables
- TRMMprecip = ('time','lat', 'lon',)
- times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ str(timelist[t])[:-6]
- latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',))
- longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip )
+ TRMMprecip = ('time', 'lat', 'lon',)
+ times = currNetCDFTRMMData.createVariable(
+ 'time', 'f8', ('time',))
+ times.units = 'hours since ' + str(timelist[t])[:-6]
+ latitude = currNetCDFTRMMData.createVariable(
+ 'latitude', 'f8', ('lat',))
+ longitude = currNetCDFTRMMData.createVariable(
+ 'longitude', 'f8', ('lon',))
+ rainFallacc = currNetCDFTRMMData.createVariable(
+ 'precipitation_Accumulation', 'f8', TRMMprecip)
rainFallacc.units = 'mm'
- longitude[:] = LON[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
+ longitude[:] = LON[0, :]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
- latitude[:] = LAT[:,0]
+ latitude[:] = LAT[:, 0]
latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
+ latitude.long_name = "Latitude"
finalCETRMMvalues = ma.zeros((brightnesstemp.shape))
- #-----------End most of NETCDF file stuff ------------------------------------
+ #-----------End most of NETCDF file stuff -----------------
- #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
- #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
+ # populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon
+ # TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this
# as cloudElement is masked
- for index,value in np.ndenumerate(cloudElement):
- if value != 0 :
- lat_index,lon_index = index
- lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value)
-
- #generate the comma separated file for GIS
+ for index, value in np.ndenumerate(cloudElement):
+ if value != 0:
+ lat_index, lon_index = index
+ lat_lon_tuple = (
+ cloudElementLat[lat_index],
+ cloudElementLon[lon_index],
+ value)
+
+ # generate the comma separated file for GIS
cloudElementLatLons.append(lat_lon_tuple)
- #temp data for CE NETCDF file
- brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value
-
- if TRMMdirName:
- finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])]
- CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]]))
-
+ # temp data for CE NETCDF file
+ brightnesstemp1[0, int(np.where(LAT[:, 0] == cloudElementLat[lat_index])[0]), int(
+ np.where(LON[0, :] == cloudElementLon[lon_index])[0])] = value
+ if TRMMdirName:
+ finalCETRMMvalues[0, int(np.where(LAT[:, 0] == cloudElementLat[lat_index])[0]), int(np.where(LON[0, :] == cloudElementLon[lon_index])[
+ 0])] = regriddedTRMM[int(np.where(LAT[:, 0] == cloudElementLat[lat_index])[0]), int(np.where(LON[0, :] == cloudElementLon[lon_index])[0])]
+ CETRMMList.append((cloudElementLat[lat_index],
+ cloudElementLon[lon_index],
+ finalCETRMMvalues[0,
+ cloudElementLat[lat_index],
+ cloudElementLon[lon_index]]))
-
brightnesstemp[:] = brightnesstemp1[:]
currNetCDFCEData.close()
@@@ -917,54 -746,51 +917,53 @@@ def findPrecipRate(TRMMdirName, timelis
minCEprecipRate = 0.0
try:
- maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
- except:
+ maxCEprecipRate = np.max(
+ finalCETRMMvalues[np.nonzero(finalCETRMMvalues)])
+ except BaseException:
maxCEprecipRate = 0.0
- #add info to CLOUDELEMENTSGRAPH
- #TODO try block
+ # add info to CLOUDELEMENTSGRAPH
+ # TODO try block
for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID):
if eachdict[1]['uniqueID'] == CEuniqueID:
- if not 'cloudElementPrecipTotal' in eachdict[1].keys():
+ if 'cloudElementPrecipTotal' not in list(eachdict[1].keys()):
eachdict[1]['cloudElementPrecipTotal'] = precipTotal
- if not 'cloudElementLatLonTRMM' in eachdict[1].keys():
+ if 'cloudElementLatLonTRMM' not in list(eachdict[1].keys()):
eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues
- if not 'TRMMArea' in eachdict[1].keys():
+ if 'TRMMArea' not in list(eachdict[1].keys()):
eachdict[1]['TRMMArea'] = TRMMArea
- if not 'CETRMMmin' in eachdict[1].keys():
+ if 'CETRMMmin' not in list(eachdict[1].keys()):
eachdict[1]['CETRMMmin'] = minCEprecipRate
- if not 'CETRMMmax' in eachdict[1].keys():
+ if 'CETRMMmax' not in list(eachdict[1].keys()):
eachdict[1]['CETRMMmax'] = maxCEprecipRate
- #clean up
+ # clean up
precipTotal = 0.0
- latsrawTRMMData =[]
+ latsrawTRMMData = []
lonsrawTRMMData = []
- latsrawCloudElements=[]
- lonsrawCloudElements=[]
- finalCETRMMvalues =[]
- CEprecipRate =[]
- brightnesstemp =[]
- TRMMdataDict ={}
+ latsrawCloudElements = []
+ lonsrawCloudElements = []
+ finalCETRMMvalues = []
+ CEprecipRate = []
+ brightnesstemp = []
+ TRMMdataDict = {}
return allCEnodesTRMMdata
-#******************************************************************
+#******************************************************************
+
+
def findCloudClusters(CEGraph):
'''
- Purpose::
- Determines the cloud clusters properties from the subgraphs in
- the graph i.e. prunes the graph according to the minimum depth
+ Purpose::
+ Determines the cloud clusters properties from the subgraphs in
+ the graph i.e. prunes the graph according to the minimum depth
- Input::
- CEGraph: a Networkx directed graph of the CEs with weighted edges
- according the area overlap between nodes (CEs) of consectuive frames
-
- Output::
- PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
+ Input::
+ CEGraph: a Networkx directed graph of the CEs with weighted edges
+ according the area overlap between nodes (CEs) of consectuive frames
+ Output::
+ PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs
-
'''
seenNode = []
@@@ -1112,148 -924,105 +1111,147 @@@ def findMCC(prunedGraph)
MCCList = checkedNodesMCC(prunedGraph, treeTraversalList)
for aDict in MCCList:
for eachNode in aDict["fullMCSMCC"]:
- addNodeMCSIdentifier(eachNode[0],eachNode[1])
-
- #do check for if MCCs overlap
+ addNodeMCSIdentifier(eachNode[0], eachNode[1])
+
+ # do check for if MCCs overlap
if MCCList:
if len(MCCList) > 1:
- for count in range(len(MCCList)): #for eachDict in MCCList:
- #if there are more than two lists
+ # for eachDict in MCCList:
+ for count in range(len(MCCList)):
+ # if there are more than two lists
if count >= 1:
- #and the first node in this list
- eachList = list(x[0] for x in MCCList[count]["possMCCList"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ # and the first node in this list
+ eachList = list(
+ x[0] for x in MCCList[count]["possMCCList"])
+ eachList.sort(key=lambda nodeID: (
+ len(nodeID.split('C')[0]), nodeID.split('C')[0]))
if eachList:
fNode = eachList[0]
- #get the lastNode in the previous possMCC list
- eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ # get the lastNode in the previous possMCC list
+ eachList = list(
+ x[0] for x in MCCList[(count - 1)]["possMCCList"])
+ eachList.sort(key=lambda nodeID: (
+ len(nodeID.split('C')[0]), nodeID.split('C')[0]))
if eachList:
lNode = eachList[-1]
- if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
- for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode):
+ if lNode in CLOUD_ELEMENT_GRAPH.predecessors(
+ fNode):
+ for aNode in CLOUD_ELEMENT_GRAPH.predecessors(
+ fNode):
if aNode in eachList and aNode == lNode:
- #if edge_data is equal or less than to the exisitng edge in the tree append one to the other
- if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']:
- MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"])
- MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"])
- MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"]
- MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
- MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
- MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"]
+ # if
+ # edge_data
+ # is
+ # equal
+ # or
+ # less
+ # than
+ # to
+ # the
+ # exisitng
+ # edge
+ # in
+ # the
+ # tree
+ # append
+ # one
+ # to
+ # the
+ # other
+ if CLOUD_ELEMENT_GRAPH.get_edge_data(
+ aNode, fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(
+ lNode, fNode)['weight']:
+ MCCList[count - 1]["possMCCList"].extend(
+ MCCList[count]["possMCCList"])
+ MCCList[count - 1]["fullMCSMCC"].extend(
+ MCCList[count]["fullMCSMCC"])
+ MCCList[count -
+ 1]["durationAandB"] += MCCList[count]["durationAandB"]
+ MCCList[count -
+ 1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"]
+ MCCList[count -
+ 1]["highestMCCnode"] = MCCList[count]["highestMCCnode"]
+ MCCList[count -
+ 1]["frameNum"] = MCCList[count]["frameNum"]
removeList.append(count)
- #update the MCCList
+ # update the MCCList
if removeList:
for i in removeList:
- if (len(MCCList)-1) > i:
+ if (len(MCCList) - 1) > i:
del MCCList[i]
- removeList =[]
-
- #check if the nodes also meet the duration criteria and the shape crieria
+ removeList = []
+
+ # check if the nodes also meet the duration criteria and the shape
+ # crieria
for eachDict in MCCList:
- #order the fullMCSMCC list, then run maximum extent and eccentricity criteria
- if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION:
+ # order the fullMCSMCC list, then run maximum extent and
+ # eccentricity criteria
+ if (eachDict["durationAandB"] *
+ TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] *
+ TRES) <= MAXIMUM_DURATION:
eachList = list(x[0] for x in eachDict["fullMCSMCC"])
- eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+ eachList.sort(key=lambda nodeID: (
+ len(nodeID.split('C')[0]), nodeID.split('C')[0]))
eachMCCList = list(x[0] for x in eachDict["possMCCList"])
- eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0]))
-
- #update the nodemcsidentifer behavior
- #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
- #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
- #ensure that everything between is listed as 'M'
- for eachNode in eachList[:(eachList.index(eachMCCList[0]))]:
- addNodeMCSIdentifier(eachNode,'I')
-
- addNodeMCSIdentifier(eachMCCList[0],'M')
-
- for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]:
+ eachMCCList.sort(key=lambda nodeID: (
+ len(nodeID.split('C')[0]), nodeID.split('C')[0]))
+
+ # update the nodemcsidentifer behavior
+ # find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I',
+ # find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D'
+ # ensure that everything between is listed as 'M'
+ for eachNode in eachList[:(
+ eachList.index(eachMCCList[0]))]:
+ addNodeMCSIdentifier(eachNode, 'I')
+
+ addNodeMCSIdentifier(eachMCCList[0], 'M')
+
+ for eachNode in eachList[(
+ eachList.index(eachMCCList[-1]) + 1):]:
addNodeMCSIdentifier(eachNode, 'D')
- #update definiteMCS list
- for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]:
+ # update definiteMCS list
+ for eachNode in orderedPath[(
+ orderedPath.index(eachMCCList[-1]) + 1):]:
addNodeMCSIdentifier(eachNode, 'D')
- #run maximum extent and eccentricity criteria
- maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList)
- #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag
- if definiteMCCFlag == True:
+ # run maximum extent and eccentricity criteria
+ maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(
+ eachList)
+ if definiteMCCFlag:
definiteMCC.append(eachList)
-
definiteMCS.append(orderedPath)
-
- #reset for next subGraph
+
+ # reset for next subGraph
aSubGraph.clear()
- orderedPath=[]
- MCCList =[]
- MCSList =[]
+ orderedPath = []
+ MCCList = []
+ MCSList = []
definiteMCSFlag = False
-
+
return definiteMCC, definiteMCS
#******************************************************************
-def traverseTree(subGraph,node, stack, checkedNodes=None):
+
+
+def traverseTree(subGraph, node, stack, checkedNodes=None):
'''
- Purpose::
- To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+ Purpose::
+ To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm
+
+ Input::
+ subGraph: a Networkx DiGraph representing a CC
+ lengthOfsubGraph: an integer representing the length of the subgraph
+ node: a string representing the node currently being checked
+ stack: a list of strings representing a list of nodes in a stack functionality
+ i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node
+ checkedNodes: a list of strings representing the list of the nodes in the traversal
- Input::
- subGraph: a Networkx DiGraph representing a CC
- lengthOfsubGraph: an integer representing the length of the subgraph
- node: a string representing the node currently being checked
- stack: a list of strings representing a list of nodes in a stack functionality
- i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node
+ Output::
checkedNodes: a list of strings representing the list of the nodes in the traversal
-
- Output::
- checkedNodes: a list of strings representing the list of the nodes in the traversal
- Assumptions:
- frames are ordered and are equally distributed in time e.g. hrly satellite images
-
+ Assumptions:
+ frames are ordered and are equally distributed in time e.g. hrly satellite images
-
'''
if len(checkedNodes) == len(subGraph):
return checkedNodes
@@@ -1270,10 -1038,10 +1268,9 @@@
if parent not in checkedNodes and parent not in stack:
for child in downOneLevel:
if child not in checkedNodes and child not in stack:
- stack.insert(0,child)
-
- stack.insert(0,parent)
+ stack.insert(0, child)
+ stack.insert(0, parent)
-
for child in downOneLevel:
if child not in checkedNodes and child not in stack:
if len(subGraph.predecessors(child)) > 1 or node in checkedNodes:
@@@ -1556,62 -1231,38 +1553,63 @@@ def updateMCCList
potentialMCCList[index]["highestMCCnode"] = node
return potentialMCCList
- #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list
- if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0:
+ # if this frameNum doesn't exist and this frameNum is less than
+ # the MCC node max frame Num (including 0), then append to
+ # fullMCSMCC list
+ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
stage = 'I'
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ if CounterCriteriaAFlag and CounterCriteriaBFlag == False:
+ potentialMCCList.append(
+ {
+ "possMCCList": [],
+ "fullMCSMCC": [
+ (node,
+ stage)],
+ "CounterCriteriaA": 1,
+ "durationAandB": 0,
+ "highestMCCnode": "",
+ "frameNum": 0})
return potentialMCCList
elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False:
- potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0})
+ potentialMCCList.append(
+ {
+ "possMCCList": [],
+ "fullMCSMCC": [
+ (node,
+ stage)],
+ "CounterCriteriaA": 0,
+ "durationAandB": 0,
+ "highestMCCnode": "",
+ "frameNum": 0})
return potentialMCCList
- #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list
- if existingFrameFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
+ # if predecessor and this frame number already exist in the MCC
+ # list, add the current node to the fullMCSMCC list
+ if existingFrameFlag:
+ if CounterCriteriaAFlag and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append((node, stage))
+ potentialMCCList[index]["CounterCriteriaA"] += 1
return potentialMCCList
- if CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- return potentialMCCList
-
- if predecessorsFlag == False:
- successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
-
- if successorsFlag == True:
- for eachNode in potentialMCCList[index]["possMCCList"]:
- if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
+ if not CounterCriteriaAFlag:
+ potentialMCCList[index]["fullMCSMCC"].append((node, stage))
+ return potentialMCCList
+
+ if not predecessorsFlag:
+ successorsFlag, index = isThereALink(
+ prunedGraph, 2, node, potentialMCCList, 2)
+
+ if successorsFlag:
+ for eachNode in potentialMCCList[index]["possMCCList"]:
+ if int(
+ (eachNode[0].split('CE')[0]).split('F')[1]) == frameNum:
existingFrameFlag = True
-
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
+
+ if CounterCriteriaAFlag and CounterCriteriaBFlag:
stage = 'M'
- potentialMCCList[index]["possMCCList"].append((node,stage))
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["possMCCList"].append(
+ (node, stage))
+ potentialMCCList[index]["fullMCSMCC"].append((node, stage))
++
if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0:
potentialMCCList[index]["frameNum"] = frameNum
potentialMCCList[index]["highestMCCnode"] = node
@@@ -1660,24 -1305,19 +1658,25 @@@
return potentialMCCList
if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
- potentialMCCList[index]["CounterCriteriaA"] +=1
+ if CounterCriteriaAFlag and CounterCriteriaBFlag == False:
+ potentialMCCList[index]["fullMCSMCC"].append(
+ (node, stage))
+ potentialMCCList[index]["CounterCriteriaA"] += 1
return potentialMCCList
elif CounterCriteriaAFlag == False:
- potentialMCCList[index]["fullMCSMCC"].append((node,stage))
+ potentialMCCList[index]["fullMCSMCC"].append(
+ (node, stage))
return potentialMCCList
else:
- successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2)
- if successorsMCSFlag == True:
- if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True:
- potentialMCCList[index]["possMCCList"].append((node,'M'))
- potentialMCCList[index]["fullMCSMCC"].append((node,'M'))
+ successorsMCSFlag, index = isThereALink(
+ prunedGraph, 2, node, potentialMCCList, 2)
+ if successorsMCSFlag:
+ if CounterCriteriaAFlag and CounterCriteriaBFlag:
+ potentialMCCList[index]["possMCCList"].append(
+ (node, 'M'))
+ potentialMCCList[index]["fullMCSMCC"].append(
+ (node, 'M'))
++
potentialMCCList[index]["durationAandB"] += 1
if frameNum > potentialMCCList[index]["frameNum"]:
potentialMCCList[index]["frameNum"] = frameNum
@@@ -2071,24 -1641,22 +2070,23 @@@ def allDescendants(path, aNode)
path = path + aNode
#i.e. PRUNED_GRAPH.predecessors(aNode) is empty
return path, numOfChildren
- except:
- #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
+ # i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception
+ except BaseException:
return path, numOfChildren
#******************************************************************
-def addInfothisDict (thisNode, cloudElementArea,criteriaB):
- '''
- Purpose::
- Update original dictionary node with information
- Input::
- thisNode: a string representing the unique ID of a node
- cloudElementArea: a floating-point number representing the area of the cloud element
- criteriaB: a masked array of floating-point numbers representing the lat,lons meeting the criteria
- Output:: None
+def addInfothisDict(thisNode, cloudElementArea, criteriaB):
+ '''
+ Purpose::
+ Update original dictionary node with information
+
+ Input::
+ thisNode: a string representing the unique ID of a node
+ cloudElementArea: a floating-point number representing the area of the cloud element
+ criteriaB: a masked array of floating-point numbers representing the lat,lons meeting the criteria
+ Output:: None
-
'''
for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode):
if eachdict[1]['uniqueID'] == thisNode:
@@@ -3323,29 -2737,24 +3321,28 @@@ def displaySize(finalMCCList)
ax.set_title(title)
ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d%H:%M:%S')
fig.autofmt_xdate()
-
-
plt.subplots_adjust(bottom=0.2)
-
- imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif'
- plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True)
-
- #if time in not already in the time list, append it
- timeList=[]
+
+ imgFilename = MAINDIRECTORY + '/images/' + str(count) + 'MCS.gif'
+ plt.savefig(
+ imgFilename,
+ facecolor=fig.get_facecolor(),
+ transparent=True)
+
+ # if time in not already in the time list, append it
+ timeList = []
count += 1
- return
+ return
#******************************************************************
-def displayPrecip(finalMCCList):
+
+
+def displayPrecip(finalMCCList):
'''
- Purpose::
- To create a figure showing the precip rate verse time for each MCS
+ Purpose::
+ To create a figure showing the precip rate verse time for each MCS
- Input::
- finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
+ Input::
+ finalMCCList: a list of dictionaries representing a list of nodes representing a MCC
Output:: None
@@@ -3730,25 -3093,24 +3727,24 @@@ def plotAccuInTimeRange(starttime, endt
accuTRMMData.createDimension('time', None)
accuTRMMData.createDimension('lat', nygrdTRMM)
accuTRMMData.createDimension('lon', nxgrdTRMM)
-
+
# variables
- TRMMprecip = ('time','lat', 'lon',)
+ TRMMprecip = ('time', 'lat', 'lon',)
times = accuTRMMData.createVariable('time', 'f8', ('time',))
- times.units = 'hours since '+ starttime[:-6]
+ times.units = 'hours since ' + starttime[:-6]
latitude = accuTRMMData.createVariable('latitude', 'f8', ('lat',))
longitude = accuTRMMData.createVariable('longitude', 'f8', ('lon',))
- rainFallacc = accuTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip)
+ rainFallacc = accuTRMMData.createVariable(
+ 'precipitation_Accumulation', 'f8', TRMMprecip)
rainFallacc.units = 'mm'
- longitude[:] = LONTRMM[0,:]
- longitude.units = "degrees_east"
- longitude.long_name = "Longitude"
+ longitude[:] = LONTRMM[0, :]
+ longitude.units = "degrees_east"
+ longitude.long_name = "Longitude"
- latitude[:] = LATTRMM[:,0]
+ latitude[:] = LATTRMM[:, 0]
latitude.units = "degrees_north"
- latitude.long_name ="Latitude"
-
+ latitude.long_name = "Latitude"
-
rainFallacc[:] = accuPrecipRate[:]
accuTRMMData.close()
@@@ -4235,22 -3515,18 +4231,23 @@@ def createTextFile(finalMCCList, identi
#******************************************************************
# PLOTTING UTIL SCRIPTS
#******************************************************************
-def to_percent(y,position):
+
+
+def to_percent(y, position):
'''
- Purpose::
- Utility script for generating the y-axis for plots
+ Purpose::
+ Utility script for generating the y-axis for plots
'''
- return (str(100*y)+'%')
+ return (str(100 * y) + '%')
#******************************************************************
+
+
def colorbar_index(ncolors, nlabels, cmap):
'''
- Purpose::
- Utility script for crating a colorbar
- Taken from http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks
+ Purpose::
+ Utility script for crating a colorbar
+ Taken from http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks
++
'''
cmap = cmap_discretize(cmap, ncolors)
mappable = cm.ScalarMappable(cmap=cmap)
@@@ -4293,22 -3567,22 +4290,19 @@@ def cmap_discretize(cmap, N)
# def preprocessingMERG(MERGdirname):
# '''
# Purpose::
-# Utility script for unzipping and converting the merg*.Z files from Mirador to
+# Utility script for unzipping and converting the merg*.Z files from Mirador to
# NETCDF format. The files end up in a folder called mergNETCDF in the directory
# where the raw MERG data is
-# NOTE: VERY RAW AND DIRTY
-
+# NOTE: VERY RAW AND DIRTY
-
# Input::
# Directory to the location of the raw MERG files, preferably zipped
-
-
# Output::
# none
--
# Assumptions::
# 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/)
-# have been installed on the system and the user can access
+# have been installed on the system and the user can access
# 2 User can write files in location where script is being called
-# 3 the files havent been unzipped
+# 3 the files havent been unzipped
# '''
# os.chdir((MERGdirname+'/'))
http://git-wip-us.apache.org/repos/asf/climate/blob/51d9dce1/mccsearch/code/mccSearchUI.py
----------------------------------------------------------------------
diff --cc mccsearch/code/mccSearchUI.py
index 36fe642,a227143..feea1d9
--- a/mccsearch/code/mccSearchUI.py
+++ b/mccsearch/code/mccSearchUI.py
@@@ -18,10 -18,10 +18,11 @@@
# Wizard for running the mccSearch program
'''
+ import os
import networkx as nx
-#mccSearch modules
-import mccSearch
+# mccSearch modules
+from mccSearch import *
+
def main():
CEGraph = nx.DiGraph()