You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by rk...@apache.org on 2023/05/16 22:16:20 UTC
[incubator-sdap-nexus] branch SDAP-465 created (now c917213)
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a change to branch SDAP-465
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
at c917213 Delete climatology directory
This branch includes the following new commits:
new c917213 Delete climatology directory
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-nexus] 01/01: Delete climatology directory
Posted by rk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch SDAP-465
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit c917213cf24c177104549dacad69368afd8917fd
Author: Riley Kuttruff <72...@users.noreply.github.com>
AuthorDate: Tue May 16 15:16:15 2023 -0700
Delete climatology directory
---
climatology/.gitignore | 11 -
climatology/clim/ClimatologySpark.py | 469 -----------
climatology/clim/ClimatologySpark2.py | 650 ----------------
climatology/clim/README.md | 32 -
climatology/clim/__init__.py | 14 -
climatology/clim/binsum.f | 64 --
climatology/clim/cache.py | 87 ---
climatology/clim/climatology.py | 247 ------
climatology/clim/climatology1.py | 247 ------
climatology/clim/climatology2.py | 467 -----------
climatology/clim/climatology3Spark.py | 432 -----------
climatology/clim/cluster.py | 98 ---
climatology/clim/cluster2.py | 98 ---
climatology/clim/datasets.py | 331 --------
climatology/clim/dparkTest.py | 23 -
climatology/clim/gaussInterp.py | 57 --
climatology/clim/gaussInterp.pyx | 145 ----
climatology/clim/gaussInterp_f.f | 219 ------
climatology/clim/gaussInterp_f.mk | 1 -
climatology/clim/gaussInterp_slow.py | 144 ----
climatology/clim/interp.f | 302 --------
climatology/clim/jobClimatology2.py | 36 -
climatology/clim/jobTest.py | 32 -
climatology/clim/orig/C/README | 6 -
climatology/clim/orig/C/binsum.c | 125 ---
climatology/clim/orig/C/clouderosion.c | 33 -
climatology/clim/orig/C/gaussinterp.readme | 159 ----
climatology/clim/orig/C/gaussinterp_C_code.tar | Bin 51200 -> 0 bytes
climatology/clim/orig/C/interp.c | 448 -----------
climatology/clim/orig/C/makefile | 33 -
climatology/clim/orig/C/setupinterp.c | 431 -----------
.../clim/orig/Fortran/armstrong_interp_code.tar | Bin 30720 -> 0 bytes
climatology/clim/orig/Fortran/binsum.f | 64 --
climatology/clim/orig/Fortran/interp.f | 302 --------
climatology/clim/orig/Fortran/makefile | 46 --
climatology/clim/orig/Fortran/passbase.f | 9 -
climatology/clim/orig/Fortran/setupinterp.f | 291 -------
climatology/clim/pixelStats.py | 232 ------
climatology/clim/plotlib.py | 857 ---------------------
climatology/clim/reroot.py | 45 --
climatology/clim/setup.py | 22 -
climatology/clim/sort.py | 57 --
climatology/clim/sparkTest.py | 31 -
climatology/clim/spatialFilter.py | 50 --
climatology/clim/spatialFilter_f.f | 121 ---
climatology/clim/spatialFilter_f.mk | 1 -
climatology/clim/split.py | 212 -----
climatology/clim/test/__init__.py | 14 -
climatology/clim/test/ccmpTest.py | 30 -
climatology/clim/timePartitions.py | 46 --
climatology/clim/util/__init__.py | 14 -
climatology/clim/util/array.py | 194 -----
climatology/clim/util/introspect.py | 49 --
climatology/clim/util/plot.py | 147 ----
climatology/clim/util/stats.py | 232 ------
climatology/clim/util/timeJ2000.py | 383 ---------
climatology/clim/util/warn.py | 57 --
climatology/clim/util/wls.py | 811 -------------------
climatology/clim/variables.py | 154 ----
climatology/clim/wls.py | 811 -------------------
climatology/setup.py | 23 -
61 files changed, 10746 deletions(-)
diff --git a/climatology/.gitignore b/climatology/.gitignore
deleted file mode 100644
index 8ff7b03..0000000
--- a/climatology/.gitignore
+++ /dev/null
@@ -1,11 +0,0 @@
-.DS_Store
-
-dist
-
-build
-
-*.egg-info
-
-*_f.so*
-
-*.nc
\ No newline at end of file
diff --git a/climatology/clim/ClimatologySpark.py b/climatology/clim/ClimatologySpark.py
deleted file mode 100755
index b474b76..0000000
--- a/climatology/clim/ClimatologySpark.py
+++ /dev/null
@@ -1,469 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-ClimatologySpark.py
-
-Compute a multi-epoch (multi-day) climatology from daily SST Level-3 grids.
-
-Simple code to be run on Spark cluster, or using multi-core parallelism on single machine.
-
-"""
-
-import sys, os, urllib.parse, urllib.request, urllib.parse, urllib.error, re, time
-import numpy as N
-import matplotlib
-matplotlib.use('Agg')
-import matplotlib.pylab as M
-
-from .variables import getVariables, close
-from .cache import retrieveFile, CachePath
-from .split import fixedSplit, splitByNDays
-from netCDF4 import Dataset, default_fillvals
-from pathos.multiprocessing import ProcessingPool as Pool
-from .plotlib import imageMap, makeMovie
-
-from .spatialFilter import spatialFilter
-from .gaussInterp import gaussInterp # calls into Fortran version gaussInterp_f.so
-#from gaussInterp_slow import gaussInterp_slow as gaussInterp # pure python, slow debuggable version
-
-#import pyximport; pyximport.install(pyimport=False)
-#from gaussInterp import gaussInterp, gaussInterp_ # attempted cython version, had issues
-
-VERBOSE = 1
-
-# Possible execution modes
-# Multicore & cluster modes use pathos pool.map(); Spark mode uses PySpark cluster.
-ExecutionModes = ['sequential', 'multicore', 'cluster', 'spark']
-
-# SST L3m 4.6km Metadata
-# SST calues are scaled integers in degrees Celsius, lon/lat is 8640 x 4320
-# Variable = 'sst', Mask = 'qual_sst', Coordinates = ['lon', 'lat']
-
-# Generate algorithmic name for N-day Climatology product
-SSTClimatologyTemplate = 'SST.L3.Global.Clim.%(period)s.%(date)s.%(version)s.nc' #??
-
-# Simple mask and average functions to get us started, then add gaussian interpolation.
-# MODIS L3 SST product, qual_sst is [-1, 2] - =0 is best data, can add =1 for better coverage
-#def qcMask(var, mask): return N.ma.array(var, mask=N.ma.make_mask(mask == 0))
-
-def qcMask(var, mask): return N.ma.masked_where(mask != 0, var)
-#def qcMask(var, mask): return N.ma.masked_where(mask < 0, var)
-
-def splitModisSst(seq, n):
- for chunk in splitByNDays(seq, n, re.compile(r'(...).L3m')):
- yield chunk
-
-def mean(a): return N.ma.mean(a, axis=0)
-
-AveragingFunctions = {'pixelMean': mean, 'gaussInterp': gaussInterp, 'spatialFilter': spatialFilter}
-
-PixelMeanConfig = {'name': 'pixelMean'}
-
-GaussInterpConfig = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 3, 'wlon': 3,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig1a = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.30, 'wlon': 0.30,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig1b = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.08, 'wlon': 0.08,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig2 = {'name': 'gaussInterp',
- 'latGrid': (89.5, -89.5, -0.25), 'lonGrid': (-180., 179., 0.25),
- 'wlat': 2., 'wlon': 2.,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-FilterGaussian = [[1, 2, 1], [2, 4, 2], [1, 2, 1]] # divide by 16
-FilterLowPass = [[1, 1, 1], [1, 1, 1], [1, 1, 1]] # divide by 9
-
-SpatialFilterConfig1 = {'name': 'spatialFilter', 'normalization': 16.,
- 'spatialFilter': N.array(FilterGaussian, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-SpatialFilterConfig2 = {'name': 'spatialFilter', 'normalization': 9.,
- 'spatialFilter': N.array(FilterLowPass, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-# Directory to cache retrieved files in
-CachePath = '~/cache'
-
-
-def climByAveragingPeriods(urls, # list of (daily) granule URLs for a long time period (e.g. a year)
- nEpochs, # compute a climatology for every N epochs (days) by 'averaging'
- nWindow, # number of epochs in window needed for averaging
- nNeighbors, # number of neighbors on EACH side in lat/lon directions to use in averaging
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- splitFn=splitModisSst, # split function to use to partition the input URL list
- maskFn=qcMask, # mask function to compute mask from mask variable
- averager='pixelMean', # averaging function to use, one of ['pixelMean', 'gaussInterp', 'spatialFilter']
- averagingConfig={}, # dict of parameters to control the averaging function (e.g. gaussInterp)
- optimization='fortran', # optimization mode (fortran or cython)
- mode='sequential', # Map across time periods of N-days for concurrent work, executed by:
- # 'sequential' map, 'multicore' using pool.map(), 'cluster' using pathos pool.map(),
- # or 'spark' using PySpark
- numNodes=1, # number of cluster nodes to use
- nWorkers=4, # number of parallel workers per node
- averagingFunctions=AveragingFunctions, # dict of possible averaging functions
- legalModes=ExecutionModes, # list of possible execution modes
- cachePath=CachePath # directory to cache retrieved files in
- ):
- '''Compute a climatology every N days by applying a mask and averaging function.
-Writes the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- if averagingConfig['name'] == 'gaussInterp':
- averagingConfig['wlat'] = nNeighbors
- averagingConfig['wlon'] = nNeighbors
- try:
- averageFn = averagingFunctions[averager]
- except:
- print('climatology: Error, Averaging function must be one of: %s' % str(averagingFunctions), file=sys.stderr)
- sys.exit(1)
-
- urlSplits = [s for s in splitFn(urls, nEpochs)]
-
- def climsContoured(urls, plot=None, fillValue=default_fillvals['f4'], format='NETCDF4', cachePath=cachePath):
- n = len(urls)
- if VERBOSE: print(urls, file=sys.stderr)
-
- var = climByAveraging(urls, variable, mask, coordinates, maskFn, averageFn, averagingConfig, optimization, cachePath)
-
- fn = os.path.split(urls[0])[1]
- inFile = os.path.join(cachePath, fn)
- method = averagingConfig['name']
- fn = os.path.splitext(fn)[0]
- day = fn[5:8]
- nDays = int(var['time'][0])
-
- if 'wlat' in averagingConfig:
- wlat = averagingConfig['wlat']
- else:
- wlat = 1
- if int(wlat) == wlat:
- outFile = 'A%s.L3m_%dday_clim_sst_4km_%s_%dnbrs.nc' % (day, nDays, method, int(wlat)) # mark each file with first day in period
- else:
- outFile = 'A%s.L3m_%dday_clim_sst_4km_%s_%4.2fnbrs.nc' % (day, nDays, method, wlat) # mark each file with first day in period
-
- outFile = writeOutNetcdfVars(var, variable, mask, coordinates, inFile, outFile, fillValue, format)
-
- if plot == 'contour':
- figFile = contourMap(var, variable, coordinates, n, outFile)
- elif plot == 'histogram':
-# figFile = histogram(var, variable, n, outFile)
- figFile = None
- else:
- figFile = None
- return (outFile, figFile)
-
- if mode == 'sequential':
- results = list(map(climsContoured, urlSplits))
- elif mode == 'multicore':
- pool = Pool(nWorkers)
- results = pool.map(climsContoured, urlSplits)
- elif mode == 'cluster':
- pass
- elif mode == 'spark':
- pass
-
- return results
-
-
-def climByAveraging(urls, # list of granule URLs for a time period
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- maskFn=qcMask, # mask function to compute mask from mask variable
- averageFn=mean, # averaging function to use
- averagingConfig={}, # parameters to control averaging function (e.g. gaussInterp)
- optimization='fortran', # optimization mode (fortran or cython)
- cachePath=CachePath
- ):
- '''Compute a climatology over N arrays by applying a mask and averaging function.
-Returns the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- n = len(urls)
- varList = [variable, mask]
- var = {}
- vtime = N.zeros((n,), N.int32)
-
- for i, url in enumerate(urls):
- try:
- path = retrieveFile(url, cachePath)
- fn = os.path.split(path)[1]
- vtime[i] = int(fn[5:8]) # KLUDGE: extract DOY from filename
- except:
- print('climByAveraging: Error, continuing without file %s' % url, file=sys.stderr)
- accum[i] = emptyVar
- continue
- if path is None: continue
- print('Read variables and mask ...', file=sys.stderr)
- try:
- var, fh = getVariables(path, varList, arrayOnly=True, order='F', set_auto_mask=False) # return dict of variable objects by name
- except:
- print('climByAveraging: Error, cannot read file %s' % path, file=sys.stderr)
- accum[i] = emptyVar
- continue
- if i == 0:
- dtype = var[variable].dtype
- if 'int' in dtype.name: dtype = N.float32
- shape = (n,) + var[variable].shape
- accum = N.ma.empty(shape, dtype, order='F')
- emptyVar = N.array(N.ma.masked_all(var[variable].shape, dtype), order='F') # totally masked variable array for missing or bad file reads
-
- print('Read coordinates ...', file=sys.stderr)
- var, fh = getVariables(path, coordinates, var, arrayOnly=True, order='F') # read coordinate arrays and add to dict
-
- var[variable] = maskFn(var[variable], var[mask]) # apply quality mask variable to get numpy MA, turned off masking done by netCDF4 library
-# var[variable] = var[variable][:]
-
- # Echo variable range for sanity check
- vals = var[variable].compressed()
- print('Variable Range: min, max:', vals.min(), vals.max(), file=sys.stderr)
-
- # Plot input grid
-# figFile = histogram(vals, variable, n, os.path.split(path)[1])
-# figFile = contourMap(var, variable, coordinates[1:], n, os.path.split(path)[1])
-
- accum[i] = var[variable] # accumulate N arrays for 'averaging"""
-# if i != 0 and i+1 != n: close(fh) # REMEMBER: closing fh loses netCDF4 in-memory data structures
- close(fh)
-
- coordinates = ['time'] + coordinates # add constructed time (days) as first coordinate
- var['time'] = vtime
-
- if averagingConfig['name'] == 'pixelMean':
- print('Doing Pixel Average over %d grids ...' % n, file=sys.stderr)
- start = time.time()
- avg = averageFn(accum) # call pixel averaging function
- end = time.time()
- print('pixelMean execution time:', (end - start), file=sys.stderr)
- outlat = var[coordinates[1]].astype(N.float32)[:]
- outlon = var[coordinates[2]].astype(N.float32)[:]
- elif averagingConfig['name'] == 'gaussInterp':
- print('Doing Gaussian Interpolation over %d grids ...' % n, file=sys.stderr)
- var[variable] = accum
- c = averagingConfig
- latGrid = c['latGrid']; lonGrid = c['lonGrid']
- if latGrid is not None and lonGrid is not None:
- outlat = N.arange(latGrid[0], latGrid[1]+latGrid[2], latGrid[2], dtype=N.float32, order='F')
- outlon = N.arange(lonGrid[0], lonGrid[1]+lonGrid[2], lonGrid[2], dtype=N.float32, order='F')
- else:
- outlat = N.array(var[coordinates[1]], dtype=N.float32, order='F')
- outlon = N.array(var[coordinates[2]], dtype=N.float32, order='F')
- varNames = [variable] + coordinates
- start = time.time()
- avg, weight, status = \
- gaussInterp(var, varNames, outlat, outlon, c['wlat'], c['wlon'],
- c['slat'], c['slon'], c['stime'], c['vfactor'], c['missingValue'],
- VERBOSE, optimization)
- end = time.time()
- var['outweight'] = weight.astype(N.float32)
- print('gaussInterp execution time:', (end - start), file=sys.stderr)
- elif averagingConfig['name'] == 'spatialFilter':
- print('Applying Spatial 3x3 Filter and then averaging over %d grids ...' % n, file=sys.stderr)
- var[variable] = accum
- c = averagingConfig
- varNames = [variable] + coordinates
- start = time.time()
- avg, count, status = \
- spatialFilter(var, varNames, c['spatialFilter'], c['normalization'],
- c['missingValue'], VERBOSE, optimization)
- end = time.time()
- print('spatialFilter execution time:', (end - start), file=sys.stderr)
- outlat = var[coordinates[1]].astype(N.float32)[:]
- outlon = var[coordinates[2]].astype(N.float32)[:]
-
- var['out'+variable] = avg.astype(N.float32) # return primary variable & mask arrays in dict
- var['out'+mask] = N.ma.getmask(avg)
- var['outlat'] = outlat
- var['outlon'] = outlon
- return var
-
-
-def writeOutNetcdfVars(var, variable, mask, coordinates, inFile, outFile, fillValue=None, format='NETCDF4'):
- '''Construct output bundle of arrays with NetCDF dimensions and attributes.
-Output variables and attributes will have same names as the input file.
- '''
- din = Dataset(inFile, 'r')
- dout = Dataset(outFile, 'w', format=format)
- print('Writing %s ...' % outFile, file=sys.stderr)
-
- # Transfer global attributes from input file
- for a in din.ncattrs():
- dout.setncattr(a, din.getncattr(a))
-
- # Add dimensions and variables, copying data
- coordDim = [dout.createDimension(coord, var['out'+coord].shape[0]) for coord in coordinates] # here output lon, lat, etc.
- for coord in coordinates:
- v = dout.createVariable(coord, var['out'+coord].dtype, (coord,))
- v[:] = var['out'+coord][:]
- primaryVar = dout.createVariable(variable, var['out'+variable].dtype, coordinates, fill_value=fillValue)
- primaryVar[:] = var['out'+variable][:] # transfer array
- maskVar = dout.createVariable(mask, 'i1', coordinates)
- maskVar[:] = var['out'+mask].astype('i1')[:]
-
- # Transfer variable attributes from input file
- for k,v in dout.variables.items():
- for a in din.variables[k].ncattrs():
- if a == 'scale_factor' or a == 'add_offset' or a == '_FillValue': continue
- v.setncattr(a, din.variables[k].getncattr(a))
- if k == variable:
- try:
-# if fillValue == None: fillValue = din.variables[k].getncattr('_FillValue') # total kludge
- if fillValue == None: fillValue = default_fillvals['f4']
-# print >>sys.stderr, default_fillvals
-# v.setncattr('_FillValue', fillValue) # set proper _FillValue for climatology array
- v.setncattr('missing_value', fillValue)
- print('Setting missing_value for primary variable %s to %f' % (variable, fillValue), file=sys.stderr)
- except:
- print('writeOutNetcdfVars: Warning, for variable %s no fill value specified or derivable from inputs.' % variable, file=sys.stderr)
- din.close()
- dout.close()
- return outFile
-
-
-def contourMap(var, variable, coordinates, n, outFile):
- figFile = os.path.splitext(outFile)[0] + '_hist.png'
- # TODO: Downscale variable array (SST) before contouring, matplotlib is TOO slow on large arrays
- vals = var[variable][:]
-
- # Fixed color scale, write file, turn off auto borders, set title, reverse lat direction so monotonically increasing??
- imageMap(var[coordinates[1]][:], var[coordinates[0]][:], var[variable][:],
- vmin=-2., vmax=45., outFile=figFile, autoBorders=False,
- title='%s %d-day Mean from %s' % (variable.upper(), n, outFile))
- print('Writing contour plot to %s' % figFile, file=sys.stderr)
- return figFile
-
-
-def histogram(vals, variable, n, outFile):
- figFile = os.path.splitext(outFile)[0] + '_hist.png'
- M.clf()
-# M.hist(vals, 47, (-2., 45.))
- M.hist(vals, 94)
- M.xlim(-5, 45)
- M.xlabel('SST in degrees Celsius')
- M.ylim(0, 300000)
- M.ylabel('Count')
- M.title('Histogram of %s %d-day Mean from %s' % (variable.upper(), n, outFile))
- M.show()
- print('Writing histogram plot to %s' % figFile, file=sys.stderr)
- M.savefig(figFile)
- return figFile
-
-
-def dailyFile2date(path, offset=1):
- '''Convert YYYYDOY string in filename to date.'''
- fn = os.path.split(path)[1]
- year = int(fn[offset:offset+4])
- doy = int(fn[offset+5:offset+8])
- return fn[5:15].replace('.', '/')
-
-
-def formatRegion(r):
- """Format lat/lon region specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- strs = [str(i).replace('-', 'm') for i in r]
- return 'region-%s-%sby%s-%s' % tuple(strs)
-
-
-def formatGrid(r):
- """Format lat/lon grid resolution specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- return str(r[0]) + 'by' + str(r[1])
-
-
-def main(args):
- nEpochs = int(args[0])
- nWindow = int(args[1])
- nNeighbors = int(args[2])
- averager = args[3]
- optimization = args[4]
- mode = args[5]
- nWorkers = int(args[6])
- urlFile = args[7]
- urls = [s.strip() for s in open(urlFile, 'r')]
- if averager == 'gaussInterp':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=GaussInterpConfig,
- mode=mode, nWorkers=nWorkers)
- elif averager == 'spatialFilter':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=SpatialFilterConfig1,
- mode=mode, nWorkers=nWorkers)
- elif averager == 'pixelMean':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=PixelMeanConfig,
- mode=mode, nWorkers=nWorkers)
- else:
- print('climatology2: Error, averager must be one of', list(AveragingFunctions.keys()), file=sys.stderr)
- sys.exit(1)
-
- if results[0][1] is not None:
- makeMovie([r[1] for r in results], 'clim.mpg')
- return results
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-
-# Old Tests:
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_10days.txt
-# python climatology2.py 5 5 3 gaussInterp fortran sequential 1 urls_sst_10days.txt
-
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_40days.txt
-# python climatology2.py 5 5 0 pixelMean fortran multicore 8 urls_sst_40days.txt
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 8 urls_sst_40days.txt
-
-# Old Production:
-# python climatology2.py 5 5 0 pixelMean fortran multicore 16 urls_sst_2015.txt >& log &
-# python climatology2.py 10 10 0 pixelMean fortran multicore 16 urls_sst_2015.txt >& log &
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 16 urls_sst_2015.txt >& log &
-
-
-# Tests:
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 0 pixelMean fortran multicore 4 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 3 gaussInterp fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 4 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 7 1 spatialFilter fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-
-# Test number of neighbors needed:
-# python climatology2.py 5 7 3 gaussInterp fortran multicore 4 urls_sst_daynight_20days_sorted.txt
-
-
-# Production:
-# python climatology2.py 5 7 0 pixelMean fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 3 gaussInterp fortran sequential 1 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 3 gaussInterp fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 1 spatialFilter fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-
diff --git a/climatology/clim/ClimatologySpark2.py b/climatology/clim/ClimatologySpark2.py
deleted file mode 100755
index 9af63db..0000000
--- a/climatology/clim/ClimatologySpark2.py
+++ /dev/null
@@ -1,650 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-ClimatologySpark2.py
-
-Compute a multi-epoch (multi-day) climatology from daily SST Level-3 grids.
-
-Simple code to be run on Spark cluster, or using multi-core parallelism on single machine via pysparkling.
-
-"""
-
-import sys, os, urllib.parse, urllib.request, urllib.parse, urllib.error, re, time, glob
-import numpy as N
-import matplotlib
-
-matplotlib.use('Agg')
-import matplotlib.pylab as M
-
-from clim.variables import getVariables, close
-from clim.cache import retrieveFile, CachePath, hdfsCopyFromLocal
-from netCDF4 import Dataset, default_fillvals
-from clim.plotlib import imageMap, makeMovie
-
-from clim.spatialFilter import spatialFilter
-from clim.gaussInterp import gaussInterp # calls into Fortran version gaussInterp_f.so
-# from clim.gaussInterp_slow import gaussInterp_slow as gaussInterp # pure python, slow debuggable version
-
-from clim.datasets import DatasetList, ModisSst, ModisChlor, MeasuresSsh
-from clim.sort import sortByKeys
-from clim.split import groupByKeys, splitByNDaysKeyed
-
-from pyspark import SparkContext, SparkConf
-
-DaskClientEndpoint = "daskclient:8786"
-
-VERBOSE = 0
-CollectAndTime = 0
-
-# Possible execution modes
-# Multicore & cluster modes use pathos pool.map(); Spark mode uses PySpark cluster.
-ExecutionModes = ['spark', 'mesos', 'multicore']
-
-
-# SST L3m 4.6km Metadata
-# SST calues are scaled integers in degrees Celsius, lon/lat is 8640 x 4320
-# Variable = 'sst', Mask = 'qual_sst', Coordinates = ['lon', 'lat']
-
-# Simple mask and average functions to get us started, then add gaussian interpolation.
-# MODIS L3 SST product, qual_sst is [-1, 2] - =0 is best data, can add =1 for better coverage
-
-def qcMask(var, mask): return N.ma.masked_where(mask != 0, var)
-
-
-# Example Configurations
-PixelMeanConfig = {'name': 'pixelMean', 'accumulators': ['count', 'mean', 'M2']}
-
-GaussInterpConfig = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 3, 'wlon': 3,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4'],
- 'optimization': 'fortran'}
-
-GaussInterpConfig1a = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.30, 'wlon': 0.30,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4'],
- 'optimization': 'fortran'}
-
-GaussInterpConfig1b = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.08, 'wlon': 0.08,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4'],
- 'optimization': 'fortran'}
-
-GaussInterpConfig2 = {'name': 'gaussInterp',
- 'latGrid': (89.5, -89.5, -0.25), 'lonGrid': (-180., 179., 0.25),
- 'wlat': 2., 'wlon': 2.,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4'],
- 'optimization': 'fortran'}
-
-FilterGaussian = [[1, 2, 1], [2, 4, 2], [1, 2, 1]] # divide by 16
-FilterLowPass = [[1, 1, 1], [1, 1, 1], [1, 1, 1]] # divide by 9
-
-SpatialFilterConfig1 = {'name': 'spatialFilter', 'normalization': 16.,
- 'spatialFilter': N.array(FilterGaussian, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-SpatialFilterConfig2 = {'name': 'spatialFilter', 'normalization': 9.,
- 'spatialFilter': N.array(FilterLowPass, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-# Three kinds of averaging supported
-AveragingFunctions = {'pixelMean': None, 'gaussInterp': gaussInterp, 'spatialFilter': spatialFilter}
-AveragingConfigs = {'pixelMean': PixelMeanConfig, 'gaussInterp': GaussInterpConfig1a,
- 'spatialFilter': SpatialFilterConfig1}
-
-
-def climByAveragingPeriods(urls, # list of (daily) granule URLs for a long time period (e.g. 15 years), *SORTED* by DOY
- datasetInfo, # class holding dataset metadata
- nEpochs, # compute a climatology for every N epochs (days) by 'averaging'
- nWindow, # number of epochs in window needed for averaging
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- reader, # reader function that reads the variable from the file into a numpy masked array
- outHdfsPath,
- # HDFS path to write outputs to (i.e. netCDF file containing stats and optional contour plot)
- splitter, # split function to use to partition the input URL list
- masker=qcMask, # mask function to compute mask from mask variable
- averager='pixelMean',
- # averaging function to use, one of ['pixelMean', 'gaussInterp', 'spatialFilter']
- averagingConfig={},
- # dict of parameters to control the averaging function (e.g. gaussInterp)
- sparkConfig='mesos,8,16',
- # Map across time periods of N-days for concurrent work, executed by:
- # 'spark to use PySpark or 'multicore' using pysparkling
- # 'spark,n,m' means use N executors and M partitions
- averagingFunctions=AveragingFunctions, # dict of possible averaging functions
- legalModes=ExecutionModes, # list of possible execution modes
- cachePath=CachePath # directory to cache retrieved files in
- ):
- '''Compute a climatology every N days by applying a mask and averaging function.
-Writes the climatology average, attributes of the primary variable, and the coordinate arrays to a series of netCDF files.
-Optionally generates a contour plot of each N-day climatology for verification.
- '''
- try:
- averageFn = averagingFunctions[averager]
- except:
- print('climatology: Error, Averaging function must be one of: %s' % str(averagingFunctions), file=sys.stderr)
- sys.exit(1)
- if 'accumulators' in averagingConfig:
- accumulators = averagingConfig['accumulators']
-
- # Split daily files into N-day periods for parallel tasks (e.g. 5-day clim means 73 periods)
- # Keyed by integer DOY
- urls = sortByKeys(urls, datasetInfo.getKeys)
- urlSplits = splitter(urls, nEpochs)
- urlSplits = groupByKeys(urlSplits)
- print('Number of URL splits = ', len(urlSplits), file=sys.stderr)
- if VERBOSE: print(urlSplits, file=sys.stderr)
- if len(urlSplits) == 0: sys.exit(1)
-
- # Compute per-pixel statistics in parallel
- with Timer("Parallel Stats"):
- if sparkConfig.startswith('dask'):
- outputFiles = parallelStatsDaskSimple(urlSplits, datasetInfo, nEpochs, variable, mask, coordinates, reader,
- outHdfsPath,
- averagingConfig, sparkConfig, accumulators)
- else:
- outputFiles = parallelStatsSparkSimple(urlSplits, datasetInfo, nEpochs, variable, mask, coordinates, reader,
- outHdfsPath,
- averagingConfig, sparkConfig, accumulators)
- return outputFiles
-
-
-def parallelStatsSparkSimple(urlSplits, ds, nEpochs, variable, mask, coordinates, reader, outHdfsPath, averagingConfig,
- sparkConfig,
- accumulators=['count', 'mean', 'M2', 'min', 'max']):
- '''Compute N-day climatology statistics in parallel using PySpark or pysparkling.'''
- with Timer("Configure Spark"):
- sparkContext, numExecutors, numPartitions = configureSpark(sparkConfig, appName='parallelClimStats')
-
- urlsRDD = sparkContext.parallelize(urlSplits, numPartitions) # partition N-day periods into tasks
-
- with CollectTimer("Map-reduce over %d partitions" % numPartitions): # map to update accumulators in parallel
- accum = urlsRDD.map(lambda urls: accumulate(urls, variable, mask, coordinates, reader, accumulators))
-
- with CollectTimer("Compute merged statistics"):
- stats = accum.map(statsFromAccumulators) # compute final statistics from accumulators
-
- with Timer("Write out climatology and optionally plot"): # write stats to netCDF file
- outputFiles = stats.map(
- lambda s: writeAndPlot(s, ds, variable, coordinates, nEpochs, averagingConfig, outHdfsPath, plot=False)) \
- .collect()
- return outputFiles
-
-
-def parallelStatsSpark(urlSplits, ds, nEpochs, variable, mask, coordinates, reader, outHdfsPath, averagingConfig,
- sparkConfig,
- accumulators=['count', 'mean', 'M2', 'min', 'max']):
- '''Compute N-day climatology statistics in parallel using PySpark or pysparkling.'''
- with Timer("Configure Spark"):
- sparkContext, numExecutors, numPartitions = configureSpark(sparkConfig, appName='parallelClimStats')
-
- urlsRDD = sparkContext.parallelize(urlSplits, numPartitions) # partition N-day periods into tasks
-
- with CollectTimer("Map-reduce over %d partitions" % numPartitions):
- merged = urlsRDD.map(lambda urls: accumulate(urls, variable, mask, coordinates, reader, accumulators)) \
- .reduceByKey(combine) # map-reduce to update accumulators in parallel
-
- with CollectTimer("Compute merged statistics"):
- stats = merged.map(statsFromAccumulators) # compute final statistics from merged accumulators
-
- with Timer("Write out climatology and optionally plot"):
- outputFiles = stats.map(
- lambda s: writeAndPlot(s, ds, variable, coordinates, nEpochs, averagingConfig, outHdfsPath, plot=False)) \
- .collect()
- return outputFiles
-
-
-def parallelStatsPipeline(urls, ds, nEpochs, variable, mask, coordinates, reader, averagingConfig, outHdfsPath,
- accumulators=['count', 'mean', 'M2', 'min', 'max']):
- '''Hide entire pipeline in a single function.'''
- outputFile = writeAndPlot(
- statsFromAccumulators(accumulate(urls, variable, mask, coordinates, reader, accumulators)),
- ds, variable, coordinates, nEpochs, averagingConfig, outHdfsPath, plot=False)
- return outputFile
-
-
-def parallelStatsDaskSimple(urlSplits, ds, nEpochs, variable, mask, coordinates, reader, outHdfsPath, averagingConfig,
- sparkConfig,
- accumulators=['count', 'mean', 'M2', 'min', 'max']):
- '''Compute N-day climatology statistics in parallel using PySpark or pysparkling.'''
- if not sparkConfig.startswith('dask,'):
- print("dask: configuration must be of form 'dask,n'", file=sys.stderr)
- sys.exit(1)
- numPartitions = int(sparkConfig.split(',')[1])
-
- with Timer("Configure Dask distributed"):
- from distributed import Client, as_completed
- client = Client(DaskClientEndpoint)
-
- print('Starting parallel Stats using Dask . . .', file=sys.stderr)
- start = time.time()
- futures = client.map(
- lambda urls: parallelStatsPipeline(urls, ds, nEpochs, variable, mask, coordinates, reader, averagingConfig,
- outHdfsPath, accumulators), urlSplits)
-
- outputFiles = []
- for future in as_completed(futures):
- outputFile = future.result()
- outputFiles.append(outputFile)
- end = time.time()
- print("parallelStats: Completed %s in %0.3f seconds." % (outputFile, (end - start)), file=sys.stderr)
- return outputFiles
-
-
-def writeAndPlot(stats, ds, variable, coordinates, nEpochs, averagingConfig, outHdfsPath, plot=False):
- '''Write statistics to a netCDF4 file and optionally make a contour plot.'''
- key, stats = stats
- doy = int(key)
- urls = stats['_meta']['urls']
- # outFile = 'A%03d.L3m_%s_%dday_clim_%s.nc' % (doy, variable, nEpochs, averagingConfig['name']) # mark each file with first day in period
- outFile = ds.genOutputName(doy, variable, nEpochs, averagingConfig)
- firstInputFile = retrieveFile(urls[0])
-
- outFile = writeStats(stats, firstInputFile, variable, coordinates, outFile) # write to netCDF4 file
- outHdfsFile = hdfsCopyFromLocal(outFile, outHdfsPath)
-
- if plot:
- coords = readCoordinates(firstInputFile, coordinates)
- plotFile = contourMap(stats['mean'], variable, coords, nEpochs, outFile + '.png')
- plotHdfsFile = hdfsCopyFromLocal(plotFile, outHdfsPath)
- return (outHdfsFile, plotHdfsFile)
- else:
- return outHdfsFile
-
-
-def configureSpark(sparkConfig, appName, memoryPerExecutor='4G', coresPerExecutor=1):
- mode, numExecutors, numPartitions = sparkConfig.split(',')
- numExecutors = int(numExecutors)
- print('numExecutors = ', numExecutors, file=sys.stderr)
- numPartitions = int(numPartitions)
- print('numPartitions = ', numPartitions, file=sys.stderr)
- if mode == 'multicore':
- print('Using pysparkling', file=sys.stderr)
- import pysparkling
- sc = pysparkling.Context()
- else:
- print('Using PySpark', file=sys.stderr)
- sparkMaster = mode
- spConf = SparkConf()
- spConf.setAppName(appName)
- spConf.set("spark.executorEnv.HOME",
- os.path.join(os.getenv('HOME'), 'spark_exec_home'))
- spConf.set("spark.executorEnv.PYTHONPATH", os.getcwd())
- spConf.set("spark.executor.memory", memoryPerExecutor)
- print('memoryPerExecutor = ', memoryPerExecutor, file=sys.stderr)
- try:
- sparkMaster = SparkMasterOverride
- except:
- pass
- if sparkMaster[:5] == "mesos":
- spConf.set("spark.cores.max", numExecutors)
- else:
- # Spark master is YARN or local[N]
- spConf.set("spark.executor.instances", numExecutors)
- spConf.set("spark.executor.cores", coresPerExecutor)
- spConf.setMaster(sparkMaster)
- sc = SparkContext(conf=spConf)
- return sc, numExecutors, numPartitions
-
-
-def readAndMask(url, variable, mask=None, cachePath=CachePath, hdfsPath=None):
- '''Read a variable from a netCDF or HDF file and return a numpy masked array.
-If the URL is remote or HDFS, first retrieve the file into a cache directory.
- '''
- v = None
- if mask:
- variables = [variable, mask]
- else:
- variables = [variable]
- try:
- path = retrieveFile(url, cachePath, hdfsPath)
- except:
- print('readAndMask: Error, continuing without file %s' % url, file=sys.stderr)
- return v
-
- try:
- print('Reading variable %s from %s' % (variable, path), file=sys.stderr)
- var, fh = getVariables(path, variables, arrayOnly=True,
- set_auto_mask=True) # return dict of variable objects by name
- v = var[
- variable] # could be masked array
- if v.shape[0] == 1: v = v[0] # throw away trivial time dimension for CF-style files
- if VERBOSE: print('Variable range: %fs to %f' % (v.min(), v.max()), file=sys.stderr)
- close(fh)
- except:
- print('readAndMask: Error, cannot read variable %s from file %s' % (variable, path), file=sys.stderr)
-
- return v
-
-
-def readCoordinates(path, coordinates=['lat', 'lon']):
- '''Read coordinate arrays from local netCDF file.'''
- var, fh = getVariables(path, coordinates, arrayOnly=True, set_auto_mask=True)
- close(fh)
- return [var[k] for k in coordinates]
-
-
-def accumulate(urls, variable, maskVar, coordinates, reader=readAndMask,
- accumulators=['count', 'mean', 'M2', 'min', 'max'], cachePath=CachePath, hdfsPath=None):
- '''Accumulate data into statistics accumulators like count, sum, sumsq, min, max, M3, M4, etc.'''
- keys, urls = urls
- print('Updating accumulators %s for key %s' % (str(accumulators), str(keys)), file=sys.stderr)
- accum = {}
- accum['_meta'] = {'urls': urls, 'coordinates': coordinates}
- for i, url in enumerate(urls):
- v = reader(url, variable, maskVar, cachePath, hdfsPath)
- if v is None: continue
-
- if i == 0:
- for k in accumulators:
- if k[0] == '_': continue
- if k == 'min':
- accum[k] = default_fillvals['f8'] * N.ones(v.shape, dtype=N.float64)
- elif k == 'max':
- accum[k] = -default_fillvals['f8'] * N.ones(v.shape, dtype=N.float64)
- elif k == 'count':
- accum[k] = N.zeros(v.shape, dtype=N.int64)
- else:
- accum[k] = N.zeros(v.shape, dtype=N.float64)
-
- if N.ma.isMaskedArray(v):
- if 'count' in accumulators:
- accum['count'] += ~v.mask
- if 'min' in accumulators:
- accum['min'] = N.ma.minimum(accum['min'], v)
- if 'max' in accumulators:
- accum['max'] = N.ma.maximum(accum['max'], v)
-
- v = N.ma.filled(v, 0.)
- else:
- if 'count' in accumulators:
- accum['count'] += 1
- if 'min' in accumulators:
- accum['min'] = N.minimum(accum['min'], v)
- if 'max' in accumulators:
- accum['max'] = N.maximum(accum['max'], v)
-
- if 'mean' in accumulators:
- n = accum['count']
- # mask = N.not_equal(n, 0)
- # subtract running mean from new values, eliminate roundoff errors
- # delta = N.choose(mask, (0, v - accum['mean']))
- delta = v - accum['mean']
- # delta_n = N.choose(mask, (0, delta/n))
- delta_n = delta / n
- delta_n = N.nan_to_num(delta_n) # set to zero if n=0 caused a NaN
- accum['mean'] += delta_n
- if 'M2' in accumulators:
- term = delta * delta_n * (n - 1)
- accum['M2'] += term
- return (keys, accum)
-
-
-def combine(a, b):
- '''Combine accumulators by summing.'''
- print('Combining accumulators . . .', file=sys.stderr)
- if 'urls' in a['_meta'] and 'urls' in b['_meta']:
- a['_meta']['urls'].extend(b['_meta']['urls'])
- if 'mean' in a:
- ntotal = a['count'] + b['count']
- # mask = N.not_equal(ntotal, 0)
- # a['mean'] = N.choose(mask, (0, (a['mean'] * a['count'] + b['mean'] * b['count']) / ntotal))
- a['mean'] = (a['mean'] * a['count'] + b['mean'] * b['count']) / ntotal
- if 'min' in a:
- if N.ma.isMaskedArray(a):
- a['min'] = N.ma.minimum(a['min'], b['min'])
- else:
- a['min'] = N.minimum(a['min'], b['min'])
- if 'max' in a:
- if N.ma.isMaskedArray(a):
- a['max'] = N.ma.maximum(a['max'], b['max'])
- else:
- a['max'] = N.maximum(a['max'], b['max'])
- for k in list(a.keys()):
- if k[0] == '_': continue
- if k != 'mean' and k != 'min' and k != 'max':
- a[k] += b[k] # just sum count and other moments
- return a
-
-
-def statsFromAccumulators(accum):
- '''Compute final statistics from accumulators.'''
- print('Computing statistics from accumulators . . .', file=sys.stderr)
- keys, accum = accum
-
- # Mask all of the accumulator arrays for zero counts
- if 'count' in accum:
- accum['count'] = N.ma.masked_less_equal(accum['count'], 0, copy=False)
- mask = accum['count'].mask
- for k in accum:
- if k[0] == '_': continue
- if k != 'count':
- accum[k] = N.ma.array(accum[k], copy=False, mask=mask)
-
- # Compute stats
- stats = {}
- if '_meta' in accum:
- stats['_meta'] = accum['_meta']
- if 'count' in accum:
- stats['count'] = accum['count']
- if 'min' in accum:
- stats['min'] = accum['min']
- if 'max' in accum:
- stats['max'] = accum['max']
- if 'mean' in accum:
- stats['mean'] = accum['mean']
- if 'M2' in accum:
- stats['stddev'] = N.sqrt(accum['M2'] / (accum['count'] - 1))
-
- # Convert stats arrays to masked arrays, keeping only cells with count > 0
- # mask = stats['count'] <= 0
- # for k in stats:
- # if k[0] == '_': continue
- # stats[k] = N.ma.masked_where(mask, stats[k], copy=False)
- return (keys, stats)
-
-
-def writeStats(stats, inputFile, variable, coordinates, outFile, format='NETCDF4', cachePath='cache'):
- '''Write out stats arrays to netCDF with some attributes.'''
- if os.path.exists(outFile): os.unlink(outFile)
- dout = Dataset(outFile, 'w', format=format)
- print('Writing stats for variable %s to %s ...' % (variable, outFile), file=sys.stderr)
- print('Shape:', stats['mean'].shape, file=sys.stderr)
- dout.setncattr('variable', variable)
- dout.setncattr('urls', str(stats['_meta']['urls']))
-
- din = Dataset(inputFile, 'r')
- # Transfer global attributes from input file
- # for a in din.ncattrs():
- # dout.setncattr(a, din.getncattr(a))
-
- print('Using coordinates & attributes from %s' % inputFile, file=sys.stderr)
- coordinatesSave = coordinates
- try:
- coordinatesFromFile = din.variables[variable].getncattr('coordinates')
- if 'lat' in coorindatesFromFile.lower() and 'lon' in coordinatesFromFile.lower():
- coordinates = coordinatesFromFile.split()
- if coordinates[0].lower() == 'time': coordinates = coordinates[
- 1:] # discard trivial time dimension for CF-style files
- else:
- coordinates = coordinatesSave # use input coordinates
- except:
- if coordinates is None or len(coordinates) == 0:
- coordinates = ('lat', 'lon') # kludge: another possibility
-
- # Add dimensions and variables, copying data
- coordDim = [dout.createDimension(coord, din.variables[coord].shape[0]) for coord in
- coordinates] # here lat, lon, alt, etc.
- for coord in coordinates:
- var = dout.createVariable(coord, din.variables[coord].dtype, (coord,))
- var[:] = din.variables[coord][:]
-
- # Add stats variables
- for k, v in list(stats.items()):
- if k[0] == '_': continue
- var = dout.createVariable(k, stats[k].dtype, coordinates)
- fillVal = default_fillvals[v.dtype.str.strip("<>")] # remove endian part of dtype to do lookup
- # print >>sys.stderr, "Setting _FillValue attribute for %s to %s" % (k, fillVal)
- var.setncattr('_FillValue', fillVal)
- var[:] = v[:]
-
- # Add attributes from variable in input file (does this make sense?)
- if k == 'count': continue
- try:
- vin = din.variables[variable]
- for a in vin.ncattrs():
- if a == 'scale_factor' or a == 'add_offset' or a == '_FillValue': continue
- var.setncattr(a, vin.getncattr(a))
- except KeyError:
- pass
-
- din.close()
- dout.close()
- return outFile
-
-
-# CollectTimer context manager
-class CollectTimer(object):
- '''Automatically collect Spark result and do timing.'''
-
- def __init__(self, name):
- self.name = name
- self._collect = CollectAndTime
-
- def __enter__(self):
- self.start = time.time()
-
- def __exit__(self, ty, val, tb):
- if not self._collect:
- return False
- else:
- # what goes here??
- end = time.time()
- print("timer: " + self.name + ": %0.3f seconds" % (end - self.start), file=sys.stderr)
- sys.stdout.flush()
- return False
-
-
-# Timer context manager
-class Timer(object):
- '''Automatically collect Spark result and do timing.'''
-
- def __init__(self, name):
- self.name = name
-
- def __enter__(self):
- self.start = time.time()
-
- def __exit__(self, ty, val, tb):
- end = time.time()
- print("timer: " + self.name + ": %0.3f seconds" % (end - self.start), file=sys.stderr)
- sys.stdout.flush()
- return False
-
-
-def contourMap(var, variable, coordinates, n, plotFile):
- '''Make contour plot for the var array using coordinates vector for lat/lon coordinates.'''
- # TODO: Downscale variable array (SST) before contouring, matplotlib is TOO slow on large arrays
- # Fixed color scale, write file, turn off auto borders, set title, etc.
- lats = coordinates[0][:]
- lons = coordinates[1][:]
- if lats[1] < lats[0]: # if latitudes decreasing, reverse
- lats = N.flipud(lats)
- var = N.flipud(var[:])
-
- imageMap(lons, lats, var,
- vmin=-2., vmax=45., outFile=plotFile, autoBorders=False,
- title='%s %d-day Mean from %s' % (variable.upper(), n, os.path.splitext(plotFile)[0]))
- print('Writing contour plot to %s' % plotFile, file=sys.stderr)
- return plotFile
-
-
-def histogram(vals, variable, n, outFile):
- figFile = os.path.splitext(outFile)[0] + '_hist.png'
- M.clf()
- # M.hist(vals, 47, (-2., 45.))
- M.hist(vals, 94)
- M.xlim(-5, 45)
- M.xlabel('SST in degrees Celsius')
- M.ylim(0, 300000)
- M.ylabel('Count')
- M.title('Histogram of %s %d-day Mean from %s' % (variable.upper(), n, outFile))
- M.show()
- print('Writing histogram plot to %s' % figFile, file=sys.stderr)
- M.savefig(figFile)
- return figFile
-
-
-def computeClimatology(datasetName, nEpochs, nWindow, averager, outHdfsPath, sparkConfig):
- '''Compute an N-day climatology for the specified dataset and write the files to HDFS.'''
- if averager not in AveragingFunctions:
- print('computeClimatology: Error, averager %s must be in set %s' % (
- averager, str(list(AveragingFunctions.keys()))), file=sys.stderr)
- sys.exit(1)
- try:
- ds = DatasetList[datasetName] # get dataset metadata class
- except:
- print('computeClimatology: Error, %s not in dataset list %s.' % (datasetName, str(DatasetList)), file=sys.stderr)
- sys.exit(1)
- urls = glob.glob(ds.UrlsPath)
- with Timer("climByAveragingPeriods"):
- results = climByAveragingPeriods(urls, ds, nEpochs, nWindow, ds.Variable, ds.Mask, ds.Coordinates,
- getattr(ds, "readAndMask", readAndMask), outHdfsPath, ds.split,
- averager=averager, averagingConfig=AveragingConfigs[averager],
- sparkConfig=sparkConfig)
- return results
-
-
-def main(args):
- dsName = args[0]
- nEpochs = int(args[1])
- nWindow = int(args[2])
- averager = args[3]
- sparkConfig = args[4]
- outHdfsPath = args[5]
-
- results = computeClimatology(dsName, nEpochs, nWindow, averager, outHdfsPath, sparkConfig)
-
- if isinstance(results[0], tuple):
- makeMovie([r[1] for r in results], 'clim.mpg')
- return results
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-
-# Debug Tests:
-# python ClimatologySpark2.py ModisSst 5 5 pixelMean multicore,1,1 cache/clim
-# python ClimatologySpark2.py ModisChlor 5 5 pixelMean multicore,1,1 cache/clim
-# python ClimatologySpark2.py MeasuresSsh 5 5 pixelMean multicore,1,1 cache/clim
-# python ClimatologySpark2.py ModisSst 5 5 pixelMean multicore,2,2 cache/clim
-
-# Production:
-# time python ClimatologySpark2.py ModisSst 5 5 pixelMean mesos,37,37 cache/clim
-# time python ClimatologySpark2.py ModisChlor 5 5 pixelMean mesos,37,37 cache/clim
-# time python ClimatologySpark2.py MeasuresSsh 5 5 pixelMean mesos,37,37 cache/clim
-
-# time python ClimatologySpark2.py ModisSst 5 5 pixelMean dask,37 cache/clim
-# time python ClimatologySpark2.py ModisChlor 5 5 pixelMean dask,37 cache/clim
-# time python ClimatologySpark2.py MeasuresSsh 5 5 pixelMean dask,37 cache/clim
diff --git a/climatology/clim/README.md b/climatology/clim/README.md
deleted file mode 100644
index f99ae80..0000000
--- a/climatology/clim/README.md
+++ /dev/null
@@ -1,32 +0,0 @@
-# climatology-gaussianInterp module
-
-Compute N-day climatology from daily ocean data files.
-
-See bottom of climatology2.py for examples of what to run.
-
-For speedup, the gaussian weighting is also implemented in Fortran
-and cython. The pure python version is abysmally slow, of course,
-and even the Fortran takes a while (evaluating too many exponentials).
-
-To build gaussInter_f.so from gaussInterp_f.f, run the line in
-gaussInterp_f.mk.
-
-
-
-3 Climatology’s to generate. Daily data is already downloaded to server-1. Location is (a) shown below
-
-1. MODIS CHL_A
-a. server-1:/data/share/datasets/MODIS_L3m_DAY_CHL_chlor_a_4km/daily_data
-b. https://oceandata.sci.gsfc.nasa.gov/MODIS-Aqua/Mapped/Daily/4km/chlor_a
-
-2. Measures SSH
-a. server-1:/data/share/datasets/MEASURES_SLA_JPL_1603/daily_data
-b. http://podaac.jpl.nasa.gov/dataset/SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL1609
-c. ftp://podaac-ftp.jpl.nasa.gov/allData/merged_alt/L4/cdr_grid/
-
-3. CCMP Wind
-a. server-1:/data/share/datasets/CCMP_V2.0_L3.0/daily_data/
-b. https://podaac.jpl.nasa.gov/dataset/CCMP_MEASURES_ATLAS_L4_OW_L3_0_WIND_VECTORS_FLK
-
-
-
diff --git a/climatology/clim/__init__.py b/climatology/clim/__init__.py
deleted file mode 100644
index 6acb5d1..0000000
--- a/climatology/clim/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/climatology/clim/binsum.f b/climatology/clim/binsum.f
deleted file mode 100644
index 96b65af..0000000
--- a/climatology/clim/binsum.f
+++ /dev/null
@@ -1,64 +0,0 @@
-c $Revision: 2.4 $
-c subprogram binsum
-
-c-- calculate interpolation weights and vsum
-
- subroutine binsum( nf, x, y, z, f,
- & xx, yy, zz, vsum,
- & ixmin, ixmax, iymin,
- & iymax, izmin, izmax,
- & imax, jmax, kmax, ndatamax )
-
- real*4 x(ndatamax),y(ndatamax),z(ndatamax),f(ndatamax)
- real*4 xx(imax),yy(jmax),zz(kmax)
- real*4 vsum(2,imax,jmax,kmax)
- integer*4 ixmin(ndatamax),ixmax(ndatamax)
- integer*4 iymin(ndatamax),iymax(ndatamax)
- integer*4 izmin(ndatamax),izmax(ndatamax)
-
- common /parms/ imin,jmin,kmin,
- & xwin2,ywin2,zwin2,
- & xh,yh,zh,
- & dx,dy,dz,
- & xlo,xhi,ylo,yhi,zlo,
- & dxd,dyd
-
- data xfac/-0.6931/
-
- do n=1,nf
- ixmin(n)=max(nint((x(n)-xwin2-xlo+0.5*dx)/dx),1)
- ixmax(n)=min(nint((x(n)+xwin2-xlo+0.5*dx)/dx),imax)
- iymin(n)=max(nint((y(n)-ywin2-ylo+0.5*dy)/dy),1)
- iymax(n)=min(nint((y(n)+ywin2-ylo+0.5*dy)/dy),jmax)
- izmin(n)=max(nint((z(n)-zwin2-zlo+0.5*dz)/dz),1)
- izmax(n)=min(nint((z(n)+zwin2-zlo+0.5*dz)/dz),kmax)
-c print *, x(n),y(n),z(n), f(n)
-c print *,' ixmin, ixmax', ixmin(n), ixmax(n)
-c print *,' iymin, iymax', iymin(n), iymax(n)
-c print *,' izmin, izmax', izmin(n), izmax(n)
- enddo
-
-
-
- do n=1,nf
- do kk=izmin(n),izmax(n)
- do jj=iymin(n),iymax(n)
- do ii=ixmin(n),ixmax(n)
-
-c- - this is the algorithm coded for weights
-
- fac=exp( xfac*(((x(n)-xx(ii))/xh)**2
- & + ((y(n)-yy(jj))/yh)**2
- & + ((z(n)-zz(kk))/zh)**2) )
-
-c print *, 'x, xx, y, yy, z, zz, fac f',
-c & x(n), xx(ii), y(n), yy(jj), z(n), zz(kk), fac, f(n)
-
- vsum(1,ii,jj,kk)=vsum(1,ii,jj,kk)+f(n)*fac
- vsum(2,ii,jj,kk)=vsum(2,ii,jj,kk)+fac
- enddo
- enddo
- enddo
- enddo
- return
- end
diff --git a/climatology/clim/cache.py b/climatology/clim/cache.py
deleted file mode 100644
index fcaef62..0000000
--- a/climatology/clim/cache.py
+++ /dev/null
@@ -1,87 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-cache.py
-
-Utilities to retrieve files and cache them at deterministic paths.
-
-"""
-
-import sys, os, urllib.parse, urllib.request, urllib.parse, urllib.error
-
-# Directory to cache retrieved files in
-CachePath = '/tmp/cache'
-
-
-def isLocalFile(url):
- '''Check if URL is a local path.'''
- u = urllib.parse.urlparse(url)
- if u.scheme == '' or u.scheme == 'file':
- if not os.path.exists(u.path):
- print('isLocalFile: File at local path does not exist: %s' % u.path, file=sys.stderr)
- return (True, u.path)
- else:
- return (False, u.path)
-
-
-def retrieveFile(url, cachePath=CachePath, hdfsPath=None, retries=3):
- '''Retrieve a file from a URL or from an HDFS path.'''
- if hdfsPath:
- hdfsFile = os.path.join(hdfsPath, url)
- return hdfsCopyToLocal(hdfsFile, cachePath)
- else:
- return retrieveFileWeb(url, cachePath, retries)
-
-
-def retrieveFileWeb(url, cachePath=CachePath, retries=3):
- '''Retrieve a file from a URL, or if it is a local path then verify it exists.'''
- if cachePath is None: cachePath = './'
- ok, path = isLocalFile(url)
- if ok: return path
-
- fn = os.path.split(path)[1]
- outPath = os.path.join(cachePath, fn)
- if os.path.exists(outPath):
- print('retrieveFile: Using cached file: %s' % outPath, file=sys.stderr)
- return outPath
- else:
- print('retrieveFile: Retrieving (URL) %s to %s' % (url, outPath), file=sys.stderr)
- for i in range(retries):
- try:
- urllib.request.urlretrieve(url, outPath)
- return outPath
- except:
- print('retrieveFile: Error retrieving file at URL: %s' % url, file=sys.stderr)
- print('retrieveFile: Retrying ...', file=sys.stderr)
- print('retrieveFile: Fatal error, Cannot retrieve file at URL: %s' % url, file=sys.stderr)
- return None
-
-
-def hdfsCopyFromLocal(src, dest):
- '''Copy local file into HDFS directory, overwriting using force switch.'''
- outPath = os.path.join(dest, os.path.split(src)[1])
- cmd = "hadoop fs -copyFromLocal -f %s %s" % (src, dest)
- print("Exec overwrite: %s" % cmd, file=sys.stderr)
- os.system(cmd)
- return outPath
-
-def hdfsCopyToLocal(src, dest):
- '''Copy HDFS file to local path, overwriting.'''
- outPath = os.path.join(dest, os.path.split(src)[1])
- os.unlink(outPath)
- cmd = "hadoop fs -copyToLocal %s %s" % (src, dest)
- print("Exec overwrite: %s" % cmd, file=sys.stderr)
- os.system(cmd)
- return outPath
diff --git a/climatology/clim/climatology.py b/climatology/clim/climatology.py
deleted file mode 100755
index 4551b7a..0000000
--- a/climatology/clim/climatology.py
+++ /dev/null
@@ -1,247 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-climatology.py
-
-Compute a multi-epoch (multi-day) climatology from daily SST Level-3 grids.
-
-Simple code to be run on Spark cluster, or using multi-core parallelism on single machine.
-
-"""
-
-import sys, os, calendar, urllib.parse, urllib.request, urllib.parse, urllib.error
-from datetime import datetime
-import numpy as N
-from .variables import getVariables, close
-#from timePartitions import partitionFilesByKey
-from .split import fixedSplit
-#from stats import Stats
-from pathos.multiprocessing import ProcessingPool as Pool
-from .plotlib import imageMap, makeMovie
-
-#from gaussInterp import gaussInterp
-
-VERBOSE = 1
-
-# Possible execution modes
-# Multicore & cluster modes use pathos pool.map(); Spark mode uses PySpark cluster.
-ExecutionModes = ['sequential', 'multicore', 'cluster', 'spark']
-
-# SST L3m 4.6km Metadata
-# SST calues are scaled integers in degrees Celsius, lat/lon is 4320 x 8640
-# Variable = 'sst', Mask = 'qual_sst', Coordinates = ['lat', 'lon']
-
-# Generate algorithmic name for N-day Climatology product
-SSTClimatologyTemplate = 'SST.L3.Global.Clim.%(period)s.%(date)s.%(version)s.nc' #??
-
-# Simple mask and average functions to get us started, then add gaussian interpolation.
-# MODIS L3 SST product, qual_sst is [-1, 2] - =0 is best data, can add =1 for better coverage
-def qcMask(var, mask): return N.ma.array(var, mask=N.ma.make_mask(mask))
-#def qcMask(var, mask): return N.ma.masked_where(mask != 0, var)
-
-def average(a): return N.ma.mean(a, axis=0)
-#AveragingFunctions = {'pixelAverage': average, 'gaussInterp': gaussInterp}
-AveragingFunctions = {'pixelAverage': average, 'gaussInterp': average}
-
-
-def climByAveragingPeriods(urls, # list of (daily) granule URLs for a long time period (e.g. a year)
- nEpochs, # compute a climatology for every N epochs (days) by 'averaging'
- nWindow, # number of epochs in window needed for averaging
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- maskFn=qcMask, # mask function to compute mask from mask variable
- averager='pixelAverage', # averaging function to use, one of ['pixelAverage', 'gaussInterp']
- mode='sequential', # Map across time periods of N-days for concurrent work, executed by:
- # 'sequential' map, 'multicore' using pool.map(), 'cluster' using pathos pool.map(),
- # or 'spark' using PySpark
- numNodes=1, # number of cluster nodes to use
- nWorkers=4, # number of parallel workers per node
- averagingFunctions=AveragingFunctions, # dict of possible averaging functions
- legalModes=ExecutionModes # list of possiblel execution modes
- ):
- '''Compute a climatology every N days by applying a mask and averaging function.
-Writes the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- try:
- averageFn = averagingFunctions[averager]
- except :
- averageFn = average
- print('climatology: Error, Averaging function must be one of: %s' % str(averagingFunctions), file=sys.stderr)
-
- urlSplits = [s for s in fixedSplit(urls, nEpochs)]
- if VERBOSE: print(urlSplits, file=sys.stderr)
-
- def climsContoured(urls):
- n = len(urls)
- var = climByAveraging(urls, variable, mask, coordinates, maskFn, averageFn)
- return contourMap(var, variable, coordinates, n, urls[0])
-
- if mode == 'sequential':
- plots = list(map(climsContoured, urlSplits))
- elif mode == 'multicore':
- pool = Pool(nWorkers)
- plots = pool.map(climsContoured, urlSplits)
- elif mode == 'cluster':
- pass
- elif mode == 'spark':
- pass
-
- plots = list(map(climsContoured, urlSplits))
- print(plots)
- return plots
-# return makeMovie(plots, 'clim.mpg')
-
-
-def climByAveraging(urls, # list of granule URLs for a time period
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- maskFn=qcMask, # mask function to compute mask from mask variable
- averageFn=average # averaging function to use
- ):
- '''Compute a climatology over N arrays by applying a mask and averaging function.
-Returns the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- n = len(urls)
- varList = [variable, mask]
- for i, url in enumerate(urls):
- fn = retrieveFile(url, '~/cache')
- if VERBOSE: print('Read variables and mask ...', file=sys.stderr)
- var, fh = getVariables(fn, varList) # return dict of variable objects by name
- if i == 0:
- dtype = var[variable].dtype
- shape = (n,) + var[variable].shape
- accum = N.ma.empty(shape, dtype)
-
- v = maskFn(var[variable], var[mask]) # apply quality mask variable to get numpy MA
-# v = var[variable][:]
- accum[i] = v # accumulate N arrays for 'averaging'
- if i+1 != len(urls): # keep var dictionary from last file to grab metadata
- close(fh) # REMEMBER: closing fh loses in-memory data structures
-
- if VERBOSE: print('Averaging ...', file=sys.stderr)
- coord, fh = getVariables(fn, coordinates) # read coordinate arrays and add to dict
- for c in coordinates: var[c] = coord[c][:]
-
- if averageFn == average:
- avg = averageFn(accum) # call averaging function
- else:
- var[variable] = accum
- if averageFn == gaussInterp:
- varNames = variable + coordinates
- avg, vweight, status = \
- gaussInterp(var, varNames, latGrid, lonGrid, wlat, wlon, slat, slon, stime, vfactor, missingValue)
-
- var['attributes'] = var[variable].__dict__ # save attributes of primary variable
- var[variable] = avg # return primary variable & mask arrays in dict
- var[mask] = N.ma.getmask(avg)
-
-# close(fh) # Can't close, lose netCDF4.Variable objects, leaking two fh
- return var
-
-
-def contourMap(var, variable, coordinates, n, url):
- p = urllib.parse.urlparse(url)
- filename = os.path.split(p.path)[1]
- return filename
- outFile = filename + '.png'
- # Downscale variable array (SST) before contouring, matplotlib is TOO slow on large arrays
- vals = var[variable][:]
-
- # Fixed color scale, write file, turn off auto borders, set title, reverse lat direction so monotonically increasing??
- imageMap(var[coordinates[1]][:], var[coordinates[0]][:], var[variable][:],
- vmin=-2., vmax=45., outFile=outFile, autoBorders=False,
- title='%s %d-day Mean from %s' % (variable.upper(), n, filename))
- print('Writing contour plot to %s' % outFile, file=sys.stderr)
- return outFile
-
-
-def isLocalFile(url):
- '''Check if URL is a local path.'''
- u = urllib.parse.urlparse(url)
- if u.scheme == '' or u.scheme == 'file':
- if not path.exists(u.path):
- print('isLocalFile: File at local path does not exist: %s' % u.path, file=sys.stderr)
- return (True, u.path)
- else:
- return (False, u.path)
-
-
-def retrieveFile(url, dir=None):
- '''Retrieve a file from a URL, or if it is a local path then verify it exists.'''
- if dir is None: dir = './'
- ok, path = isLocalFile(url)
- fn = os.path.split(path)[1]
- outPath = os.path.join(dir, fn)
- if not ok:
- if os.path.exists(outPath):
- print('retrieveFile: Using cached file: %s' % outPath, file=sys.stderr)
- else:
- try:
- print('retrieveFile: Retrieving (URL) %s to %s' % (url, outPath), file=sys.stderr)
- urllib.request.urlretrieve(url, outPath)
- except:
- print('retrieveFile: Cannot retrieve file at URL: %s' % url, file=sys.stderr)
- return None
- return outPath
-
-
-def dailyFile2date(path, offset=1):
- '''Convert YYYYDOY string in filename to date.'''
- fn = os.path.split(path)[1]
- year = int(fn[offset:offset+4])
- doy = int(fn[offset+5:offset+8])
- return fn[5:15].replace('.', '/')
-
-
-def formatRegion(r):
- """Format lat/lon region specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- strs = [str(i).replace('-', 'm') for i in r]
- return 'region-%s-%sby%s-%s' % tuple(strs)
-
-
-def formatGrid(r):
- """Format lat/lon grid resolution specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- return str(r[0]) + 'by' + str(r[1])
-
-
-def main(args):
- nEpochs = int(args[0])
- nWindow = int(args[1])
- averager = args[2]
- mode = args[3]
- nWorkers = int(args[4])
- urlFile = args[5]
- urls = [s.strip() for s in open(urlFile, 'r')]
- return climByAveragingPeriods(urls, nEpochs, nWindow, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, mode=mode, nWorkers=nWorkers)
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-
-# python climatology.py 5 5 pixelAverage sequential 1 urls_sst_10days.txt
-# python climatology.py 5 5 gaussianInterp multicore 8 urls_sst_40days.txt
-
diff --git a/climatology/clim/climatology1.py b/climatology/clim/climatology1.py
deleted file mode 100755
index 4551b7a..0000000
--- a/climatology/clim/climatology1.py
+++ /dev/null
@@ -1,247 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-climatology.py
-
-Compute a multi-epoch (multi-day) climatology from daily SST Level-3 grids.
-
-Simple code to be run on Spark cluster, or using multi-core parallelism on single machine.
-
-"""
-
-import sys, os, calendar, urllib.parse, urllib.request, urllib.parse, urllib.error
-from datetime import datetime
-import numpy as N
-from .variables import getVariables, close
-#from timePartitions import partitionFilesByKey
-from .split import fixedSplit
-#from stats import Stats
-from pathos.multiprocessing import ProcessingPool as Pool
-from .plotlib import imageMap, makeMovie
-
-#from gaussInterp import gaussInterp
-
-VERBOSE = 1
-
-# Possible execution modes
-# Multicore & cluster modes use pathos pool.map(); Spark mode uses PySpark cluster.
-ExecutionModes = ['sequential', 'multicore', 'cluster', 'spark']
-
-# SST L3m 4.6km Metadata
-# SST calues are scaled integers in degrees Celsius, lat/lon is 4320 x 8640
-# Variable = 'sst', Mask = 'qual_sst', Coordinates = ['lat', 'lon']
-
-# Generate algorithmic name for N-day Climatology product
-SSTClimatologyTemplate = 'SST.L3.Global.Clim.%(period)s.%(date)s.%(version)s.nc' #??
-
-# Simple mask and average functions to get us started, then add gaussian interpolation.
-# MODIS L3 SST product, qual_sst is [-1, 2] - =0 is best data, can add =1 for better coverage
-def qcMask(var, mask): return N.ma.array(var, mask=N.ma.make_mask(mask))
-#def qcMask(var, mask): return N.ma.masked_where(mask != 0, var)
-
-def average(a): return N.ma.mean(a, axis=0)
-#AveragingFunctions = {'pixelAverage': average, 'gaussInterp': gaussInterp}
-AveragingFunctions = {'pixelAverage': average, 'gaussInterp': average}
-
-
-def climByAveragingPeriods(urls, # list of (daily) granule URLs for a long time period (e.g. a year)
- nEpochs, # compute a climatology for every N epochs (days) by 'averaging'
- nWindow, # number of epochs in window needed for averaging
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- maskFn=qcMask, # mask function to compute mask from mask variable
- averager='pixelAverage', # averaging function to use, one of ['pixelAverage', 'gaussInterp']
- mode='sequential', # Map across time periods of N-days for concurrent work, executed by:
- # 'sequential' map, 'multicore' using pool.map(), 'cluster' using pathos pool.map(),
- # or 'spark' using PySpark
- numNodes=1, # number of cluster nodes to use
- nWorkers=4, # number of parallel workers per node
- averagingFunctions=AveragingFunctions, # dict of possible averaging functions
- legalModes=ExecutionModes # list of possiblel execution modes
- ):
- '''Compute a climatology every N days by applying a mask and averaging function.
-Writes the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- try:
- averageFn = averagingFunctions[averager]
- except :
- averageFn = average
- print('climatology: Error, Averaging function must be one of: %s' % str(averagingFunctions), file=sys.stderr)
-
- urlSplits = [s for s in fixedSplit(urls, nEpochs)]
- if VERBOSE: print(urlSplits, file=sys.stderr)
-
- def climsContoured(urls):
- n = len(urls)
- var = climByAveraging(urls, variable, mask, coordinates, maskFn, averageFn)
- return contourMap(var, variable, coordinates, n, urls[0])
-
- if mode == 'sequential':
- plots = list(map(climsContoured, urlSplits))
- elif mode == 'multicore':
- pool = Pool(nWorkers)
- plots = pool.map(climsContoured, urlSplits)
- elif mode == 'cluster':
- pass
- elif mode == 'spark':
- pass
-
- plots = list(map(climsContoured, urlSplits))
- print(plots)
- return plots
-# return makeMovie(plots, 'clim.mpg')
-
-
-def climByAveraging(urls, # list of granule URLs for a time period
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- maskFn=qcMask, # mask function to compute mask from mask variable
- averageFn=average # averaging function to use
- ):
- '''Compute a climatology over N arrays by applying a mask and averaging function.
-Returns the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- n = len(urls)
- varList = [variable, mask]
- for i, url in enumerate(urls):
- fn = retrieveFile(url, '~/cache')
- if VERBOSE: print('Read variables and mask ...', file=sys.stderr)
- var, fh = getVariables(fn, varList) # return dict of variable objects by name
- if i == 0:
- dtype = var[variable].dtype
- shape = (n,) + var[variable].shape
- accum = N.ma.empty(shape, dtype)
-
- v = maskFn(var[variable], var[mask]) # apply quality mask variable to get numpy MA
-# v = var[variable][:]
- accum[i] = v # accumulate N arrays for 'averaging'
- if i+1 != len(urls): # keep var dictionary from last file to grab metadata
- close(fh) # REMEMBER: closing fh loses in-memory data structures
-
- if VERBOSE: print('Averaging ...', file=sys.stderr)
- coord, fh = getVariables(fn, coordinates) # read coordinate arrays and add to dict
- for c in coordinates: var[c] = coord[c][:]
-
- if averageFn == average:
- avg = averageFn(accum) # call averaging function
- else:
- var[variable] = accum
- if averageFn == gaussInterp:
- varNames = variable + coordinates
- avg, vweight, status = \
- gaussInterp(var, varNames, latGrid, lonGrid, wlat, wlon, slat, slon, stime, vfactor, missingValue)
-
- var['attributes'] = var[variable].__dict__ # save attributes of primary variable
- var[variable] = avg # return primary variable & mask arrays in dict
- var[mask] = N.ma.getmask(avg)
-
-# close(fh) # Can't close, lose netCDF4.Variable objects, leaking two fh
- return var
-
-
-def contourMap(var, variable, coordinates, n, url):
- p = urllib.parse.urlparse(url)
- filename = os.path.split(p.path)[1]
- return filename
- outFile = filename + '.png'
- # Downscale variable array (SST) before contouring, matplotlib is TOO slow on large arrays
- vals = var[variable][:]
-
- # Fixed color scale, write file, turn off auto borders, set title, reverse lat direction so monotonically increasing??
- imageMap(var[coordinates[1]][:], var[coordinates[0]][:], var[variable][:],
- vmin=-2., vmax=45., outFile=outFile, autoBorders=False,
- title='%s %d-day Mean from %s' % (variable.upper(), n, filename))
- print('Writing contour plot to %s' % outFile, file=sys.stderr)
- return outFile
-
-
-def isLocalFile(url):
- '''Check if URL is a local path.'''
- u = urllib.parse.urlparse(url)
- if u.scheme == '' or u.scheme == 'file':
- if not path.exists(u.path):
- print('isLocalFile: File at local path does not exist: %s' % u.path, file=sys.stderr)
- return (True, u.path)
- else:
- return (False, u.path)
-
-
-def retrieveFile(url, dir=None):
- '''Retrieve a file from a URL, or if it is a local path then verify it exists.'''
- if dir is None: dir = './'
- ok, path = isLocalFile(url)
- fn = os.path.split(path)[1]
- outPath = os.path.join(dir, fn)
- if not ok:
- if os.path.exists(outPath):
- print('retrieveFile: Using cached file: %s' % outPath, file=sys.stderr)
- else:
- try:
- print('retrieveFile: Retrieving (URL) %s to %s' % (url, outPath), file=sys.stderr)
- urllib.request.urlretrieve(url, outPath)
- except:
- print('retrieveFile: Cannot retrieve file at URL: %s' % url, file=sys.stderr)
- return None
- return outPath
-
-
-def dailyFile2date(path, offset=1):
- '''Convert YYYYDOY string in filename to date.'''
- fn = os.path.split(path)[1]
- year = int(fn[offset:offset+4])
- doy = int(fn[offset+5:offset+8])
- return fn[5:15].replace('.', '/')
-
-
-def formatRegion(r):
- """Format lat/lon region specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- strs = [str(i).replace('-', 'm') for i in r]
- return 'region-%s-%sby%s-%s' % tuple(strs)
-
-
-def formatGrid(r):
- """Format lat/lon grid resolution specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- return str(r[0]) + 'by' + str(r[1])
-
-
-def main(args):
- nEpochs = int(args[0])
- nWindow = int(args[1])
- averager = args[2]
- mode = args[3]
- nWorkers = int(args[4])
- urlFile = args[5]
- urls = [s.strip() for s in open(urlFile, 'r')]
- return climByAveragingPeriods(urls, nEpochs, nWindow, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, mode=mode, nWorkers=nWorkers)
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-
-# python climatology.py 5 5 pixelAverage sequential 1 urls_sst_10days.txt
-# python climatology.py 5 5 gaussianInterp multicore 8 urls_sst_40days.txt
-
diff --git a/climatology/clim/climatology2.py b/climatology/clim/climatology2.py
deleted file mode 100755
index 4da7e6f..0000000
--- a/climatology/clim/climatology2.py
+++ /dev/null
@@ -1,467 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-climatology2.py
-
-Compute a multi-epoch (multi-day) climatology from daily SST Level-3 grids.
-
-Simple code to be run on Spark cluster, or using multi-core parallelism on single machine.
-
-"""
-
-import sys, os, urllib.parse, urllib.request, urllib.parse, urllib.error, re, time
-import numpy as N
-import matplotlib
-matplotlib.use('Agg')
-import matplotlib.pylab as M
-
-from .variables import getVariables, close
-from .cache import retrieveFile, CachePath
-from .split import fixedSplit, splitByNDays
-from netCDF4 import Dataset, default_fillvals
-from pathos.multiprocessing import ProcessingPool as Pool
-from .plotlib import imageMap, makeMovie
-
-from .spatialFilter import spatialFilter
-from .gaussInterp import gaussInterp # calls into Fortran version gaussInterp_f.so
-#from gaussInterp_slow import gaussInterp_slow as gaussInterp # pure python, slow debuggable version
-
-#import pyximport; pyximport.install(pyimport=False)
-#from gaussInterp import gaussInterp, gaussInterp_ # attempted cython version, had issues
-
-VERBOSE = 1
-
-# Possible execution modes
-# Multicore & cluster modes use pathos pool.map(); Spark mode uses PySpark cluster.
-ExecutionModes = ['sequential', 'multicore', 'cluster', 'spark']
-
-# SST L3m 4.6km Metadata
-# SST calues are scaled integers in degrees Celsius, lon/lat is 8640 x 4320
-# Variable = 'sst', Mask = 'qual_sst', Coordinates = ['lon', 'lat']
-
-# Generate algorithmic name for N-day Climatology product
-SSTClimatologyTemplate = 'SST.L3.Global.Clim.%(period)s.%(date)s.%(version)s.nc' #??
-
-# Simple mask and average functions to get us started, then add gaussian interpolation.
-# MODIS L3 SST product, qual_sst is [-1, 2] - =0 is best data, can add =1 for better coverage
-#def qcMask(var, mask): return N.ma.array(var, mask=N.ma.make_mask(mask == 0))
-
-def qcMask(var, mask): return N.ma.masked_where(mask != 0, var)
-#def qcMask(var, mask): return N.ma.masked_where(mask < 0, var)
-
-def splitModisSst(seq, n):
- for chunk in splitByNDays(seq, n, re.compile(r'(...).L3m'), keyed=False):
- yield chunk
-
-def mean(a): return N.ma.mean(a, axis=0)
-
-AveragingFunctions = {'pixelMean': mean, 'gaussInterp': gaussInterp, 'spatialFilter': spatialFilter}
-
-PixelMeanConfig = {'name': 'pixelMean'}
-
-GaussInterpConfig = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 3, 'wlon': 3,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig1a = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.30, 'wlon': 0.30,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig1b = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.08, 'wlon': 0.08,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig2 = {'name': 'gaussInterp',
- 'latGrid': (89.5, -89.5, -0.25), 'lonGrid': (-180., 179., 0.25),
- 'wlat': 2., 'wlon': 2.,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-FilterGaussian = [[1, 2, 1], [2, 4, 2], [1, 2, 1]] # divide by 16
-FilterLowPass = [[1, 1, 1], [1, 1, 1], [1, 1, 1]] # divide by 9
-
-SpatialFilterConfig1 = {'name': 'spatialFilter', 'normalization': 16.,
- 'spatialFilter': N.array(FilterGaussian, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-SpatialFilterConfig2 = {'name': 'spatialFilter', 'normalization': 9.,
- 'spatialFilter': N.array(FilterLowPass, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-
-
-def climByAveragingPeriods(urls, # list of (daily) granule URLs for a long time period (e.g. a year)
- nEpochs, # compute a climatology for every N epochs (days) by 'averaging'
- nWindow, # number of epochs in window needed for averaging
- nNeighbors, # number of neighbors on EACH side in lat/lon directions to use in averaging
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- splitFn=splitModisSst, # split function to use to partition the input URL list
- maskFn=qcMask, # mask function to compute mask from mask variable
- averager='pixelMean', # averaging function to use, one of ['pixelMean', 'gaussInterp', 'spatialFilter']
- averagingConfig={}, # dict of parameters to control the averaging function (e.g. gaussInterp)
- optimization='fortran', # optimization mode (fortran or cython)
- mode='sequential', # Map across time periods of N-days for concurrent work, executed by:
- # 'sequential' map, 'multicore' using pool.map(), 'cluster' using pathos pool.map(),
- # or 'spark' using PySpark
- numNodes=1, # number of cluster nodes to use
- nWorkers=4, # number of parallel workers per node
- averagingFunctions=AveragingFunctions, # dict of possible averaging functions
- legalModes=ExecutionModes, # list of possible execution modes
- cachePath=CachePath # directory to cache retrieved files in
- ):
- '''Compute a climatology every N days by applying a mask and averaging function.
-Writes the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- if averagingConfig['name'] == 'gaussInterp':
- averagingConfig['wlat'] = nNeighbors
- averagingConfig['wlon'] = nNeighbors
- try:
- averageFn = averagingFunctions[averager]
- except:
- print('climatology: Error, Averaging function must be one of: %s' % str(averagingFunctions), file=sys.stderr)
- sys.exit(1)
-
- urlSplits = [s for s in splitFn(urls, nEpochs)]
-
- def climsContoured(urls, plot=None, fillValue=default_fillvals['f4'], format='NETCDF4', cachePath=cachePath):
- n = len(urls)
- if VERBOSE: print(urls, file=sys.stderr)
-
- var = climByAveraging(urls, variable, mask, coordinates, maskFn, averageFn, averagingConfig, optimization, cachePath)
-
- fn = os.path.split(urls[0])[1]
- inFile = os.path.join(cachePath, fn)
- method = averagingConfig['name']
- fn = os.path.splitext(fn)[0]
- day = fn[5:8]
- nDays = int(var['time'][0])
-
- if 'wlat' in averagingConfig:
- wlat = averagingConfig['wlat']
- else:
- wlat = 1
- if int(wlat) == wlat:
- outFile = 'A%s.L3m_%dday_clim_sst_4km_%s_%dnbrs.nc' % (day, nDays, method, int(wlat)) # mark each file with first day in period
- else:
- outFile = 'A%s.L3m_%dday_clim_sst_4km_%s_%4.2fnbrs.nc' % (day, nDays, method, wlat) # mark each file with first day in period
-
- outFile = writeOutNetcdfVars(var, variable, mask, coordinates, inFile, outFile, fillValue, format)
-
- if plot == 'contour':
- figFile = contourMap(var, variable, coordinates, n, outFile)
- elif plot == 'histogram':
-# figFile = histogram(var, variable, n, outFile)
- figFile = None
- else:
- figFile = None
- return (outFile, figFile)
-
- if mode == 'sequential':
- results = list(map(climsContoured, urlSplits))
- elif mode == 'multicore':
- pool = Pool(nWorkers)
- results = pool.map(climsContoured, urlSplits)
- elif mode == 'cluster':
- pass
- elif mode == 'spark':
- pass
-
- return results
-
-
-def climByAveraging(urls, # list of granule URLs for a time period
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- maskFn=qcMask, # mask function to compute mask from mask variable
- averageFn=mean, # averaging function to use
- averagingConfig={}, # parameters to control averaging function (e.g. gaussInterp)
- optimization='fortran', # optimization mode (fortran or cython)
- cachePath=CachePath
- ):
- '''Compute a climatology over N arrays by applying a mask and averaging function.
-Returns the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- n = len(urls)
- varList = [variable, mask]
- var = {}
- vtime = N.zeros((n,), N.int32)
-
- for i, url in enumerate(urls):
- try:
- path = retrieveFile(url, cachePath)
- fn = os.path.split(path)[1]
- vtime[i] = int(fn[5:8]) # KLUDGE: extract DOY from filename
- except:
- print('climByAveraging: Error, continuing without file %s' % url, file=sys.stderr)
- accum[i] = emptyVar
- continue
- if path is None: continue
- print('Read variables and mask ...', file=sys.stderr)
- try:
- var, fh = getVariables(path, varList, arrayOnly=True, order='F', set_auto_mask=False) # return dict of variable objects by name
- except:
- print('climByAveraging: Error, cannot read file %s' % path, file=sys.stderr)
- accum[i] = emptyVar
- continue
- if i == 0:
- dtype = var[variable].dtype
- if 'int' in dtype.name: dtype = N.float32
- shape = (n,) + var[variable].shape
- accum = N.ma.empty(shape, dtype, order='F')
- emptyVar = N.array(N.ma.masked_all(var[variable].shape, dtype), order='F') # totally masked variable array for missing or bad file reads
-
- print('Read coordinates ...', file=sys.stderr)
- var, fh = getVariables(path, coordinates, var, arrayOnly=True, order='F') # read coordinate arrays and add to dict
-
- var[variable] = maskFn(var[variable], var[mask]) # apply quality mask variable to get numpy MA, turned off masking done by netCDF4 library
-# var[variable] = var[variable][:]
-
- # Echo variable range for sanity check
- vals = var[variable].compressed()
- print('Variable Range: min, max:', vals.min(), vals.max(), file=sys.stderr)
-
- # Plot input grid
-# figFile = histogram(vals, variable, n, os.path.split(path)[1])
-# figFile = contourMap(var, variable, coordinates[1:], n, os.path.split(path)[1])
-
- accum[i] = var[variable] # accumulate N arrays for 'averaging"""
-# if i != 0 and i+1 != n: close(fh) # REMEMBER: closing fh loses netCDF4 in-memory data structures
- close(fh)
-
- coordinates = ['time'] + coordinates # add constructed time (days) as first coordinate
- var['time'] = vtime
-
- if averagingConfig['name'] == 'pixelMean':
- print('Doing Pixel Average over %d grids ...' % n, file=sys.stderr)
- start = time.time()
- avg = averageFn(accum) # call pixel averaging function
- end = time.time()
- print('pixelMean execution time:', (end - start), file=sys.stderr)
- outlat = var[coordinates[1]].astype(N.float32)[:]
- outlon = var[coordinates[2]].astype(N.float32)[:]
- elif averagingConfig['name'] == 'gaussInterp':
- print('Doing Gaussian Interpolation over %d grids ...' % n, file=sys.stderr)
- var[variable] = accum
- c = averagingConfig
- latGrid = c['latGrid']; lonGrid = c['lonGrid']
- if latGrid is not None and lonGrid is not None:
- outlat = N.arange(latGrid[0], latGrid[1]+latGrid[2], latGrid[2], dtype=N.float32, order='F')
- outlon = N.arange(lonGrid[0], lonGrid[1]+lonGrid[2], lonGrid[2], dtype=N.float32, order='F')
- else:
- outlat = N.array(var[coordinates[1]], dtype=N.float32, order='F')
- outlon = N.array(var[coordinates[2]], dtype=N.float32, order='F')
- varNames = [variable] + coordinates
- start = time.time()
- avg, weight, status = \
- gaussInterp(var, varNames, outlat, outlon, c['wlat'], c['wlon'],
- c['slat'], c['slon'], c['stime'], c['vfactor'], c['missingValue'],
- VERBOSE, optimization)
- end = time.time()
- var['outweight'] = weight.astype(N.float32)
- print('gaussInterp execution time:', (end - start), file=sys.stderr)
- elif averagingConfig['name'] == 'spatialFilter':
- print('Applying Spatial 3x3 Filter and then averaging over %d grids ...' % n, file=sys.stderr)
- var[variable] = accum
- c = averagingConfig
- varNames = [variable] + coordinates
- start = time.time()
- avg, count, status = \
- spatialFilter(var, varNames, c['spatialFilter'], c['normalization'],
- c['missingValue'], VERBOSE, optimization)
- end = time.time()
- print('spatialFilter execution time:', (end - start), file=sys.stderr)
- outlat = var[coordinates[1]].astype(N.float32)[:]
- outlon = var[coordinates[2]].astype(N.float32)[:]
-
- var['out'+variable] = avg.astype(N.float32) # return primary variable & mask arrays in dict
- var['out'+mask] = N.ma.getmask(avg)
- var['outlat'] = outlat
- var['outlon'] = outlon
- return var
-
-
-def writeOutNetcdfVars(var, variable, mask, coordinates, inFile, outFile, fillValue=None, format='NETCDF4'):
- '''Construct output bundle of arrays with NetCDF dimensions and attributes.
-Output variables and attributes will have same names as the input file.
- '''
- din = Dataset(inFile, 'r')
- dout = Dataset(outFile, 'w', format=format)
- print('Writing %s ...' % outFile, file=sys.stderr)
-
- # Transfer global attributes from input file
- for a in din.ncattrs():
- dout.setncattr(a, din.getncattr(a))
-
- # Add dimensions and variables, copying data
- coordDim = [dout.createDimension(coord, var['out'+coord].shape[0]) for coord in coordinates] # here output lon, lat, etc.
- for coord in coordinates:
- v = dout.createVariable(coord, var['out'+coord].dtype, (coord,))
- v[:] = var['out'+coord][:]
- primaryVar = dout.createVariable(variable, var['out'+variable].dtype, coordinates, fill_value=fillValue)
- primaryVar[:] = var['out'+variable][:] # transfer array
- maskVar = dout.createVariable(mask, 'i1', coordinates)
- maskVar[:] = var['out'+mask].astype('i1')[:]
-
- # Transfer variable attributes from input file
- for k,v in dout.variables.items():
- for a in din.variables[k].ncattrs():
- if a == 'scale_factor' or a == 'add_offset' or a == '_FillValue': continue
- v.setncattr(a, din.variables[k].getncattr(a))
- if k == variable:
- try:
-# if fillValue == None: fillValue = din.variables[k].getncattr('_FillValue') # total kludge
- if fillValue == None: fillValue = default_fillvals['f4']
-# print >>sys.stderr, default_fillvals
-# v.setncattr('_FillValue', fillValue) # set proper _FillValue for climatology array
- v.setncattr('missing_value', fillValue)
- print('Setting missing_value for primary variable %s to %f' % (variable, fillValue), file=sys.stderr)
- except:
- print('writeOutNetcdfVars: Warning, for variable %s no fill value specified or derivable from inputs.' % variable, file=sys.stderr)
- din.close()
- dout.close()
- return outFile
-
-
-def contourMap(var, variable, coordinates, n, outFile):
- figFile = os.path.splitext(outFile)[0] + '_hist.png'
- # TODO: Downscale variable array (SST) before contouring, matplotlib is TOO slow on large arrays
- vals = var[variable][:]
-
- # Fixed color scale, write file, turn off auto borders, set title, reverse lat direction so monotonically increasing??
- imageMap(var[coordinates[1]][:], var[coordinates[0]][:], var[variable][:],
- vmin=-2., vmax=45., outFile=figFile, autoBorders=False,
- title='%s %d-day Mean from %s' % (variable.upper(), n, outFile))
- print('Writing contour plot to %s' % figFile, file=sys.stderr)
- return figFile
-
-
-def histogram(vals, variable, n, outFile):
- figFile = os.path.splitext(outFile)[0] + '_hist.png'
- M.clf()
-# M.hist(vals, 47, (-2., 45.))
- M.hist(vals, 94)
- M.xlim(-5, 45)
- M.xlabel('SST in degrees Celsius')
- M.ylim(0, 300000)
- M.ylabel('Count')
- M.title('Histogram of %s %d-day Mean from %s' % (variable.upper(), n, outFile))
- M.show()
- print('Writing histogram plot to %s' % figFile, file=sys.stderr)
- M.savefig(figFile)
- return figFile
-
-
-def dailyFile2date(path, offset=1):
- '''Convert YYYYDOY string in filename to date.'''
- fn = os.path.split(path)[1]
- year = int(fn[offset:offset+4])
- doy = int(fn[offset+5:offset+8])
- return fn[5:15].replace('.', '/')
-
-
-def formatRegion(r):
- """Format lat/lon region specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- strs = [str(i).replace('-', 'm') for i in r]
- return 'region-%s-%sby%s-%s' % tuple(strs)
-
-
-def formatGrid(r):
- """Format lat/lon grid resolution specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- return str(r[0]) + 'by' + str(r[1])
-
-
-def main(args):
- nEpochs = int(args[0])
- nWindow = int(args[1])
- nNeighbors = int(args[2])
- averager = args[3]
- optimization = args[4]
- mode = args[5]
- nWorkers = int(args[6])
- urlFile = args[7]
- urls = [s.strip() for s in open(urlFile, 'r')]
- if averager == 'gaussInterp':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=GaussInterpConfig,
- mode=mode, nWorkers=nWorkers)
- elif averager == 'spatialFilter':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=SpatialFilterConfig1,
- mode=mode, nWorkers=nWorkers)
- elif averager == 'pixelMean':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=PixelMeanConfig,
- mode=mode, nWorkers=nWorkers)
- else:
- print('climatology2: Error, averager must be one of', list(AveragingFunctions.keys()), file=sys.stderr)
- sys.exit(1)
-
- if results[0][1] is not None:
- makeMovie([r[1] for r in results], 'clim.mpg')
- return results
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-
-# Old Tests:
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_10days.txt
-# python climatology2.py 5 5 3 gaussInterp fortran sequential 1 urls_sst_10days.txt
-
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_40days.txt
-# python climatology2.py 5 5 0 pixelMean fortran multicore 8 urls_sst_40days.txt
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 8 urls_sst_40days.txt
-
-# Old Production:
-# python climatology2.py 5 5 0 pixelMean fortran multicore 16 urls_sst_2015.txt >& log &
-# python climatology2.py 10 10 0 pixelMean fortran multicore 16 urls_sst_2015.txt >& log &
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 16 urls_sst_2015.txt >& log &
-
-
-# Tests:
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 0 pixelMean fortran multicore 4 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 3 gaussInterp fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 4 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 7 1 spatialFilter fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-
-# Test number of neighbors needed:
-# python climatology2.py 5 7 3 gaussInterp fortran multicore 4 urls_sst_daynight_20days_sorted.txt
-
-
-# Production:
-# python climatology2.py 5 7 0 pixelMean fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 3 gaussInterp fortran sequential 1 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 3 gaussInterp fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 1 spatialFilter fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-
diff --git a/climatology/clim/climatology3Spark.py b/climatology/clim/climatology3Spark.py
deleted file mode 100755
index 859b528..0000000
--- a/climatology/clim/climatology3Spark.py
+++ /dev/null
@@ -1,432 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-climatology3Spark.py
-
-Compute a multi-epoch (multi-day) climatology from daily SST Level-3 grids.
-
-Simple code to be run on Spark cluster, or using multi-core parallelism on single machine.
-
-"""
-
-import sys, os, urllib.parse, urllib.request, urllib.parse, urllib.error, re, time
-import numpy as N
-import matplotlib
-matplotlib.use('Agg')
-import matplotlib.pylab as M
-
-from .variables import getVariables, close
-from .cache import retrieveFile, CachePath
-from .split import fixedSplit, splitByNDays
-from netCDF4 import Dataset, default_fillvals
-from .plotlib import imageMap, makeMovie
-
-from .spatialFilter import spatialFilter
-from .gaussInterp import gaussInterp # calls into Fortran version gaussInterp_f.so
-#from gaussInterp_slow import gaussInterp_slow as gaussInterp # pure python, slow debuggable version
-
-VERBOSE = 1
-
-# Possible execution modes
-ExecutionModes = ['multicore', 'spark']
-
-# SST L3m 4.6km Metadata
-# SST calues are scaled integers in degrees Celsius, lon/lat is 8640 x 4320
-# Variable = 'sst', Mask = 'qual_sst', Coordinates = ['lon', 'lat']
-
-# Generate algorithmic name for N-day Climatology product
-SSTClimatologyTemplate = 'SST.L3.Global.Clim.%(period)s.%(date)s.%(version)s.nc' #??
-
-# Simple mask and average functions to get us started, then add gaussian interpolation.
-# MODIS L3 SST product, qual_sst is [-1, 2] - =0 is best data, can add =1 for better coverage
-#def qcMask(var, mask): return N.ma.array(var, mask=N.ma.make_mask(mask == 0))
-
-def qcMask(var, mask): return N.ma.masked_where(mask != 0, var)
-#def qcMask(var, mask): return N.ma.masked_where(mask < 0, var)
-
-def splitModisSst(seq, n):
- for chunk in splitByNDays(seq, n, re.compile(r'(...).L3m')):
- yield chunk
-
-AveragingFunctions = {'pixelMean': mean, 'gaussInterp': gaussInterp, 'spatialFilter': spatialFilter}
-
-PixelMeanConfig = {'name': 'pixelMean'}
-
-GaussInterpConfig = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 3, 'wlon': 3,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig1a = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.30, 'wlon': 0.30,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig1b = {'name': 'gaussInterp',
- 'latGrid': None, 'lonGrid': None, # None means use input lat/lon grid
- 'wlat': 0.08, 'wlon': 0.08,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-GaussInterpConfig2 = {'name': 'gaussInterp',
- 'latGrid': (89.5, -89.5, -0.25), 'lonGrid': (-180., 179., 0.25),
- 'wlat': 2., 'wlon': 2.,
- 'slat': 0.15, 'slon': 0.15, 'stime': 1,
- 'vfactor': -0.6931, 'missingValue': default_fillvals['f4']}
-
-FilterGaussian = [[1, 2, 1], [2, 4, 2], [1, 2, 1]] # divide by 16
-FilterLowPass = [[1, 1, 1], [1, 1, 1], [1, 1, 1]] # divide by 9
-
-SpatialFilterConfig1 = {'name': 'spatialFilter', 'normalization': 16.,
- 'spatialFilter': N.array(FilterGaussian, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-SpatialFilterConfig2 = {'name': 'spatialFilter', 'normalization': 9.,
- 'spatialFilter': N.array(FilterLowPass, dtype=N.int32),
- 'missingValue': default_fillvals['f4']}
-
-# Directory to cache retrieved files in
-CachePath = '~/cache'
-
-
-
-def climByAveragingPeriods(urls, # list of (daily) granule URLs for a long time period (e.g. a year)
- nEpochs, # compute a climatology for every N epochs (days) by 'averaging'
- nWindow, # number of epochs in window needed for averaging
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- splitFn=splitModisSst, # split function to use to partition the input URL list
- maskFn=qcMask, # mask function to compute mask from mask variable
- averager='pixelMean', # averaging function to use, one of ['pixelMean', 'gaussInterp', 'spatialFilter']
- averagingConfig={}, # dict of parameters to control the averaging function (e.g. gaussInterp)
- optimization='fortran', # optimization mode (fortran or cython)
- mode='multicore', # Map across time periods of N-days for concurrent work, executed by:
- # 'multicore' using pysparkling or 'spark' using Spark/Mesos cluster
- numNodes=1, # number of cluster nodes to use
- nWorkers=4, # number of parallel workers per node
- averagingFunctions=AveragingFunctions, # dict of possible averaging functions
- legalModes=ExecutionModes, # list of possible execution modes
- cachePath=CachePath # directory to cache retrieved files in
- ):
- '''Compute a climatology every N days by applying a mask and averaging function.
-Writes the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- if averagingConfig['name'] == 'gaussInterp':
- averagingConfig['wlat'] = nNeighbors
- averagingConfig['wlon'] = nNeighbors
- try:
- averageFn = averagingFunctions[averager]
- except:
- print('climatology: Error, Averaging function must be one of: %s' % str(averagingFunctions), file=sys.stderr)
- sys.exit(1)
-
- urlSplits = [s for s in splitFn(urls, nEpochs)]
-
- if mode == 'multicore':
- pass
- elif mode == 'spark':
- pass
-
- def climsContoured(urls, plot=None, fillValue=default_fillvals['f4'], format='NETCDF4', cachePath=cachePath):
- n = len(urls)
- if VERBOSE: print(urls, file=sys.stderr)
-
- var = climByAveraging(urls, variable, mask, coordinates, maskFn, averageFn, averagingConfig, optimization, cachePath)
-
- fn = os.path.split(urls[0])[1]
- inFile = os.path.join(cachePath, fn)
- method = averagingConfig['name']
- fn = os.path.splitext(fn)[0]
- day = fn[5:8]
- nDays = int(var['time'][0])
-
- if 'wlat' in averagingConfig:
- wlat = averagingConfig['wlat']
- else:
- wlat = 1
- if int(wlat) == wlat:
- outFile = 'A%s.L3m_%dday_clim_sst_4km_%s_%dnbrs.nc' % (day, nDays, method, int(wlat)) # mark each file with first day in period
- else:
- outFile = 'A%s.L3m_%dday_clim_sst_4km_%s_%4.2fnbrs.nc' % (day, nDays, method, wlat) # mark each file with first day in period
-
- outFile = writeOutNetcdfVars(var, variable, mask, coordinates, inFile, outFile, fillValue, format)
-
- if plot == 'contour':
- figFile = contourMap(var, variable, coordinates, n, outFile)
- elif plot == 'histogram':
-# figFile = histogram(var, variable, n, outFile)
- figFile = None
- else:
- figFile = None
- return (outFile, figFile)
-
-
- return results
-
-
-def accumulateClim(urls, # list of granule URLs for a time period
- variable, # name of primary variable in file
- mask, # name of mask variable
- coordinates, # names of coordinate arrays to read and pass on (e.g. 'lat' and 'lon')
- maskFn=qcMask, # mask function to compute mask from mask variable
- averageFn=mean, # averaging function to use
- averagingConfig={}, # parameters to control averaging function (e.g. gaussInterp)
- optimization='fortran', # optimization mode (fortran or cython)
- cachePath=CachePath
- ):
- '''Compute a climatology over N arrays by applying a mask and averaging function.
-Returns the averaged variable grid, attributes of the primary variable, and the coordinate arrays in a dictionary.
-***Assumption: This routine assumes that the N grids will fit in memory.***
- '''
- print('accumulateClim: Doing %s ...' % averagingConfig['name'], file=sys.stderr)
- varList = [variable, mask]
- for i, url in enumerate(urls):
- try:
- path = retrieveFile(url, cachePath)
- fn = os.path.split(path)[1]
- vtime = int(fn[5:8]) # KLUDGE: extract DOY from filename
- except:
- print('climByAveraging: Error, continuing without file %s' % url, file=sys.stderr)
- continue
- if path is None: continue
- try:
- print('Reading file %s ...' % path, file=sys.stderr)
- var, fh = getVariables(path, varList, arrayOnly=True, order='F', set_auto_mask=False) # return dict of variable objects by name
- except:
- print('climByAveraging: Error, cannot read file %s' % path, file=sys.stderr)
- continue
- if i == 0:
- dtype = var[variable].dtype
- if 'int' in dtype.name: dtype = N.float32
- shape = var[variable].shape
- vsum = N.ma.empty(shape, dtype, order='F')
- vcount = N.ma.empty(shape, dtype, order='F')
- emptyVar = N.array(N.ma.masked_all(var[variable].shape, dtype), order='F') # totally masked variable array for missing or bad file reads
-
- print('Read coordinates ...', file=sys.stderr)
- var, fh = getVariables(path, coordinates, var, arrayOnly=True, order='F') # read coordinate arrays and add to dict
-
- var[variable] = maskFn(var[variable], var[mask]) # apply quality mask variable to get numpy MA, turned off masking done by netCDF4 library
-
- # Echo variable range for sanity check
- vals = var[variable].compressed()
- print('Variable Range: min, max:', vals.min(), vals.max(), file=sys.stderr)
-
- if averagingConfig['name'] == 'pixelMean':
- vsum += var[variable] # update accumulators
- vcount += ~var[mask]
-
- elif averagingConfig['name'] == 'gaussInterp':
- var[variable] = accum
- c = averagingConfig
- latGrid = c['latGrid']; lonGrid = c['lonGrid']
- if latGrid is not None and lonGrid is not None:
- outlat = N.arange(latGrid[0], latGrid[1]+latGrid[2], latGrid[2], dtype=N.float32, order='F')
- outlon = N.arange(lonGrid[0], lonGrid[1]+lonGrid[2], lonGrid[2], dtype=N.float32, order='F')
- else:
- outlat = N.array(var[coordinates[1]], dtype=N.float32, order='F')
- outlon = N.array(var[coordinates[2]], dtype=N.float32, order='F')
- varNames = [variable] + coordinates
- start = time.time()
- avg, weight, status = \
- gaussInterp(var, varNames, outlat, outlon, c['wlat'], c['wlon'],
- c['slat'], c['slon'], c['stime'], c['vfactor'], c['missingValue'],
- VERBOSE, optimization)
- end = time.time()
- vcount = weight.astype(N.float32)
- vsum = avg
- print('gaussInterp execution time:', (end - start), file=sys.stderr)
- elif averagingConfig['name'] == 'spatialFilter':
- var[variable] = accum
- c = averagingConfig
- varNames = [variable] + coordinates
- start = time.time()
- avg, vcount, status = \
- spatialFilter(var, varNames, c['spatialFilter'], c['normalization'],
- c['missingValue'], VERBOSE, optimization)
- vsum = avg
- end = time.time()
- print('spatialFilter execution time:', (end - start), file=sys.stderr)
- close(fh)
-
- return (vcount, vsum)
-
-
-def writeOutNetcdfVars(var, variable, mask, coordinates, inFile, outFile, fillValue=None, format='NETCDF4'):
- '''Construct output bundle of arrays with NetCDF dimensions and attributes.
-Output variables and attributes will have same names as the input file.
- '''
- din = Dataset(inFile, 'r')
- dout = Dataset(outFile, 'w', format=format)
- print('Writing %s ...' % outFile, file=sys.stderr)
-
- # Transfer global attributes from input file
- for a in din.ncattrs():
- dout.setncattr(a, din.getncattr(a))
-
- # Add dimensions and variables, copying data
- coordDim = [dout.createDimension(coord, var['out'+coord].shape[0]) for coord in coordinates] # here output lon, lat, etc.
- for coord in coordinates:
- v = dout.createVariable(coord, var['out'+coord].dtype, (coord,))
- v[:] = var['out'+coord][:]
- primaryVar = dout.createVariable(variable, var['out'+variable].dtype, coordinates, fill_value=fillValue)
- primaryVar[:] = var['out'+variable][:] # transfer array
- maskVar = dout.createVariable(mask, 'i1', coordinates)
- maskVar[:] = var['out'+mask].astype('i1')[:]
-
- # Transfer variable attributes from input file
- for k,v in dout.variables.items():
- for a in din.variables[k].ncattrs():
- if a == 'scale_factor' or a == 'add_offset' or a == '_FillValue': continue
- v.setncattr(a, din.variables[k].getncattr(a))
- if k == variable:
- try:
-# if fillValue == None: fillValue = din.variables[k].getncattr('_FillValue') # total kludge
- if fillValue == None: fillValue = default_fillvals['f4']
-# print >>sys.stderr, default_fillvals
-# v.setncattr('_FillValue', fillValue) # set proper _FillValue for climatology array
- v.setncattr('missing_value', fillValue)
- print('Setting missing_value for primary variable %s to %f' % (variable, fillValue), file=sys.stderr)
- except:
- print('writeOutNetcdfVars: Warning, for variable %s no fill value specified or derivable from inputs.' % variable, file=sys.stderr)
- din.close()
- dout.close()
- return outFile
-
-
-def contourMap(var, variable, coordinates, n, outFile):
- figFile = os.path.splitext(outFile)[0] + '_hist.png'
- # TODO: Downscale variable array (SST) before contouring, matplotlib is TOO slow on large arrays
- vals = var[variable][:]
-
- # Fixed color scale, write file, turn off auto borders, set title, reverse lat direction so monotonically increasing??
- imageMap(var[coordinates[1]][:], var[coordinates[0]][:], var[variable][:],
- vmin=-2., vmax=45., outFile=figFile, autoBorders=False,
- title='%s %d-day Mean from %s' % (variable.upper(), n, outFile))
- print('Writing contour plot to %s' % figFile, file=sys.stderr)
- return figFile
-
-
-def histogram(vals, variable, n, outFile):
- figFile = os.path.splitext(outFile)[0] + '_hist.png'
- M.clf()
-# M.hist(vals, 47, (-2., 45.))
- M.hist(vals, 94)
- M.xlim(-5, 45)
- M.xlabel('SST in degrees Celsius')
- M.ylim(0, 300000)
- M.ylabel('Count')
- M.title('Histogram of %s %d-day Mean from %s' % (variable.upper(), n, outFile))
- M.show()
- print('Writing histogram plot to %s' % figFile, file=sys.stderr)
- M.savefig(figFile)
- return figFile
-
-
-def dailyFile2date(path, offset=1):
- '''Convert YYYYDOY string in filename to date.'''
- fn = os.path.split(path)[1]
- year = int(fn[offset:offset+4])
- doy = int(fn[offset+5:offset+8])
- return fn[5:15].replace('.', '/')
-
-
-def formatRegion(r):
- """Format lat/lon region specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- strs = [str(i).replace('-', 'm') for i in r]
- return 'region-%s-%sby%s-%s' % tuple(strs)
-
-
-def formatGrid(r):
- """Format lat/lon grid resolution specifier as string suitable for file name."""
- if isinstance(r, str):
- return r
- else:
- return str(r[0]) + 'by' + str(r[1])
-
-
-def main(args):
- nEpochs = int(args[0])
- nWindow = int(args[1])
- nNeighbors = int(args[2])
- averager = args[3]
- optimization = args[4]
- mode = args[5]
- nWorkers = int(args[6])
- urlFile = args[7]
- urls = [s.strip() for s in open(urlFile, 'r')]
- if averager == 'gaussInterp':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=GaussInterpConfig,
- mode=mode, nWorkers=nWorkers)
- elif averager == 'spatialFilter':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=SpatialFilterConfig1,
- mode=mode, nWorkers=nWorkers)
- elif averager == 'pixelMean':
- results = climByAveragingPeriods(urls, nEpochs, nWindow, nNeighbors, 'sst', 'qual_sst', ['lat', 'lon'],
- averager=averager, optimization=optimization, averagingConfig=PixelMeanConfig,
- mode=mode, nWorkers=nWorkers)
- else:
- print('climatology2: Error, averager must be one of', list(AveragingFunctions.keys()), file=sys.stderr)
- sys.exit(1)
-
- if results[0][1] is not None:
- makeMovie([r[1] for r in results], 'clim.mpg')
- return results
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-
-# Old Tests:
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_10days.txt
-# python climatology2.py 5 5 3 gaussInterp fortran sequential 1 urls_sst_10days.txt
-
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_40days.txt
-# python climatology2.py 5 5 0 pixelMean fortran multicore 8 urls_sst_40days.txt
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 8 urls_sst_40days.txt
-
-# Old Production:
-# python climatology2.py 5 5 0 pixelMean fortran multicore 16 urls_sst_2015.txt >& log &
-# python climatology2.py 10 10 0 pixelMean fortran multicore 16 urls_sst_2015.txt >& log &
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 16 urls_sst_2015.txt >& log &
-
-
-# Tests:
-# python climatology2.py 5 5 0 pixelMean fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 0 pixelMean fortran multicore 4 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 3 gaussInterp fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 5 3 gaussInterp fortran multicore 4 urls_sst_daynight_5days_sorted.txt
-# python climatology2.py 5 7 1 spatialFilter fortran sequential 1 urls_sst_daynight_5days_sorted.txt
-
-# Test number of neighbors needed:
-# python climatology2.py 5 7 3 gaussInterp fortran multicore 4 urls_sst_daynight_20days_sorted.txt
-
-
-# Production:
-# python climatology2.py 5 7 0 pixelMean fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 3 gaussInterp fortran sequential 1 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 3 gaussInterp fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-# python climatology2.py 5 7 1 spatialFilter fortran multicore 4 urls_sst_daynight_2003_2015_sorted.txt
-
diff --git a/climatology/clim/cluster.py b/climatology/clim/cluster.py
deleted file mode 100644
index d52b852..0000000
--- a/climatology/clim/cluster.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# cluster.py -- Parallel map function that uses a distributed cluster and multicore within each node
-#
-
-import os, sys
-from .split import fixedSplit
-import pp
-#import dispy
-
-Servers = ("server-1", "server-2", "server-3", "server-4")
-NCores = 8
-
-
-def splitSeq(seq, n):
- '''Split a sequence into N almost even parts.'''
- m = int(len(seq)/n)
- return fixedSplit(seq, m)
-
-
-def dmap(func, # function to parallel map across split sequence
- seq, # sequence of data
- nCores, # number of cores to use on each node
- servers=[], # list of servers in the cluster
- splitterFn=splitSeq, # function to use to split the sequence of data
- ):
- '''Parallel operations for a cluster of nodes, each with multiple cores.
- '''
- nServers = len(servers)
- if nServers == 0: servers = ('localhost',)
- jobServer = pp.Server(nCores, ppservers=servers)
- splits = splitSeq(seq, nServers * nCores)
- jobs = [(s, jobServer.submit(func, (s,), ('splitSeq',), globals=globals())) for s in splits]
- results = [(j[0], j[1]()) for j in jobs]
- return results
-
-
-class PPCluster:
- '''Parallel operations for a cluster of nodes, each with multiple cores.
- '''
- def __init__(self, nCores=NCores, # number of cores to use on each node
- servers=Servers, # list of servers in the cluster
- splitterFn=splitSeq, # function to use to split the sequence of data
- ):
- self.nCores = nCores
- self.servers = servers
- self.nServers = len(servers)
- self.splitterFn = splitterFn
- self.jobServer = pp.Server(nCores, ppservers=servers)
-
- def dmap(self, func, seq):
- '''Distributed map function that automatically uses a cluster of nodes and a set number of cores.
-A sequence of data is split across the cluster.
- '''
- splits = splitSeq(seq, self.nServers * self.nCores)
- jobs = [(s, self.jobServer.submit(func, (s,), (splitterFn,), globals=globals())) for s in splits]
- results = [(j[0], j[1]()) for j in jobs]
- return results
-
-
-# Tests follow.
-
-def extractDay(url, substring):
- '''Extract integer DOY from filename like urls.'''
- i = url.find(substring)
- return int(url[5:8])
-
-
-def main(args):
- nCores = int(args[0])
- servers = tuple(eval(args[1]))
- urlFile = args[2]
- urls = [s.strip() for s in open(urlFile, 'r')]
-
-# results = PPCluster(nCores, servers).dmap(lambda u: extractDOY(u, 'A2015'), urls)
- results = dmap(lambda u: extractDOY(u, 'A2015'), urls, nCores, servers, splitSeq)
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-# python cluster.py 8 '["deepdata-1"]' urls_sst_2015.txt
-# python cluster.py 1 '["deepdata-1", "deepdata-2", "deepdata-3", "deepdata-4"]' urls_sst_2015.txt
-# python cluster.py 8 '["deepdata-1", "deepdata-2", "deepdata-3", "deepdata-4"]' urls_sst_2015.txt
-
diff --git a/climatology/clim/cluster2.py b/climatology/clim/cluster2.py
deleted file mode 100644
index d52b852..0000000
--- a/climatology/clim/cluster2.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# cluster.py -- Parallel map function that uses a distributed cluster and multicore within each node
-#
-
-import os, sys
-from .split import fixedSplit
-import pp
-#import dispy
-
-Servers = ("server-1", "server-2", "server-3", "server-4")
-NCores = 8
-
-
-def splitSeq(seq, n):
- '''Split a sequence into N almost even parts.'''
- m = int(len(seq)/n)
- return fixedSplit(seq, m)
-
-
-def dmap(func, # function to parallel map across split sequence
- seq, # sequence of data
- nCores, # number of cores to use on each node
- servers=[], # list of servers in the cluster
- splitterFn=splitSeq, # function to use to split the sequence of data
- ):
- '''Parallel operations for a cluster of nodes, each with multiple cores.
- '''
- nServers = len(servers)
- if nServers == 0: servers = ('localhost',)
- jobServer = pp.Server(nCores, ppservers=servers)
- splits = splitSeq(seq, nServers * nCores)
- jobs = [(s, jobServer.submit(func, (s,), ('splitSeq',), globals=globals())) for s in splits]
- results = [(j[0], j[1]()) for j in jobs]
- return results
-
-
-class PPCluster:
- '''Parallel operations for a cluster of nodes, each with multiple cores.
- '''
- def __init__(self, nCores=NCores, # number of cores to use on each node
- servers=Servers, # list of servers in the cluster
- splitterFn=splitSeq, # function to use to split the sequence of data
- ):
- self.nCores = nCores
- self.servers = servers
- self.nServers = len(servers)
- self.splitterFn = splitterFn
- self.jobServer = pp.Server(nCores, ppservers=servers)
-
- def dmap(self, func, seq):
- '''Distributed map function that automatically uses a cluster of nodes and a set number of cores.
-A sequence of data is split across the cluster.
- '''
- splits = splitSeq(seq, self.nServers * self.nCores)
- jobs = [(s, self.jobServer.submit(func, (s,), (splitterFn,), globals=globals())) for s in splits]
- results = [(j[0], j[1]()) for j in jobs]
- return results
-
-
-# Tests follow.
-
-def extractDay(url, substring):
- '''Extract integer DOY from filename like urls.'''
- i = url.find(substring)
- return int(url[5:8])
-
-
-def main(args):
- nCores = int(args[0])
- servers = tuple(eval(args[1]))
- urlFile = args[2]
- urls = [s.strip() for s in open(urlFile, 'r')]
-
-# results = PPCluster(nCores, servers).dmap(lambda u: extractDOY(u, 'A2015'), urls)
- results = dmap(lambda u: extractDOY(u, 'A2015'), urls, nCores, servers, splitSeq)
-
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-# python cluster.py 8 '["deepdata-1"]' urls_sst_2015.txt
-# python cluster.py 1 '["deepdata-1", "deepdata-2", "deepdata-3", "deepdata-4"]' urls_sst_2015.txt
-# python cluster.py 8 '["deepdata-1", "deepdata-2", "deepdata-3", "deepdata-4"]' urls_sst_2015.txt
-
diff --git a/climatology/clim/datasets.py b/climatology/clim/datasets.py
deleted file mode 100644
index 29562b8..0000000
--- a/climatology/clim/datasets.py
+++ /dev/null
@@ -1,331 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-'''
-datasets.py -- Routines for dataset-specfic capabilities: file handling, readers, etc.
-
-One Class for each dataset containing static methods and constants/templates, etc.
-
-'''
-
-import sys, os, re, datetime
-
-import numpy as np
-
-from .split import splitByNDaysKeyed, groupByKeys, extractKeys
-
-
-def splitModisAod(seq, n):
- return splitByNDaysKeyed(seq, n, re.compile(r'(....)(..)(..)'), lambda y, m, d: ymd2doy(y, m, d))
-
-
-def splitAvhrrSst(seq, n):
- return splitByNDays_Avhrr(seq, n, re.compile(r'^(....)(..)(..)'))
-
-
-class ModisSst:
- ExpectedRunTime = "28m"
- UrlsPath = "/data/share/datasets/MODIS_L3_AQUA_11UM_V2014.0_4KM_DAILY/daily_data/A*SST*.nc"
- ExampleFileName = 'A2010303.L3m_DAY_NSST_sst_4km.nc'
- GetKeysRegex = r'A(....)(...).L3m_DAY_(.)S'
-
- VariableName = 'sst'
- Mask = None
- Coordinates = ['lat', 'lon']
-
- OutputClimTemplate = ''
-
- @staticmethod
- def keysTransformer(s): return (s[1], s[0], s[2]) # DOY, YEAR, N=night / S=day
-
- @staticmethod
- def getKeys(url):
- return extractKeys(url, ModisSst.GetKeysRegex, ModisSst.keysTransformer)
-
- @staticmethod
- def split(seq, n):
- return [u for u in splitByNDaysKeyed(seq, n, ModisSst.GetKeysRegex, ModisSst.keysTransformer)]
-
- @staticmethod
- def genOutputName(doy, variable, nEpochs, averagingConfig):
- return 'A%03d.L3m_%s_%dday_clim_%s.nc' % (
- doy, variable, nEpochs, averagingConfig['name']) # mark each file with first day in period
-
-
-class ModisChlor:
- ExpectedRunTime = "11m"
- UrlsPath = "/Users/greguska/githubprojects/nexus/nexus-ingest/developer-box/data/modis_aqua_chl/A*chlor*.nc"
- ExampleFileName = "A2013187.L3m_DAY_CHL_chlor_a_4km.nc"
- GetKeysRegex = r'A(....)(...).L3m.*CHL'
-
- Variable = 'chlor_a'
- Mask = None
- Coordinates = ['lat', 'lon']
-
- OutputClimTemplate = ''
-
- @staticmethod
- def keysTransformer(s): return (s[1], s[0]) # DOY, YEAR
-
- @staticmethod
- def getKeys(url):
- return extractKeys(url, ModisChlor.GetKeysRegex, ModisChlor.keysTransformer)
-
- @staticmethod
- def split(seq, n):
- return [u for u in splitByNDaysKeyed(seq, n, ModisChlor.GetKeysRegex, ModisChlor.keysTransformer)]
-
- @staticmethod
- def genOutputName(doy, variable, nEpochs, averagingConfig):
- return 'A%03d.L3m_%s_%dday_clim_%s.nc' % (
- doy, variable, nEpochs, averagingConfig['name']) # mark each file with first day in period
-
-
-class MeasuresSsh:
- ExpectedRunTime = "2m22s"
- UrlsPath = "/data/share/datasets/MEASURES_SLA_JPL_1603/daily_data/ssh_grids_v1609*12.nc"
- ExampleFileName = "ssh_grids_v1609_2006120812.nc"
- GetKeysRegex = r'ssh.*v1609_(....)(..)(..)12.nc'
-
- Variable = 'SLA' # sea level anomaly estimate
- Mask = None
- Coordinates = ['Longitude', 'Latitude'] # Time is first (len=1) coordinate, will be removed
-
- OutputClimTemplate = ''
-
- @staticmethod
- def keysTransformer(s): return (ymd2doy(s[0], s[1], s[2]), s[0]) # DOY, YEAR
-
- @staticmethod
- def getKeys(url):
- return extractKeys(url, MeasuresSsh.GetKeysRegex, MeasuresSsh.keysTransformer)
-
- @staticmethod
- def split(seq, n):
- return [u for u in splitByNDaysKeyed(seq, n, MeasuresSsh.GetKeysRegex, MeasuresSsh.keysTransformer)]
-
- @staticmethod
- def genOutputName(doy, variable, nEpochs, averagingConfig):
- return "ssh_grids_v1609_%03d_%dday_clim_%s.nc" % (int(doy), nEpochs, averagingConfig['name'])
-
-
-class CCMPWind:
- ExpectedRunTime = "?"
- UrlsPath = "/data/share/datasets/CCMP_V2.0_L3.0/daily_data/CCMP_Wind*_V02.0_L3.0_RSS_uncompressed.nc"
- ExampleFileName = "CCMP_Wind_Analysis_20160522_V02.0_L3.0_RSS_uncompressed.nc"
- GetKeysRegex = r'CCMP_Wind_Analysis_(....)(..)(..)_V.*.nc'
-
- Variable = 'Wind_Magnitude' # to be computed as sqrt(uwnd^2 + vwnd^2)
- Mask = None
- Coordinates = ['latitude', 'longitude']
-
- OutputClimTemplate = ''
-
- @staticmethod
- def keysTransformer(s):
- return (ymd2doy(s[0], s[1], s[2]), s[0]) # DOY, YEAR
-
- @staticmethod
- def getKeys(url):
- return extractKeys(url, CCMPWind.GetKeysRegex, CCMPWind.keysTransformer)
-
- @staticmethod
- def split(seq, n):
- return [u for u in splitByNDaysKeyed(seq, n, CCMPWind.GetKeysRegex, CCMPWind.keysTransformer)]
-
- @staticmethod
- def genOutputName(doy, variable, nEpochs, averagingConfig):
- return "CCMP_Wind_Analysis_V02.0_L3.0_RSS_%03d_%dday_clim_%s.nc" % (int(doy), nEpochs, averagingConfig['name'])
-
- @staticmethod
- def readAndMask(url, variable, mask=None, cachePath='/tmp/cache', hdfsPath=None):
- """
- Read a variable from a netCDF or HDF file and return a numpy masked array.
- If the URL is remote or HDFS, first retrieve the file into a cache directory.
- """
- from .variables import getVariables, close
- v = None
- if mask:
- variables = [variable, mask]
- else:
- variables = [variable]
- try:
- from .cache import retrieveFile
- path = retrieveFile(url, cachePath, hdfsPath)
- except:
- print('readAndMask: Error, continuing without file %s' % url, file=sys.stderr)
- return v
-
- if CCMPWind.Variable in variables:
- var, fh = getVariables(path, ['uwnd','vwnd'], arrayOnly=True,
- set_auto_mask=True) # return dict of variable objects by name
- uwnd_avg = np.average(var['uwnd'], axis=0)
- vwnd_avg = np.average(var['vwnd'], axis=0)
- wind_magnitude = np.sqrt(np.add(np.multiply(uwnd_avg, uwnd_avg), np.multiply(vwnd_avg, vwnd_avg)))
- v = wind_magnitude
- if v.shape[0] == 1: v = v[0] # throw away trivial time dimension for CF-style files
- close(fh)
- else:
- try:
- print('Reading variable %s from %s' % (variable, path), file=sys.stderr)
- var, fh = getVariables(path, variables, arrayOnly=True,
- set_auto_mask=True) # return dict of variable objects by name
- v = var[
- variable] # could be masked array
- if v.shape[0] == 1: v = v[0] # throw away trivial time dimension for CF-style files
- close(fh)
- except:
- print('readAndMask: Error, cannot read variable %s from file %s' % (variable, path), file=sys.stderr)
-
- return v
-
-class MonthlyClimDataset:
- ExpectedRunTime = "2m"
- UrlsPath = ''
- ExampleFileName = ''
- GetKeysRegex = r'(YYYY)(MM)(DD)' # Regex to extract year, month, day
- Variable = 'var' # Variable name in granule
- Mask = None
- Coordinates = ['lat', 'lon']
- OutputClimTemplate = ''
-
- @staticmethod
- def keysTransformer(s):
- return (s[1],s[0]) # MONTH, YEAR
-
- @staticmethod
- def getKeys(url):
- return extractKeys(url, MonthlyClimDataset.GetKeysRegex,
- MonthlyClimDataset.keysTransformer)
-
- @staticmethod
- def split(seq, n):
- return [u for u in splitByNDaysKeyed(seq, n,
- MonthlyClimDataset.GetKeysRegex,
- MonthlyClimDataset.keysTransformer)]
-
- @staticmethod
- def genOutputName(month, variable, nEpochs, averagingConfig):
- # Here we use the 15th of the month to get DOY and just use any
- # non-leap year.
- doy = datetime2doy(ymd2datetime(2017, month, 15))
- return 'monthly_clim_%s_%03d_month%02d_nepochs%d_%s.nc' % (
- variable, doy, month, nEpochs,
- averagingConfig['name']) # mark each file with month
-
-
-class SMAP_L3M_SSS(MonthlyClimDataset):
- UrlsPath = "/data/share/datasets/SMAP_L3_SSS/monthly/RSS_smap_SSS_monthly_*.nc"
- ExampleFileName = 'RSS_smap_SSS_monthly_2015_04_v02.0.nc'
- GetKeysRegex = r'RSS_smap_SSS_monthly_(....)_(..)_v02'
- Variable = 'sss_smap'
-
- @staticmethod
- def getKeys(url):
- return extractKeys(url, SMAP_L3M_SSS.GetKeysRegex,
- SMAP_L3M_SSS.keysTransformer)
-
- @staticmethod
- def split(seq, n):
- return [u for u in splitByNDaysKeyed(seq, n,
- SMAP_L3M_SSS.GetKeysRegex,
- SMAP_L3M_SSS.keysTransformer)]
-
- @staticmethod
- def genOutputName(month, variable, nEpochs, averagingConfig):
- # Here we use the 15th of the month to get DOY and just use any
- # non-leap year.
- doy = datetime2doy(ymd2datetime(2017, month, 15))
- return '%s_L3m_clim_doy%03d_month%02d_nepochs%d_%s.nc' % (
- variable, doy, month, nEpochs,
- averagingConfig['name']) # mark each file with month
-
-class GRACE_Tellus(MonthlyClimDataset):
- GetKeysRegex = r'GRCTellus.JPL.(....)(..)(..).GLO'
- Variable = 'lwe_thickness' # Liquid_Water_Equivalent_Thickness
-
- @staticmethod
- def getKeys(url):
- return extractKeys(url, GRACE_Tellus.GetKeysRegex,
- GRACE_Tellus.keysTransformer)
-
- @staticmethod
- def split(seq, n):
- return [u for u in splitByNDaysKeyed(seq, n,
- GRACE_Tellus.GetKeysRegex,
- GRACE_Tellus.keysTransformer)]
-
- @staticmethod
- def genOutputName(month, variable, nEpochs, averagingConfig):
- # Here we use the 15th of the month to get DOY and just use any
- # non-leap year.
- doy = datetime2doy(ymd2datetime(2017, month, 15))
- return 'GRACE_Tellus_monthly_%s_%03d_month%02d_nepochs%d_%s.nc' % (
- variable, doy, month, nEpochs,
- averagingConfig['name']) # mark each file with month
-
-class GRACE_Tellus_monthly_land(GRACE_Tellus):
- UrlsPath = "/data/share/datasets/GRACE_Tellus/monthly_land/GRCTellus.JPL.*.nc"
- ExampleFileName = "GRCTellus.JPL.20150122.GLO.RL05M_1.MSCNv02CRIv02.land.nc"
-
- @staticmethod
- def genOutputName(month, variable, nEpochs, averagingConfig):
- # Here we use the 15th of the month to get DOY and just use any
- # non-leap year.
- doy = datetime2doy(ymd2datetime(2017, month, 15))
- return 'GRACE_Tellus_monthly_land_%s_%03d_month%02d_nepochs%d_%s.nc' % (
- variable, doy, month, nEpochs,
- averagingConfig['name']) # mark each file with month
-
-class GRACE_Tellus_monthly_ocean(GRACE_Tellus):
- UrlsPath = "/data/share/datasets/GRACE_Tellus/monthly_ocean/GRCTellus.JPL.*.nc"
- ExampleFileName = "GRCTellus.JPL.20150122.GLO.RL05M_1.MSCNv02CRIv02.ocean.nc"
-
- @staticmethod
- def genOutputName(month, variable, nEpochs, averagingConfig):
- # Here we use the 15th of the month to get DOY and just use any
- # non-leap year.
- doy = datetime2doy(ymd2datetime(2017, month, 15))
- return 'GRACE_Tellus_monthly_ocean_%s_%03d_month%02d_nepochs%d_%s.nc'%(
- variable, doy, month, nEpochs,
- averagingConfig['name']) # mark each file with month
-
-
-DatasetList = {'ModisSst': ModisSst, 'ModisChlor': ModisChlor,
- 'MeasuresSsh': MeasuresSsh, 'CCMPWind': CCMPWind,
- 'SMAP_L3M_SSS': SMAP_L3M_SSS,
- 'GRACE_Tellus_monthly_ocean': GRACE_Tellus_monthly_ocean,
- 'GRACE_Tellus_monthly_land': GRACE_Tellus_monthly_land}
-
-
-# Utils follow.
-def ymd2doy(year, mon, day):
- return datetime2doy(ymd2datetime(year, mon, day))
-
-
-def ymd2datetime(y, m, d):
- y, m, d = list(map(int, (y, m, d)))
- return datetime.datetime(y, m, d)
-
-
-def datetime2doy(dt):
- return int(dt.strftime('%j'))
-
-
-def doy2datetime(year, doy):
- '''Convert year and DOY (day of year) to datetime object.'''
- return datetime.datetime(int(year), 1, 1) + datetime.timedelta(int(doy) - 1)
-
-
-def doy2month(year, doy): return doy2datetime(year, doy).strftime('%m')
diff --git a/climatology/clim/dparkTest.py b/climatology/clim/dparkTest.py
deleted file mode 100644
index 072a923..0000000
--- a/climatology/clim/dparkTest.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import dpark
-
-lines = dpark.textFile("/usr/share/dict/words", 128)
-#lines = dpark.textFile("/usr/share/doc/gcc-4.4.7/README.Portability", 128)
-words = lines.flatMap(lambda x:x.split()).map(lambda x:(x,1))
-wc = words.reduceByKey(lambda x,y:x+y).collectAsMap()
-print(wc[wc.keys[0]])
-
diff --git a/climatology/clim/gaussInterp.py b/climatology/clim/gaussInterp.py
deleted file mode 100644
index 75b9a18..0000000
--- a/climatology/clim/gaussInterp.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# gaussInterp routine -- Gaussian weighted smoothing in lat, lon, and time
-#
-# Based on Ed Armstrong's routines.
-#
-# Calls into optimized routines in Fortran or cython.
-# See gaussInterp_f.f or gaussInterp.pyx
-#
-
-import numpy as N, time
-from gaussInterp_f import gaussinterp_f
-
-
-def gaussInterp(var, # bundle of input arrays: masked variable, coordinates
- varNames, # list of names in order: primary variable, coordinates in order lat, lon, time
- outlat, outlon, # output lat/lon coordinate vectors
- wlat, wlon, # window of lat/lon neighbors to gaussian weight, expressed in delta lat (degrees)
- slat, slon, stime, # sigma for gaussian downweighting with distance in lat, lon (deg), & time (days)
- vfactor=-0.6931, # factor in front of gaussian expression
- missingValue=-9999., # value to mark missing values in interp result
- verbose=1, # integer to set verbosity level
- optimization='fortran'): # Mode of optimization, using 'fortran' or 'cython'
- '''Gaussian interpolate in lat, lon, and time to a different lat/lon grid, and over a time window to the center time.
-Bundle of arrays (var) contains a 3D masked variable and coordinate arrays for lat, lon, and time read from netdf/hdf files.
-Returns the 2D interpolated variable (masked) and a status for failures.
- '''
- v = var[varNames[0]][:] # real*4 in Fortran code, is v.dtype correct?
- vmask = N.ma.getmask(v).astype('int8')[:] # integer*1, convert bool mask to one-byte integer for Fortran
- vtime = var[varNames[1]][:] # integer*4 in Fortran
- lat = var[varNames[2]][:] # real*4
- lon = var[varNames[3]][:] # real*4
- if optimization == 'fortran':
- vinterp, vweight, status = \
- gaussinterp_f(v, vmask, vtime, lat, lon,
- outlat, outlon, wlat, wlon, slat, slon, stime, vfactor, missingValue, verbose)
- else:
- vinterp, vweight, status = \
- gaussInterp_(v, vmask, vtime, lat, lon,
- outlat, outlon, wlat, wlon, slat, slon, stime, vfactor, missingValue, verbose)
-
- vinterp = N.ma.masked_where(vweight == 0.0, vinterp)
- return (vinterp, vweight, status)
-
diff --git a/climatology/clim/gaussInterp.pyx b/climatology/clim/gaussInterp.pyx
deleted file mode 100644
index 95b607c..0000000
--- a/climatology/clim/gaussInterp.pyx
+++ /dev/null
@@ -1,145 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-c
-c gaussInterp routine -- Gaussian weighted smoothing in lat, lon, and time
-c
-c Based on Ed Armstrong's routines. Designed to be called from python using f2py.
-c
-c
-c Gaussian weighting = exp( vfactor * (((x - x0)/sx)^2 + ((y - y0)/sy)^2 + ((t - t0)/st)^2 ))
-c
-c where deltas are distances in lat, lon and time and sx, sy, st are one e-folding sigmas.
-c
-c Cutoffs for neighbors allowed in the interpolation are set by distance in lat/lon (see dlat/dlon);
-c for time all epochs are included.
-c
-
-import sys
-import numpy as np
-cimport numpy as np
-from math import exp
-
-from gaussInterp_f import gaussInterp_f
-
-Var_t = np.float
-ctypedef np.float_t Var_t
-
-VERBOSE = 1
-
-
-def gaussInterp(var, # bundle of input arrays: masked variable, coordinates
- varNames, # list of names in order: primary variable, coordinates in order lat, lon, time
- outlat, outlon, # output lat/lon coordinate vectors
- wlat, wlon, # window of lat/lon neighbors to gaussian weight, expressed in delta lat (degrees)
- slat, slon, stime, # sigma for gaussian downweighting with distance in lat, lon (deg), & time (days)
- vfactor=-0.6931, # factor in front of gaussian expression
- missingValue=-9999., # value to mark missing values in interp result
- verbose=VERBOSE, # integer to set verbosity level
- optimization='cython'): # Mode of optimization, using 'fortran' or 'cython'
- '''Gaussian interpolate in lat, lon, and time to a different lat/lon grid, and over a time window to the center time.
-Bundle of arrays (var) contains a 3D masked variable and coordinate arrays for lat, lon, and time read from netdf/hdf files.
-Returns the 2D interpolated variable (masked) and a status for failures.
- '''
- v = var[varNames[0]][:]
- vmask = np.ma.getmask(v)[:]
- vtime = var[varNames[1]][:]
- lat = var[varNames[2]][:]
- lon = var[varNames[3]][:]
- if optimization == 'fortran':
- vinterp, vweight, status =
- gaussInterp_f(v, vmask, vtime, lat, lon,
- outlat, outlon, wlat, wlon, slat, slon, stime, vfactor, missingValue)
- else:
- vinterp, vweight, status =
- gaussInterp_(v, vmask, vtime, lat, lon,
- outlat, outlon, wlat, wlon, slat, slon, stime, vfactor, missingValue)
- vinterp = np.ma.array(vinterp, mask=np.ma.make_mask(vweight)) # apply mask
- return (vinterp, vweight, status)
-
-
-#@cython.boundscheck(False)
-def gaussInterp_(np.ndarray[Var_t, ndim=3] var, # variable & mask arrays with dimensions of lat,lon,time
- np.ndarray[Var_t, ndim=3] vmask,
- np.ndarray[np.int_t, ndim=1] vtime,
- np.ndarray[np.float_t, ndim=1] lat,
- np.ndarray[np.float_t, ndim=1] lon, # coordinate vectors for inputs
- np.ndarray[np.float_t, ndim=1] outlat, # coordinate vectors for grid to interpolate to
- np.ndarray[np.float_t, ndim=1] outlon,
- float wlat, float wlon, # window of lat/lon neighbors to gaussian weight, expressed in delta lat (degrees)
- float slat, float slon, float stime, # sigma for gaussian downweighting with distance in lat, lon (deg), & time (days)
- float vfactor, # factor in front of gaussian expression
- float missingValue): # value to mark missing values in interp result
- '''Gaussian interpolate in lat, lon, and time to a different lat/lon grid, and over a time window to the center time.
-Returns the 2D interpolated variable (masked), the weight array, and a status for failures.
- '''
- assert var.dtype == Var_t and mask.dtype == np.int
-
- cdef np.ndarray[Var_t, ndim=2] vinterp = np.zeros( (outlat.shape[0], outlon.shape[0]), dtype=Var_t ) # interpolated variable, missing values not counted
- cdef np.ndarray[Var_t, ndim=2] vweight = np.zeros( (outlat.shape[0], outlon.shape[0]), dtype=Var_t ) # weight of values interpolated (can be zero)
- cdef int status = 0 # negative status indicates error
-
- cdef int imin, imax, jmin, jmax
- cdef int iin, jin, kin
- cdef int i, j
-
- cdef int ntime = time.shape[0]
- cdef int nlat = lat.shape[0]
- cdef int nlon = lon.shape[0]
-
- cdef int noutlat = outlat.shape[0]
- cdef int noutlon = outlon.shape[0]
-
- cdef float wlat2 = wlat / 2.
- cdef float wlon2 = wlon / 2.
- cdef float lat0 = lat[0]
- cdef float lon0 = lon[0]
- cdef float dlat = lat[1] - lat[0]
- cdef float dlon = lon[1] - lon[0]
- cdef double midTime = time[int(ntime/2 + 0.5)]
-
- for i xrange(noutlat):
- print >>sys.stderr, outlat[i]
- for j in xrange(noutlon):
- imin = clamp(int((outlat[i] - wlat2 - lat0)/dlat + 0.5), 0, nlat-1)
- imax = clamp(int((outlat[i] + wlat2 - lat0)/dlat + 0.5), 0, nlat-1)
- jmin = clamp(int((outlon[j] - wlon2 - lon0)/dlon + 0.5), 0, nlon-1)
- jmax = clamp(int((outlon[j] + wlon2 - lon0)/dlon + 0.5), 0, nlon-1)
-
- for kin in xrange(ntime):
- for iin in xrange(imin, imax+1):
- for jin in xrange(jmin, jmax+1):
- if not vmask[kin, iin, jin]:
- fac = exp( vfactor *
- (((outlat[i] - lat[iin])/slat)**2
- + ((outlon[j] - lon[jin])/slon)**2
- + ((midTime - vtime[kin])/stime)**2))
- val = var[kin, iin, jin]
- if VERBOSE > 1: print >>sys.stderr, kin, iin, jin, vtime[kin], lat[iin], lon[jin], val, fac, val*fac
-
- vinterp[i,j] = vinterp[i,j] + val * fac
- vweight[i,j] = vweight[i,j] + fac
-
- if vweight[i,j] != 0.0:
- vinterp[i,j] = vinterp[i,j] / vweight[i,j]
- else:
- vinterp[i,j] = missingValue
-
- return (vinterp, vweight, status)
-
-
-cdef int clamp(int i, int n, int m):
- if i < n: return n
- if i > m: return m
- return i
diff --git a/climatology/clim/gaussInterp_f.f b/climatology/clim/gaussInterp_f.f
deleted file mode 100644
index 9366870..0000000
--- a/climatology/clim/gaussInterp_f.f
+++ /dev/null
@@ -1,219 +0,0 @@
-c
-c gaussInterp routine -- Gaussian weighted smoothing in lat, lon, and time
-c
-c Based on Ed Armstrong's routines. Designed to be called from python using f2py.
-c
-c
-c Gaussian weighting = exp( vfactor * (((x - x0)/sx)^2 + ((y - y0)/sy)^2 + ((t - t0)/st)^2 ))
-c
-c where deltas are distances in lat, lon and time and sx, sy, st are one e-folding sigmas.
-c
-c Cutoffs for neighbors allowed in the interpolation are set by distance in lat/lon (see wlat/wlon);
-c for time all epochs are included.
-
-
- subroutine gaussInterp_f(var, mask,
- & time, lat, lon,
- & outlat, outlon,
- & wlat, wlon,
- & slat, slon, stime,
- & vfactor,
- & missingValue,
- & verbose,
- & vinterp, vweight, status,
- & ntime, nlat, nlon, noutlat, noutlon)
-
- implicit none
- integer*4 ntime, nlat, nlon
-c ! prepared 3D array of variable data over time, lon, & lat
- real*4 var(ntime, nlat, nlon)
-c ! variable to be interpolated, over ntime epochs
- integer*1 mask(ntime, nlat, nlon)
-c ! pixel quality mask
- integer*4 time(ntime)
-c ! time epochs to gaussian-interpolate over
- real*4 lat(nlat)
- ! latitude corrdinate vector
- real*4 lon(nlon)
- ! longitude corrdinate vector
-cf2py intent(in) var, mask, time, lat, lon !! annotations for f2py processor
- integer*4 noutlat, noutlon
- real*4 outlat(noutlat), outlon(noutlon)
-c ! lat/lon grid to interpolate to
-cf2py intent(in) outlat, outlon
- real*4 wlat, wlon
-c ! window of lat/lon neighbors to gaussian weight, expressed in delta lat (degrees)
-c ! if an integer, then it is the number of neighbors on EACH side
-cf2py intent(in) wlat, wlon
- real*4 slat, slon, stime
-c ! sigma for gaussian downweighting with distance in lat, lon, & time
-cf2py intent(in) slat, slon, stime
- real*4 vfactor
-c ! factor in front of gaussian expression
- real*4 missingValue
-c ! value to mark missing values in interp result
- integer*4 verbose
-c ! integer to set verbosity level
-cf2py intent(in) vfactor, missingValue
- real*4 vinterp(noutlat, noutlon)
-c ! interpolated variable using gaussians, missing values not counted
- real*4 vweight(noutlat, noutlon)
-c ! weight of values interpolated (can be zero), if so should become missing value
- integer*4 status
-c ! negative status indicates error
-cf2py intent(out) vinterp, vweight, status
-
- integer*4 clamp
- integer*4 imin, imax, jmin, jmax, itmp
- integer*4 iin, jin, kin
- integer*4 iMidTime
- integer*4 i, j, iwlat, iwlon
-
- real*4 wlat2, wlon2, lat1, lon1, dlat, dlon, fac
- real*4 varmin, varmax, val
- real*8 midTime
- logical*1 nLatNeighbors, nLonNeighbors
-
- write(6, *) 'Echoing inputs ...'
- write(6, *) 'ntime, nlat, nlon:', ntime, nlat, nlon
- write(6, *) 'noutlat, noutlon:', noutlat, noutlon
- write(6, *) 'wlat, wlon:', wlat, wlon
- write(6, *) 'slat, slon, stime:', slat, slon, stime
- write(6, *) 'vfactor:', vfactor
- write(6, *) 'missingValue:', missingValue
-
- status = 0
- if (int(wlat) .eq. wlat) then
- iwlat = int(wlat)
- nLatNeighbors = .TRUE.
- write(6, *) 'Using', iwlat,
- & 'neighbors on each side in the lat direction.'
- else
- nLatNeighbors = .FALSE.
- end if
- if (int(wlon) .eq. wlon) then
- iwlon = int(wlon)
- nLonNeighbors = .TRUE.
- write(6, *) 'Using', iwlon,
- & 'neighbors on each side in the lon direction.'
- else
- nLonNeighbors = .FALSE.
- end if
-
- wlat2 = wlat / 2.
- wlon2 = wlon / 2.
- lat1 = lat(1)
- lon1 = lon(1)
- dlat = lat(2) - lat(1)
- dlon = lon(2) - lon(1)
- iMidTime = int(ntime/2 + 0.5)
- midTime = time(iMidTime)
-
- if (verbose .gt. 3) then
- write(6, *) 'time:', time
- write(6, *) 'lat:', lat
- write(6, *) 'lon:', lon
- write(6, *) 'outlat:', outlat
- write(6, *) 'outlon:', outlon
-c write(6, *) 'mask(iMidTime):', mask(iMidTime,:,:)
- write(6, *) 'var(iMidTime):', var(iMidTime,:,:)
- end if
-
- do i = 1, noutlat
- if (verbose .gt. 1) write(6, *) outlat(i)
- do j = 1, noutlon
- vinterp(i,j) = 0.0
- vweight(i,j) = 0.0
- if (verbose .gt. 3) then
- write(6, *) '(i,j) = ', i, j
- write(6, *) '(outlat,outlon) = ', outlat(i), outlon(j)
- end if
-
- if (nLatNeighbors) then
- imin = clamp(i - iwlat, 1, nlat)
- imax = clamp(i + iwlat, 1, nlat)
- else
- imin = clamp(int((outlat(i) - wlat2 - lat1)/dlat + 0.5),
- & 1, nlat)
- imax = clamp(int((outlat(i) + wlat2 - lat1)/dlat + 0.5),
- & 1, nlat)
- end if
- if (imin .gt. imax) then
-c ! input latitudes descending, so swap
- itmp = imin
- imin = imax
- imax = itmp
- end if
-
- if (nLonNeighbors) then
- jmin = clamp(j - iwlon, 1, nlon)
- jmax = clamp(j + iwlon, 1, nlon)
- else
- jmin = clamp(int((outlon(j) - wlon2 - lon1)/dlon + 0.5),
- & 1, nlon)
- jmax = clamp(int((outlon(j) + wlon2 - lon1)/dlon + 0.5),
- & 1, nlon)
- end if
-
- if (verbose .gt. 2) then
- write(6, *) '(imin,imax,minlat,maxlat) = ',
- & imin, imax, lat(imin), lat(imax)
- write(6, *) '(jmin,jmax,minlon,maxlon) = ',
- & jmin, jmax, lon(jmin), lon(jmax)
- end if
-
- if (verbose .gt. 4 .and. i .eq. noutlat/2
- & .and. j .eq. noutlon/2) then
- write(6, *) 'Echoing factors ...'
- end if
-
- do kin = 1, ntime
- varmin = 0.
- varmax = 0.
- do iin = imin, imax
- do jin = jmin, jmax
- if (mask(kin,iin,jin) .eq. 0) then
- fac = exp( vfactor *
- & (((outlat(i) - lat(iin))/slat)**2
- & + ((outlon(j) - lon(jin))/slon)**2
- & + ((midTime - time(kin))/stime)**2))
-
- val = var(kin,iin,jin)
- if (verbose .gt. 4 .and. i .eq. noutlat/2
- & .and. j .eq. noutlon/2) then
- write(6, *) kin, iin, jin, time(kin),
- & lat(iin), lon(jin), val, fac, val*fac
- end if
-
- vinterp(i,j) = vinterp(i,j) + val * fac
- vweight(i,j) = vweight(i,j) + fac
- if (verbose .gt. 3) then
- varmin = min(varmin, val)
- varmax = max(varmax, val)
- end if
- end if
- end do
- end do
- if (verbose .gt. 3) then
- write(6, *) '(itime, varmin, varmax) = ',
- & kin, varmin, varmax
- end if
- end do
- if (vweight(i,j).ne. 0.0) then
- vinterp(i,j) = vinterp(i,j) / vweight(i,j)
- else
- vinterp(i,j) = missingValue
- end if
- end do
- end do
- return
- end
-
-
- integer*4 function clamp(i, n, m)
- integer*4 i, n, m
- clamp = i
- if (i .lt. n) clamp = n
- if (i .gt. m) clamp = m
- end
-
diff --git a/climatology/clim/gaussInterp_f.mk b/climatology/clim/gaussInterp_f.mk
deleted file mode 100644
index 548ce33..0000000
--- a/climatology/clim/gaussInterp_f.mk
+++ /dev/null
@@ -1 +0,0 @@
-f2py -c -m gaussInterp_f gaussInterp_f.f
diff --git a/climatology/clim/gaussInterp_slow.py b/climatology/clim/gaussInterp_slow.py
deleted file mode 100644
index 4ad6676..0000000
--- a/climatology/clim/gaussInterp_slow.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# gaussInterp_slow routine -- Gaussian weighted smoothing in lat, lon, and time
-#
-# Based on Ed Armstrong's routines. Pure python implementation.
-#
-#
-# Gaussian weighting = exp( vfactor * (((x - x0)/sx)^2 + ((y - y0)/sy)^2 + ((t - t0)/st)^2 ))
-#
-# where deltas are distances in lat, lon and time and sx, sy, st are one e-folding sigmas.
-#
-# Cutoffs for neighbors allowed in the interpolation are set by distance in lat/lon (see dlat/dlon);
-# for time all epochs are included.
-#
-
-import sys
-import numpy as np
-from math import exp
-from numba import jit, int32
-
-VERBOSE = 0
-
-
-def gaussInterp_slow(var, # bundle of input arrays: masked variable, coordinates
- varNames, # list of names in order: primary variable, coordinates in order lat, lon, time
- outlat, outlon, # output lat/lon coordinate vectors
- wlat, wlon, # window of lat/lon neighbors to gaussian weight, expressed in delta lat (degrees)
- slat, slon, stime, # sigma for gaussian downweighting with distance in lat, lon (deg), & time (days)
- vfactor=-0.6931, # factor in front of gaussian expression
- missingValue=-9999., # value to mark missing values in interp result
- verbose=VERBOSE, # integer to set verbosity level
- optimization='python'): # Mode of optimization, using 'fortran' or 'cython' or 'python'
- '''Gaussian interpolate in lat, lon, and time to a different lat/lon grid, and over a time window to the center time.
-Bundle of arrays (var) contains a 3D masked variable and coordinate arrays for lat, lon, and time read from netdf/hdf files.
-Returns the 2D interpolated variable (masked) and a status for failures.
- '''
- v = var[varNames[0]][:]
- vmask = np.ma.getmask(v)[:]
- vtime = var[varNames[1]][:]
- lat = var[varNames[2]][:]
- lon = var[varNames[3]][:]
-
- vinterp, vweight, status = \
- gaussInterp_(v, vmask, vtime, lat, lon,
- outlat, outlon, wlat, wlon, slat, slon, stime, vfactor, missingValue)
-
- vinterp = np.ma.masked_where(vweight == 0.0, vinterp)
- return (vinterp, vweight, status)
-
-#@jit(nopython=False)
-def gaussInterp_(var, # variable & mask arrays with dimensions of time, lon, lat
- vmask,
- vtime, # coordinate vectors for inputs
- lat,
- lon,
- outlat, # coordinate vectors for grid to interpolate to
- outlon,
- wlat, wlon, # window of lat/lon neighbors to gaussian weight, expressed in delta lat (degrees)
- slat, slon, stime, # sigma for gaussian downweighting with distance in lat, lon (deg), & time (days)
- vfactor, # factor in front of gaussian expression
- missingValue): # value to mark missing values in interp result
- '''Gaussian interpolate in lat, lon, and time to a different lat/lon grid, and over a time window to the center time.
-Returns the 2D interpolated variable (masked), the weight array, and a status for failures.
- '''
- vinterp = np.zeros( (outlat.shape[0], outlon.shape[0]), dtype=var.dtype ) # interpolated variable, missing values not counted
- vweight = np.zeros( (outlat.shape[0], outlon.shape[0]), dtype=var.dtype ) # weight of values interpolated (can be zero)
- status = 0 # negative status indicates error
-
- ntime = vtime.shape[0]
- nlat = lat.shape[0]
- nlon = lon.shape[0]
-
- noutlat = outlat.shape[0]
- noutlon = outlon.shape[0]
-
- midTime = vtime[int(ntime/2 + 0.5)]
- wlat2 = wlat / 2.
- wlon2 = wlon / 2.
- lat0 = lat[0]
- lon0 = lon[0]
- dlat = lat[1] - lat[0]
- dlon = lon[1] - lon[0]
-
- for i in range(noutlat):
- print(outlat[i], file=sys.stderr)
- for j in range(noutlon):
- if VERBOSE: print('\n(i,j) = %d, %d' % (i, j), file=sys.stderr)
- if VERBOSE: print('\n(outlat,outlon) = %f, %f' % (outlat[i], outlon[j]), file=sys.stderr)
-
- imin = clamp(int((outlat[i] - wlat2 - lat0)/dlat + 0.5), 0, nlat-1)
- imax = clamp(int((outlat[i] + wlat2 - lat0)/dlat + 0.5), 0, nlat-1)
- if imin > imax: (imin, imax) = (imax, imin) # input latitudes could be descending
- if VERBOSE: print('(imin, imax) = %d, %d' % (imin, imax), file=sys.stderr)
- if VERBOSE: print('(minlat, maxlat) = %f, %f' % (lat[imin], lat[imax]), file=sys.stderr)
- jmin = clamp(int((outlon[j] - wlon2 - lon0)/dlon + 0.5), 0, nlon-1)
- jmax = clamp(int((outlon[j] + wlon2 - lon0)/dlon + 0.5), 0, nlon-1)
- if VERBOSE: print('(jmin, jmax) = %d, %d' % (jmin, jmax), file=sys.stderr)
- if VERBOSE: print('(minlon, maxlon) = %f, %f' % (lon[jmin], lon[jmax]), file=sys.stderr)
-# stencil = np.zeros( (ntime, imax-imin+1, jmax-jmin+1) )
-
- for kin in range(ntime):
- for iin in range(imin, imax+1):
- for jin in range(jmin, jmax+1):
- if not vmask[kin,iin,jin]:
- fac = exp( vfactor *
- (((outlat[i] - lat[iin])/slat)**2
- + ((outlon[j] - lon[jin])/slon)**2
- + ((midTime - vtime[kin])/stime)**2))
-# stencil[kin, iin-imin, jin-jmin] = fac
- val = var[kin, iin, jin]
- if VERBOSE > 1: print(kin, iin, jin, vtime[kin], lat[iin], lon[jin], val, fac, val*fac, file=sys.stderr)
-
- vinterp[i,j] = vinterp[i,j] + val * fac
- vweight[i,j] = vweight[i,j] + fac
-
-
- if vweight[i,j] != 0.0:
- vinterp[i,j] = vinterp[i,j] / vweight[i,j]
-# if VERBOSE > 1: print >>sys.stderr, 'stencil:\n', stencil
-# if VERBOSE: print >>sys.stderr, 'stencil max:\n', np.max(np.max(stencil))
-# if VERBOSE: print >>sys.stderr, 'stencil min:\n', np.min(np.min(stencil))
- else:
- vinterp[i,j] = missingValue
-
- return (vinterp, vweight, status)
-
-#@jit( int32(int32,int32,int32), nopython=False)
-def clamp(i, n, m):
- if i < n: return n
- if i > m: return m
- return i
diff --git a/climatology/clim/interp.f b/climatology/clim/interp.f
deleted file mode 100644
index 8977798..0000000
--- a/climatology/clim/interp.f
+++ /dev/null
@@ -1,302 +0,0 @@
-#include "interp.h"
-c $Revision: 2.12 $
-c subprogram interp.f
-c reads hdf file, extract data, call bisum(), calculates
-c and writes out interpolated maps
-
-
- subroutine interp( temp, sst_inter, sst_seq, sl,
- & x, y, z, f,
- & xx, yy, zz, vsum, ixmin, ixmax,
- & iymin, iymax, izmin, izmax, ndatamax,
- & x_length, y_length, imax, jmax, kmax )
-
- integer x_length,y_length
- integer dummy
-
-c-- dimension the arrays
- real*4 temp(x_length,y_length)
- real*4 sl(imax,jmax,kmax)
- real*4 sst_inter(kmax), sst_seq(imax)
- real*4 x(ndatamax),y(ndatamax),z(ndatamax),f(ndatamax)
- real*4 xx(imax),yy(jmax),zz(kmax)
- real*4 vsum(2,imax,jmax,kmax)
- real*4 ULlon, ULlat, LRlon, LRlat
- integer*4 ixmin(ndatamax),ixmax(ndatamax)
- integer*4 iymin(ndatamax),iymax(ndatamax)
- integer*4 izmin(ndatamax),izmax(ndatamax)
-
-c-- hdf data array must be hardwired, no dync alloc :(
- byte in_data( XSIZE, YSIZE )
-
- character*80 plabel
- character*50 infile
- character*30 outfile
- character*150 datafile
- character*30 basename
- character*35 tmpfile
- character*10 mapformat
- character*15 outformat
- character*3 cday
- character*4 cyr
- character*200 zcatcmd
-
- common /parms/ imin,jmin,kmin,
- & xwin2,ywin2,zwin2,
- & xh,yh,zh,
- & dx,dy,dz,
- & xlo,xhi,ylo,yhi,zlo,
- & dxd,dyd
-
- common /fileio/ infile,outfile,istart,
- & iend,mapformat,outformat
-
- common /hdfindex/ icolLeft,icolRight,irowUpper,irowLower
-
- common /basefile/ basename
-
-c-- open output file
- open(unit=20,status='new',form=outformat,file=outfile,
- & err=1000,iostat=ierr )
- plabel='sst interpolation'
-
-c-- initialize vsum to '0's
- do k1=1,2
- do k2=1,kmax
- do k3=1,jmax
- do k4=1,imax
- vsum(k1,k4,k3,k2)=0.
- enddo
- enddo
- enddo
- enddo
-
-c-- initialize arrays of long, lat and time intervals
- do ii=imin,imax
- xx(ii)=xlo+float(ii-1)*dx ! Long to interpolate to
- enddo
- do jj=jmin,jmax
- yy(jj)=ylo+float(jj-1)*dy ! Lat to interpolate to
- enddo
- do kk=kmin,kmax
- zz(kk)=zlo+float(kk-1)*dz ! Time to interpolate to
- enddo
-
- do n=1,ndatamax
- x(n)=0.0
- y(n)=0.0
- z(n)=0.0
- f(n)=0.0
- ixmin(n) = 0
- ixmax(n) = 0
- iymin(n) = 0
- iymax(n) = 0
- izmin(n) = 0
- izmax(n) = 0
- enddo
-
-c-- slope (cal) and offset to convert DN to SST
- cal= 0.15
- offset = 3.0
-
-c-- Open input file list
- open(unit=15,status='old',file=infile,access='direct',
- & recl=RECSIZE,form='formatted',err=1500,iostat=ierr)
-
-c-- read a infile one record at a time and process . . .
- do k=istart,iend
- ipts=0
- read( 15,'(a)',rec=k ) datafile
-c print *, datafile
-
- icompress = 0
-c-- hack to get trailing 'Z' (if compressed)
-c do islash = 150, 1, -1
-c if ( datafile(islash:islash) .eq. 'Z' ) then
-c icompress = 1
-c goto 101
-c endif
-c enddo
-c101 continue
-
-c-- 12/27/99: C call for basename implemented
-c-- variable basename returned via common statement
- call getbase( datafile )
- print *,'\n file: ', basename
-
-c-- cyr and cday parsed from basename
-c-- (e.g., parsed from '1988332h54dd-gdm.hdf')
- cday=basename( 5:7 )
- cyr=basename( 1:4 )
-
-c-- if unix compressed, zcat to a temporary filename
- if( basename(22:22) .eq. 'Z' ) then
- icompress = 1
- tmpfile = '/tmp/' // basename(1:20)
- zcatcmd = 'zcat ' // datafile( 1:len(datafile)-1 )
- & // ' > ' // tmpfile
- call system( zcatcmd )
- datafile = tmpfile
- endif
-
-c-- convert iday character to integer
- read(cday,'(i3)') iday
- read(cyr,'(i4)') iyr
-c write(6,*)'cday = ',cday
-c write(6,*)'cyr = ',cyr
-
-c***> HDF call: d8gimg()
- retn=d8gimg(datafile,in_data,x_length,
- & y_length,dummy)
-
-c-- read hdf DN values and convert to SST
- do i=icolLeft, icolRight
- do j=irowUpper, irowLower
- xlat=-89.75+dyd*float(j-1)
-
-c-- center output in Pacific Ocean
-c if( i .lt. x_length ) ix = i-(x_length/2)
-c if( i .le. (x_length/2) ) ix = i+(x_length/2)
-
-c-- center output in Atlantic (default)
- ix = i
-
-c-- convert signed byte to float
-c if ( in_data(i,j) .lt. 0 .and. abs(xlat) .lt. 70. ) then
- if ( in_data(i,j).lt. 0 ) then
- temp(ix,j)=float( in_data(i,j) ) + 256
- else
- temp(ix,j)=float( in_data(i,j) )
- endif
-
-
-
-C MULTIPLY THE PATHFINDER DIGITAL NUMBER BY THE CALIBRATION NUMBER (0.15)
-C AND ADD THE OFFSET (-3.0) TO GET DEGREES CELSIUS
-
- if ( temp(ix,j).gt.0 ) then
- ipts=ipts+1
- f(ipts)=( cal*temp(ix,j) ) - offset
- x(ipts) = ( dxd*float(ix-1) ) - 180. + dxd/2
- y(ipts) = ( 90. - (dyd*float(j-1)) ) - dyd/2
- z(ipts)=float(iday)+float(iyr-1985)*365.
- if(iyr.gt.1988) z(ipts)=z(ipts)+1
- if(iyr.gt.1992) z(ipts)=z(ipts)+1
- if(iyr.gt.1996) z(ipts)=z(ipts)+1
- if(iyr.gt.2000) z(ipts)=z(ipts)+1
- if(iyr.gt.2004) z(ipts)=z(ipts)+1
- endif
- enddo
- enddo
-
- nfnew=ipts
- print *, ' no of pts in file ',' = ', nfnew
-
-c-- calculate interpolation weights and vsum
-c-- arrays passed directly... a common statement
-c-- does not seem to work.
- call binsum( nfnew, x, y, z, f,
- & xx, yy, zz, vsum,
- & ixmin, ixmax, iymin,
- & iymax, izmin, izmax,
- & imax, jmax, kmax, ndatamax )
-
- if( icompress .eq. 1 ) call system( 'rm -f ' // tmpfile )
-c-- ..... read next hdf file from infile
- enddo
-
-c-- all input files processed; calculate interpolated SST
- do kk=1,kmax
- do jj=1,jmax
- do ii=1,imax
- sl(ii,jj,kk)=0.
- if (vsum(2,ii,jj,kk).gt.0) then
- sl(ii,jj,kk)=vsum(1,ii,jj,kk)/vsum(2,ii,jj,kk)
- endif
- enddo
- enddo
- enddo
-
-c-- write output as map "interleaved" or map "sequential"
-c-- "interleaved" is the original implementation
-c-- both formats preceded by header
-
-c-- "interleaved" refers to each row in the output
-c-- file representing a unique lon, lat position with columns
-c-- of associated sst values (maps)
-c-- i.e. row one: lon1 lat1 sst1 sst2 sst3....lastsst
-c-- row two: lon2 lat1 sst1 sst2 sst3....lastsst
-
-c-- "sequential" refers to each map written as rows and
-c-- columns of lat, lon with the array element representing the
-c-- sst at that geo-position for that map.
-c-- each map is written sequentially in the file
-
-c-- geo-positions of UL and LR corners
- ULlon = -(180 - ( ((icolLeft-1) * dxd) + dxd/2 ))
- ULlat = (90 - ( ((irowUpper-1) * dyd) + dyd/2 ))
- LRlon = -(180 - ( (icolRight * dxd) - dxd/2 ))
- LRlat = (90 - ( (irowLower * dyd) - dyd/2 ))
-
-c-- version number, "f" refers to fortran version
- version = 'f2.9'
-
-c-- write the 3 record header
- if( outformat .eq. 'formatted' ) then
- write(20,'(a)'), plabel
- write(20,*) imax,jmax,kmax
- write(20,*) dx,dy,dz
- write(20,*) ULlon,ULlat,LRlon,LRlat
- elseif( outformat .eq. 'unformatted' ) then
- write(20) imax,jmax,kmax
- write(20) dx,dy,dz
- write(20) ULlon,ULlat,LRlon,LRlat
- endif
-
- if( mapformat .eq. 'interleave' ) then
- print *, '\n map output is interleave'
- do jj=1,jmax
- do ii=1,imax
- do kk=1,kmax
- sst_inter(kk)=sl(ii,jj,kk)
- enddo
- if( outformat .eq. 'formatted' ) then
- write(20,*) ii,jj,sst_inter
- else
- write(20) ( sst_inter(i), i=1,kmax )
- endif
- enddo
- enddo
-
- else if( mapformat .eq. 'sequential' ) then
- print *, '\n map output is sequential'
- do kk=1,kmax
- do jj=1,jmax
- do ii=1,imax
- sst_seq(ii)=sl(ii,jj,kk)
- enddo
- if( outformat .eq. 'formatted' ) then
- write(20,*) jj,kk,sst_seq
- else
- write(20) ( sst_seq(i), i=1,imax )
- endif
- enddo
- enddo
- endif
-
- print *, '\ndone. . . '
- close( 15,status='keep',err=2000,iostat=ierr )
- close( 20,status='keep',err=2500,iostat=ierr )
- stop
-
- 1000 print *, 'Error opening output: ', outfile, 'error num: ', ierr
- goto 102
- 1500 print *, 'Error opening input: ', infile, 'error num: ', ierr
- goto 102
- 2000 print *, 'Error closing input: ', infile, 'error num: ', ierr
- goto 102
- 2500 print *, 'Error closing output: ', outfile, 'error num: ', ierr
- goto 102
- 102 continue
-
- end
diff --git a/climatology/clim/jobClimatology2.py b/climatology/clim/jobClimatology2.py
deleted file mode 100644
index 499a4fd..0000000
--- a/climatology/clim/jobClimatology2.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# jobClimatology2.py -- mrjob class for computing climatologies by gaussian interpolation
-#
-
-from mrjob.job import MRJob
-from .climatology2 import climByAveragingPeriods
-
-
-class MRClimatologyByGaussInterp(MRJob):
-
- def mapper(self, _, line):
- yield "chars", len(line)
- yield "words", len(line.split())
- yield "lines", 1
-
- def reducer(self, key, values):
- yield key, sum(values)
-
-
-if __name__ == '__main__':
- MRWordFrequencyCount.run()
-
diff --git a/climatology/clim/jobTest.py b/climatology/clim/jobTest.py
deleted file mode 100644
index 15e250f..0000000
--- a/climatology/clim/jobTest.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from mrjob.job import MRJob
-
-
-class MRWordFrequencyCount(MRJob):
-
- def mapper(self, _, line):
- yield "chars", len(line)
- yield "words", len(line.split())
- yield "lines", 1
-
- def reducer(self, key, values):
- yield key, sum(values)
-
-
-if __name__ == '__main__':
- MRWordFrequencyCount.run()
-
diff --git a/climatology/clim/orig/C/README b/climatology/clim/orig/C/README
deleted file mode 100644
index 7e50ce0..0000000
--- a/climatology/clim/orig/C/README
+++ /dev/null
@@ -1,6 +0,0 @@
-2 Mar 00
-
-`test1-100day' was produced by gaussinterp (C version) using /sst/vol7/PATHFINDER/avhrrlists/test.list
-and /sst/vol7/PATHFINDER/parmfiles/interp.test.parms. First 100 files in test.list were processed.
-
-
diff --git a/climatology/clim/orig/C/binsum.c b/climatology/clim/orig/C/binsum.c
deleted file mode 100644
index 724245c..0000000
--- a/climatology/clim/orig/C/binsum.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* $Revision: 1.6 $ */
-/* calculate interpolation weights and vsum */
-
-#include <math.h>
-#include "interp.h"
-
-#define min(x,y) ( (x) < (y) ? (x) : (y) )
-#define max(x,y) ( (x) > (y) ? (x) : (y) )
-#define XFAC -0.6931
-
- binsum( int nf ) {
-
- register int i, ii, jj, kk, n;
- int lon_180_index;
- float fac;
- float long_diff_deg, long_diff_rad;
- float west_long_endpoint, east_long_endpoint;
-
- /* index in array xx (longitude interp bins) for long. 180
- in extended portion of array beyond imax */
- if( all_zones )
- lon_180_index = parms.imax + long_extend / 2;
-
- /* determine spatial/temporal interpolation bins for every point */
- for( n=0; n < nf; n++ ) {
-
- /* longitude bins ranges */
- /* Check for the discontinuities along long. 180. If crossing long. 180
- set ixmin and ixmax to reflect indicies in "extended" portion of
- longitude bins in array xx */
-
- west_long_endpoint = x[n] - parms.xwin2;
- east_long_endpoint = x[n] + parms.xwin2;
-
- ixmin[n] = (int) floorf( (x[n]-parms.xwin2-parms.xlo)/parms.dx );
- ixmax[n] = (int) floorf( (x[n]+parms.xwin2-parms.xlo)/parms.dx );
-
- /* crossing west of 180 in search and global interp */
- if( west_long_endpoint < -180. && east_long_endpoint > -180. &&
- all_zones ) {
-
- ixmin[n] = lon_180_index - abs( ixmin[n] );
- ixmax[n] = lon_180_index + ixmax[n];
-/* printf( " ixmin, ixmax %d %d in extended array\n", ixmin[n], ixmax[n] ); */
- }
-
- /* crossing east of 180 in search and global interp */
- else if( west_long_endpoint < 180. && east_long_endpoint > 180. &&
- all_zones ) {
-
- ixmin[n] = lon_180_index - ( parms.imax - ixmin[n] );
- ixmax[n] = lon_180_index + abs( parms.imax - ixmax[n] );
-/* printf( "ixmin, ixmax %d %d in extended array\n", ixmin[n], ixmax[n] ); */
- }
-
- /* no crossing of 180 in search */
- else {
- ixmin[n] = max( (int) floorf( (x[n]-parms.xwin2-parms.xlo)/parms.dx ), 0 );
- ixmax[n] = min( (int) floorf( (x[n]+parms.xwin2-parms.xlo)/parms.dx ), parms.imax - 1 );
- }
-
- /* lat bin ranges */
- iymin[n] = max( (int) floorf( (y[n]-parms.ywin2-parms.ylo)/parms.dy ), 0 );
- iymax[n] = min( (int) floorf( (y[n]+parms.ywin2-parms.ylo)/parms.dy ), parms.jmax - 1 );
-
- /* temporal bin ranges */
- izmin[n] = max( (int) floorf( (z[n]-parms.zwin2-parms.zlo)/parms.dz ), 0 );
- izmax[n] = min( (int) floorf( (z[n]+parms.zwin2-parms.zlo)/parms.dz ), parms.kmax - 1 );
-
- }
-
-
- /* weight the SST and sum */
- for( n=0; n < nf; n++ ) {
- for( kk=izmin[n]; kk <= izmax[n]; kk++ ) {
- for( jj=iymin[n]; jj <= iymax[n]; jj++ ) {
- for( i=ixmin[n]; i <= ixmax[n]; i++ ) {
-
- /* Derive the guassian weights. Because
- longitudes can cross 180, e.g., the ranges
- can be 175 to -175, use the acos(cos( long_diff ))) */
- long_diff_deg = x[n] - xx[i];
- long_diff_rad = acos( cos(deg2rad(long_diff_deg)) );
-
- fac = exp( XFAC * ( pow( (rad2deg(long_diff_rad)
- /parms.xh),2 )
- + pow( ((y[n]-yy[jj])/parms.yh),2 )
- + pow( ((z[n]-zz[kk])/parms.zh),2 ) ));
-
- /* if crossing long. 180 must drop weighted SST
- in correct longitude bin between 0 and imax */
-
- /* range crossing east of 180 */
- if( i >= lon_180_index && all_zones ) {
- ii = i - lon_180_index;
-
-/* printf(" orig bin is %d, correct bin is %d\n", i, ii ); */
- }
- /* range crossing west of 180 */
- else if( (i < lon_180_index && i >= parms.imax) && all_zones ) {
- ii = i - long_extend / 2;
-
-/* printf(" orig bin is %d, correct bin is %d\n", i, ii ); */
- }
- /* no crossing */
- else {
- ii = i;
- }
-
- vsum[0][ii][jj][kk] += f[n]*fac;
- vsum[1][ii][jj][kk] += fac;
- }
- }
- }
- }
- }
-
-
-
-
-
-
-
-
-
diff --git a/climatology/clim/orig/C/clouderosion.c b/climatology/clim/orig/C/clouderosion.c
deleted file mode 100644
index 6dec692..0000000
--- a/climatology/clim/orig/C/clouderosion.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/* $Revision:$ */
-/* Apply the Casey cloud erosion to the satellite data.
- This algorithm flags as cloudy all data with one
- nearest neighbor of a Pathfinder flagged cloud pixel. */
-
-#include "interp.h"
-#include "hdf.h"
-
-clouderosion( float in_data, int i, int j ) {
- int row, col;
-
- /* for each image pixel loop over
- nearest neighbors; if any neightbor is
- a cloud, set temp[j][i] to cloud value (0)
- */
- for( col = i-1; col <= i+1; col++ ) {
- for( row = j-1; row <= j+1; row++ ) {
- printf( "col is %d, row is %d\n", col, row );
- /* make sure we are searching within image array bounds */
- if( col >= 0 && col < parms.x_length &&
- row >= 0 && row < parms.y_length ) {
- printf( "valid col is %d, valid row is %d\n\n", col, row );
- if( in_data[col][row] == 0 ) {
- temp[j][i] = 0.;
- printf( "in_data[%d][%d] is %c\n", col, row, in_data[col][row] );
- printf( " setting temp[%d][%d] to zero\n\n\n", j,i );
- break;
- }
- }
- }
- }
-
-}
diff --git a/climatology/clim/orig/C/gaussinterp.readme b/climatology/clim/orig/C/gaussinterp.readme
deleted file mode 100644
index 4a0e017..0000000
--- a/climatology/clim/orig/C/gaussinterp.readme
+++ /dev/null
@@ -1,159 +0,0 @@
-05 Oct 00
-ed armstrong NASA/JPL/Caltech
-
-`gaussinterp' is a program designed to spatially and temporally
-interpolate SST pathfinder data using gaussian methodology. This
-description is for the C version. To run the program from the command
-line:
-
-% gaussinterp <infile> <beg_rec> <parmfile> <outfile>
-
-where
- infile = file containing list of files to process
- (ascii; fixed length records; temporally sorted from
- old to recent)
- beg_rec = first record (file) in infile processed
- parmfile = input parameter file
- outfile = interpolated output file (ascii or binary)
-
-
-`infile' must be ascii with a fixed record length of 150 bytes. Use
-`pad.pl' to convert variable or other fixed record length files to
-150. `infile' records should be in monotonically ascending in time
-(year and julian day). beg_rec is the record number of the temporal
-start of the interpolation. The day for the file corresponding to this
-record number is also the first map center date. The next map center is
-'first_map_center' + time_step (parameter dz; see parameter file
-description) and so forth. `outfile' can be either an ascii or binary
-file, sequential or interleaved format; see below for more
-information. `parmfile' is an ascii file read by the program that
-contains many interpolation and output parameters.
-
-
-** Parameter file description **
-
-~~~~ example of parameter file: ~~~~~
-200000
-720 360
-10
-3.0 3.0 20.0
-1.0 1.0 10.0
-1.0 1.0 8.0
--180. 180.
--90. 90.
-sequential
-unformatted
-1
-1
-6
-~~~~ end example ~~~~
-
-record 1: ndatamax = maximum points in each file
- 2: xlength, y_length = HDF data array: cols and rows
- 3: kmax = number of "maps" or time "steps"
- 4: xwin2, ywin2, zwin2 = longitude, latitude "search"
- range (decimal degrees), time search range (days)
- 5: xh, yh, zh = e folding scaling for long, lat (decimal
- degrees) and time (days)
- 6: dx, dy, dz = longitude, latitude interpolation
- step (decimal degrees), and
- time step (days)
- 7: xlo, xhi = minimum, maximum longitude (-180 to 180 decimal degrees)
- 8: ylo, yhi = minimum, maximum latitude (-90 to 90 decimal degrees)
- 9: 'interleave' or 'sequential' map format
- 10: 'formatted' (ascii) or 'unformatted' (binary) output file
- 11: cloud erosion flag: 1=cloud erosion; 0=no cloud erosion
- 12: all pixel data flag: 1="all pixel data"; 0="best pixel" data
- 13: quality flag value: quality flag value (0-7) to use for "all pixel" data;
- ignored if "best pixel" data used
-
-
-Some notes on the parameter file:
-
- * ndatamax should be x_length*y_length but in reality is much
- less since much of the data are missing due to clouds.
-
- * the user can subset into the image array by choosing proper
- xlo, xhi, ylo, yhi. xlo must always be less than xhi; the same
- for ylo and yhi.
-
- * interleave format refers to each row in the output file
- representing a unique lon, lat position with columns of
- associated sst values (maps)
- i.e. row one: lon1 lat1 sst1 sst2 sst3....lastsst
- row two: lon2 lat1 sst1 sst2 sst3....lastsst
-
- * sequential format refers to each map written as rows and
- columns of lat, lon with the array element representing the sst
- at that geo-position for that map. Each map is written
- sequentially in the output file.
-
- * generally unformatted (binary) output should be chosen as
- this results in a smaller output file size. Using binary format
- each pixel is written as a 1 byte char (0-255).
-
- * cloud erosion refers to the Casey cloud erosion algorithm
- used in the NSIPP (Casey) Pathfinder climatology.
-
-
-** Header description **
-
-For each output format a header is written:
- ASCII format:
- record 1: some text
- record 2: imax, jmax, kmax
- imax = width of output in pixels
- jmax = height of output in pixels
- kmax = number of "maps" or images
- record 3: xwin2, ywin2, zwin2
- spatial and temporal search "radii"
- in decimal degrees and days
- record 4: xh, yh, zh
- e folding scales in decimal degrees
- record 5: dx,dy,dz
- interpolation intervals
- in decimal degrees
- record 6: ULx, ULy, LRx, LRy
- upper left and lower right longitude
- and latitudes in decimal degrees
- record 7: date.startdate, date.enddate
- first and last map center dates in
- YYYYDDD (e.g., 1995010)
-
- BINARY (C) [word is 4 bytes, types are indicated]:
- word 1 [char]: magic number
- words 2-4 [int]: imax, jmax, kmax
- words 5-7 [float]: xwin2, ywin2, zwin2
- words 8-10 [float]: xh, yh, zh
- words 11-13 [float]: dx,dy,dz
- words 14-17 [float]: ULx, ULy, LRx, LRy
- word 18 [int]: first map center date
- word 19 [int]: last map center date
- [total header size is 76 bytes]
-
-** Additional information **
-
-- The parameter file can be edited to change/update all entries but
-care must be taken to avoid typos. The program is compiled for a fixed
-record length for the input datafile. These can be modified in
-`interp.h' and the program recompiled.
-
-- In the case of global interpolation where xlo=-180 and xhi=180, the
-program will correctly interpolate across longitude 180 East (all_zones
-is true). That is, there is no "cutoff" in the interpolation if the
-search window for a particular satellite pixel crosses this longitude.
-In all other cases this is not so. For example, if xlo=-180 and
-xhi=-150, interpolations in the vicinity of longitude -180 are "cutoff"
-at this boundary, i.e., points west of -180 do not contribute to the
-interpolation. Since this is a regional subset, the interpolations are
-also "cutoff" in the vicinity of longitude -150 (data east of -150 do
-not contribute). By extension this is also true for a subset in
-latitude.
-
-- The first line in the usage dump (type gaussinterp with no arguments)
-will specify if the executable is from Fortran or C code.
-
-- Sample parameter files can be found in
-/sst/vol7/PATHFINDER/interp/parmfiles . Datafile lists can be found in
-/sst/vol7/PATHFINDER/interp/avhrrlists .
-
diff --git a/climatology/clim/orig/C/gaussinterp_C_code.tar b/climatology/clim/orig/C/gaussinterp_C_code.tar
deleted file mode 100644
index 5db4b0e..0000000
Binary files a/climatology/clim/orig/C/gaussinterp_C_code.tar and /dev/null differ
diff --git a/climatology/clim/orig/C/interp.c b/climatology/clim/orig/C/interp.c
deleted file mode 100644
index c057875..0000000
--- a/climatology/clim/orig/C/interp.c
+++ /dev/null
@@ -1,448 +0,0 @@
-/* $Revision: 1.20 $ */
-
-/* SYNOPSIS
- called by setupinterp(). This subroutine reads each
- HDF file, passes data to binsum() for interpolation
- calculation and writes output image(s).
-*/
-
-#include <stdio.h>
-#include <string.h>
-#include <libgen.h>
-#include <hdf.h>
-#include "interp.h"
-
-/* C binary file id = 3890345459 (0xE7E1F5F3) */
-#define MAGIC 3890345459L
-
- interp() {
- uint8 in_data[parms.y_length][parms.x_length];
- uint8 quality_data[parms.y_length][parms.x_length];
- uint8 aerosol_corr[parms.y_length][parms.x_length];
-
- intn retn, haspal;
- int32 width, height;
-
- float ULlon, ULlat, LRlon, LRlat;
- float cal, offset, xlat;
- int k1,k2,k3,k4,ii,jj,kk,n,i,j,k;
- int ipts, iyr, iday;
- int jj_reverse;
- int row, col;
- unsigned long int magic_num;
-
- FILE *fid_infile, *fid_outfile, *fid_datafile;
- FILE *fid_aerosol_list, *fid_aerosol_file;
-
- char plabel[40];
- char datafile[RECSIZE];
- char aerosol_file[200];
- char *basefile;
- char cday[3];
- char cyr[4];
- char tmpstr[22];
- char aerosol_tmpstr[30];
- char tmpfile[35];
- char aerosol_tmpfile[45];
- char zcatcmd[200];
- char rmcmd[45];
- char compress[1];
- char version[4];
- char sstDN;
- /* short int sstDN; */
-
- /* open output file */
- fid_outfile = fopen( fileio.outfile, "w" );
-
-
- /* Set arrays of interpolation bins.
- Long and lat bin centers set to 1/2
- interpolation step */
-
- /* Long to interpolate to:
- 1st for loop sets interpolation bins
- between the ranges of xlo to xhi.
- If all_zones is true, 2nd and 3rd for
- loops set bins for those "extended" interpolations
- that must cross longitude 180. */
- for( ii=parms.imin; ii < parms.imax; ii++ )
- xx[ii] = parms.xlo + (float)(ii)*parms.dx + parms.dx/2;
-
- if( all_zones ) {
- /* west of long. 180 */
- for( ii=0; ii < long_extend/2; ii++ )
- xx[ii+parms.imax] = 180 - (long_extend/2 * parms.dx) + (float)(ii)*parms.dx
- + parms.dx/2;
-
- /* east of long. 180; sets center point */
- for( ii=0; ii <= long_extend/2; ii++ )
- xx[ii+parms.imax+long_extend/2] = -180 + (float)(ii)*parms.dx
- + parms.dx/2;
- }
-
- /* for( ii = 0; ii <= parms.imax + long_extend; ii++)
- printf( " ii: %d xx[ii] = %f \n", ii, xx[ii] ); */
-
-
- /* Lat to interpolate to */
- for( jj=parms.jmin; jj < parms.jmax; jj++ )
- yy[jj] = parms.ylo + (float)(jj)*parms.dy + parms.dy/2;
-
- /* Time to interpolate to */
- for( kk=parms.kmin; kk < parms.kmax; kk++ )
- zz[kk] = parms.zlo + (float)(kk)*parms.dz;
-
- /* slope (cal) and offset to convert DN to SST */
- cal = 0.15;
- offset = 3.0;
-
- /* open input data file list */
- if( (fid_infile = fopen( fileio.infile, "r" )) == NULL ){
- printf( "\nCan not open %s\n", fileio.infile );
- exit(1);
- }
-
- /* position to proper record in file */
- fseek( fid_infile, ( (fileio.istart-1)*RECSIZE ), 0 );
-
-
-
- /* open input aerosol file list if necessary */
- if(aerosol) {
- if( (fid_aerosol_list = fopen( fileio.aerosol_list, "r" )) == NULL ){
- printf( "\nCan not open %s\n", fileio.aerosol_list );
- exit(1);
- }
- /* position to proper record in file */
- fseek( fid_aerosol_list, ( (fileio.istart-1)*RECSIZE ), 0 );
-
- }
-
-
- /* read infile one record at a time and process . . . */
- for( k=fileio.istart; k <= fileio.iend; k++ ){
-
- ipts = 0;
- strcpy( compress, " " );
- strcpy( tmpstr, " " );
-
- fgets( datafile, sizeof(datafile)+1, fid_infile );
-
- /* strip off newline if necessary */
- if( datafile[ strlen(datafile) ] == '\n' ){
- datafile[ strlen(datafile) ] = '\0';
- }
- /* printf("datafile is %s \n", datafile ); */
-
- /* check to make sure datafile actually exists
- if not ..... quit program */
- fid_datafile = fopen( datafile, "r" );
- if( fid_datafile == NULL ) {
- printf( "could not open:\n" );
- printf( "%s \n", datafile );
- printf( ". . . quitting program \n" );
- exit(2);
- }
- fclose( fid_datafile );
-
- basefile = basename( &datafile[0] );
- printf( "\nfile: %s\n", basefile );
-
- /* e.g, parse year and jul day from 1985004h54dd-gdm.hdf */
- strcpy( tmpstr, basefile );
- strncpy( cyr, &tmpstr[0], 4 );
- iyr = atoi( cyr );
- strncpy( cday, &tmpstr[4], 3 );
- iday = atoi( cday );
- /* printf(" iday is %d\n", iday );
- printf("iyr is %d\n", iyr );
- */
-
- /* if unix compressed, zcat to a temporary filename */
- strncpy( compress, &tmpstr[21], 1 );
- if( strcmp( compress, "Z" ) == 0 ) {
- strcpy( tmpstr, basefile );
- strcpy( tmpfile, "/tmp/" );
- strncat( tmpfile, &tmpstr[0], 20 );
-
- strcpy( zcatcmd, "zcat ");
- strcat( zcatcmd, datafile );
- strcat( zcatcmd, " > " );
- strcat( zcatcmd, tmpfile );
-
- (void) system( zcatcmd );
- strcpy( datafile, tmpfile );
- }
-
- /* do the same for the aerosol file if necessary */
- if( aerosol ) {
- strcpy( compress, " " );
- strcpy( aerosol_tmpstr, " " );
-
- fgets( aerosol_file, sizeof(aerosol_file)+1, fid_aerosol_list );
- /* printf("aerosol_file is %s \n", aerosol_file ); */
-
- /* strip off newline if necessary */
- if( aerosol_file[ strlen(aerosol_file) ] == '\n' ){
- aerosol_file[ strlen(aerosol_file) ] = '\0';
- }
-
-
- basefile = basename( &aerosol_file[0] );
- strcpy( aerosol_tmpstr, basefile );
- printf( "aerosol file: %s\n", basefile );
-
-
- /* if unix compressed, zcat to a temporary filename */
- strncpy( compress, &aerosol_tmpstr[27], 1 );
- if( strcmp( compress, "Z" ) == 0 ) {
- strcpy( aerosol_tmpstr, basefile );
- strcpy( aerosol_tmpfile, "/tmp/" );
- strncat( aerosol_tmpfile, &aerosol_tmpstr[0], 26 );
-
- strcpy( zcatcmd, "zcat ");
- strcat( zcatcmd, aerosol_file );
- strcat( zcatcmd, " > " );
- strcat( zcatcmd, aerosol_tmpfile );
-
- (void) system( zcatcmd );
- strcpy( aerosol_file, aerosol_tmpfile );
- }
- }
-
- /* read the aerosol file into an array */
- if( aerosol ) {
- fid_aerosol_file = fopen( aerosol_file, "r" );
- fread( aerosol_corr, sizeof(uint8), parms.y_length * parms.x_length, fid_aerosol_file );
- fclose( fid_aerosol_file );
- }
-
- /* read HDF SST data dimensions */
- retn = DFR8getdims( datafile, &width, &height, &haspal );
- /* printf( "HDF width and height: %d %d \n", width, height ); */
-
- if( width != parms.x_length || height != parms.y_length ) {
- printf( "\nbounds mismatch after reading HDF file dimensions !!\n" );
- printf( "width: %d, parms.x_length: %d \n", width, parms.x_length );
- printf( "height: %d, parms.y_length: %d \n", height, parms.y_length );
- printf( "skipping %s . . . \n\n", datafile );
- continue;
- }
-
- /* ------------------------------------------------- */
- /* Read the SST image of the HDF file.
- HDF call: DFR8getimage() */
- /* ------------------------------------------------- */
-
- retn=DFR8getimage(datafile, (VOIDP)in_data, parms.x_length,
- parms.y_length, NULL);
-
- /* ------------------------------------------------- */
- /* If "all pixel" data choosen read the quality
- flag image */
- /* ------------------------------------------------- */
-
- if( sst_type )
- retn=DFR8getimage(datafile, (VOIDP)quality_data,
- parms.x_length, parms.y_length, NULL);
-
-
-
- /* ------------------------------------------------- */
- /* Read hdf file DN values and convert to SST.
- If "all pixel" data read only those pixels
- with a quality flag greater than or equal to
- quality_flag */
- /* ------------------------------------------------- */
-
- for( i=hdf.icolLeft; i <= hdf.icolRight; i++ ) {
- for( j=hdf.irowUpper; j <= hdf.irowLower; j++ ) {
-
- /* only conider aerosol correction greater than 5 cnt */
- if( aerosol && aerosol_corr[j][i] > 5 && in_data[j][i] > 0 )
- in_data[j][i] = in_data[j][i] + aerosol_corr[j][i];
-
- /* perform Casey cloud erosion on non-zero SST data */
- /* if nearest neighbor is a cloud, set pixel to cloud (0) */
- if( in_data[j][i] > 0 && cloud_erosion ) {
- for( col = i-1; col <= i+1; col++ ) {
- for( row = j-1; row <= j+1; row++ ) {
- /* make sure we are searching within
- image array bounds */
- if( col >= 0 && col < parms.x_length &&
- row >= 0 && row < parms.y_length &&
- in_data[row][col] == 0 ) {
- in_data[j][i] = 0;
- goto end_erosion;
- }
- }
- }
- }
- end_erosion:
-
-
- /* if all_pixel (sst_type = 1) filter for quality level,
- or if best_sst accept data values greater than 0 */
-
- /*printf( "quality flag is %d \n", quality_flag);
- printf (" sst_type is %d \n", sst_type);
- */
-
- if( (sst_type && quality_data[j][i] >= quality_flag) ||
- (!sst_type && in_data[j][i] > 0) ) {
- /* printf (" quality_data = %d \n", quality_data[j][i]); */
-
- f[ipts] = cal * (float)in_data[j][i] - offset;
- x[ipts] = -180. + parms.dxd * (float)(i) + parms.dxd/2;
- y[ipts] = 90. - parms.dyd * (float)(j) - parms.dyd/2;
- z[ipts] = (float)( iday+( (iyr-1985)*365 ) );
-
- if(iyr > 1988) z[ipts]=z[ipts]+1;
- if(iyr > 1992) z[ipts]=z[ipts]+1;
- if(iyr > 1996) z[ipts]=z[ipts]+1;
- if(iyr > 2000) z[ipts]=z[ipts]+1;
- if(iyr > 2004) z[ipts]=z[ipts]+1;
- if(iyr > 2008) z[ipts]=z[ipts]+1;
- ipts++;
- }
- }
- }
-
- /* calculate interpolation weights and vsum */
-
- printf(" num of pts in file: %d \n", ipts );
- binsum( ipts );
-
- if( strcmp( compress, "Z" ) == 0 ) {
- strcpy( rmcmd, "rm -f " );
- strcat( rmcmd, tmpfile );
- system( rmcmd );
- if( aerosol ) {
- strcpy( rmcmd, "rm -f " );
- strcat( rmcmd, aerosol_tmpfile );
- system( rmcmd );
- }
- }
- /* ..... read next hdf file from infile */
-
- }
- fclose( fid_infile );
- if( aerosol )
- fclose( fid_aerosol_list );
-
- /* all input files processed; calculate interpolated SST maps */
- for( kk=0; kk < parms.kmax; kk++ ) {
- for( jj=0; jj < parms.jmax; jj++ ) {
- for( ii=0; ii < parms.imax; ii++ ) {
- if ( vsum[1][ii][jj][kk] > 0 ) {
-
- /* reverse jj indicies to flip N to S */
- jj_reverse = parms.jmax - jj - 1;
- sl[ii][jj_reverse][kk] =
- vsum[0][ii][jj][kk] / vsum[1][ii][jj][kk];
- }
- }
- }
- }
-
- /* write output as map "interleaved" or map "sequential".
- both formats preceded by header.
- see gaussinterp.readme for more info.
- */
-
- /* geo-positions of UL and LR corners */
- ULlon = parms.xlo + parms.dx/2;
- ULlat = parms.yhi - parms.dy/2;
- LRlon = parms.xhi - parms.dx/2;
- LRlat = parms.ylo + parms.dy/2;
-
- /* version number, "c" refers to C version */
- strcpy( version, "c1.3" );
- strcpy( plabel, "sst interpolation");
-
- /* write the header */
- if( strcmp( fileio.outformat, "formatted" ) == 0 ) {
- fprintf( fid_outfile, "%s -- %s\n ", version, plabel );
- fprintf( fid_outfile, "%d %d %d\n",
- parms.imax,parms.jmax,parms.kmax );
- fprintf( fid_outfile, "%f %f %f\n",
- parms.xwin2,parms.ywin2,parms.zwin2 );
- fprintf( fid_outfile, "%f %f %f\n",
- parms.xh,parms.yh,parms.zh );
- fprintf( fid_outfile, "%f %f %f\n",
- parms.dx,parms.dy,parms.dz );
- fprintf( fid_outfile, "%f %f %f %f\n",
- ULlon,ULlat,LRlon,LRlat );
- fprintf( fid_outfile, "%ld %ld\n",
- date.startdate, date.enddate );
- }
- else if( strcmp( fileio.outformat, "unformatted" ) == 0 ) {
- magic_num = MAGIC;
- fwrite( &magic_num, sizeof(magic_num),1,fid_outfile );
- fwrite( &parms.imax, sizeof(parms.imax),1,fid_outfile );
- fwrite( &parms.jmax, sizeof(parms.jmax),1,fid_outfile );
- fwrite( &parms.kmax, sizeof(parms.kmax),1,fid_outfile );
- fwrite( &parms.xwin2, sizeof(parms.xwin2),1,fid_outfile );
- fwrite( &parms.ywin2, sizeof(parms.ywin2),1,fid_outfile );
- fwrite( &parms.zwin2, sizeof(parms.zwin2),1,fid_outfile );
- fwrite( &parms.xh, sizeof(parms.xh),1,fid_outfile );
- fwrite( &parms.yh, sizeof(parms.yh),1,fid_outfile );
- fwrite( &parms.zh, sizeof(parms.zh),1,fid_outfile );
- fwrite( &parms.dx, sizeof(parms.dx),1,fid_outfile );
- fwrite( &parms.dy, sizeof(parms.dy),1,fid_outfile );
- fwrite( &parms.dz, sizeof(parms.dz),1,fid_outfile );
- fwrite( &ULlon, sizeof(ULlon),1,fid_outfile );
- fwrite( &ULlat, sizeof(ULlat),1,fid_outfile );
- fwrite( &LRlon, sizeof(LRlon),1,fid_outfile );
- fwrite( &LRlat, sizeof(LRlat),1,fid_outfile );
- fwrite( &date.startdate, sizeof(date.startdate),1,fid_outfile);
- fwrite( &date.enddate, sizeof(date.enddate),1,fid_outfile);
- /* binary header is 76 bytes */
- }
-
- /* write the maps */
- if( strcmp( fileio.mapformat, "interleave" ) == 0 ) {
- printf( "\n map output is interleave\n" );
- for( jj=0; jj < parms.jmax; jj++ ) {
- for( ii=0; ii < parms.imax; ii++ ) {
- for( kk=0; kk < parms.kmax; kk++ ) {
- sst_inter[kk] = sl[ii][jj][kk];
-
- /* scale sst back to 0-255 DNs */
- if( sst_inter[kk] != 0 )
- sstDN = (char) nint( (sst_inter[kk] + offset) / cal );
- else
- sstDN = 0; /* O's in array sst_inter are missing data */
-
- if( strcmp( fileio.outformat, "formatted" ) == 0 )
- fprintf( fid_outfile, "%d ", sstDN );
- else
- fwrite( &sstDN, sizeof(char),1,fid_outfile );
-
- }
- }
- }
- } else if( strcmp( fileio.mapformat, "sequential" ) == 0 ) {
- printf( "\n map output is sequential\n" );
- for( kk=0; kk < parms.kmax; kk++ ) {
- for( jj=0; jj < parms.jmax; jj++ ) {
- for( ii=0; ii < parms.imax; ii++ ) {
- sst_seq[ii] = sl[ii][jj][kk];
-
- /* scale sst back to 0-255 DNs */
- if( sst_seq[ii] != 0 )
- sstDN = (char) nint( (sst_seq[ii] + offset) / cal );
- else
- sstDN = 0; /* O's in array sst_seq are missing data */
-
- if( strcmp( fileio.outformat, "formatted" ) == 0 )
- fprintf( fid_outfile, "%d ", sstDN );
- else
- fwrite( &sstDN, sizeof(char),1,fid_outfile );
-
- }
- }
- }
- }
- fclose( fid_outfile );
-}
diff --git a/climatology/clim/orig/C/makefile b/climatology/clim/orig/C/makefile
deleted file mode 100644
index ab004b3..0000000
--- a/climatology/clim/orig/C/makefile
+++ /dev/null
@@ -1,33 +0,0 @@
-# makefile for guassinterp (C version).
-
-# C compiler
-CC = cc
-#HDF = /usr/local/hdf
-HDF = /usr/local/hdf4.1r3-n32
-
-CFLAGS = -n32 -I$(HDF)/include -O -w
-#LIBS = -L$(HDF)/lib -lmfhdf -ldf -ljpeg -lz -lgen -lm
-LIBS = -L$(HDF)/lib -ldf -ljpeg -lz -lgen -lm
-BIN = /sst/vol7/PATHFINDER/bin/sgi
-PGM = gaussinterp
-
-SRC = setupinterp.c interp.c binsum.c
-OBJ = setupinterp.o interp.o binsum.o
-
-$(PGM): $(OBJ)
- $(CC) $(CFLAGS) -o $(PGM) $(OBJ) $(LIBS)
-
-setupinterp.o:
- $(CC) $(CFLAGS) $(DEFINE) -c setupinterp.c
-interp.o:
- $(CC) $(CFLAGS) $(DEFINE) -c interp.c
-binsum.o:
- $(CC) $(CFLAGS) $(DEFINE) -c binsum.c
-
-
-install: $(PGM)
- cp $(PGM) $(BIN)
-
-clean:
- rm -f $(OBJ) $(PGM)
-
diff --git a/climatology/clim/orig/C/setupinterp.c b/climatology/clim/orig/C/setupinterp.c
deleted file mode 100644
index 9ca67b3..0000000
--- a/climatology/clim/orig/C/setupinterp.c
+++ /dev/null
@@ -1,431 +0,0 @@
-/* NAME gaussinterp (C version)
- 16 Jan 00 earmstrong NASA/JPL
- $Revision: 1.16 $
-
- DESCRIPTION
- Gaussian interpolation program modified to map SST (HDF) data.
- This program is a C port of the fortran version of gaussinterp.
-
- SYNOPSIS
- setupinterp() allocates space for arrays, calls interp()
- which does the interpolation
-
- USAGE
- % gaussinterp <infile> <beg_rec> <parmfile> <outfile> */
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <libgen.h>
-#include <math.h>
-#include "interp.h"
-/* see interp.h for global structs and arrays */
-
-main( int argc, char *argv[] ) {
-
- int iday, iyr, calloc_bytes, rec_step;
- long max_infile_size;
-
- int value;
- FILE *fid_infile, *fid_parm;
-
- char datafile[RECSIZE];
- char *basefile;
- char *parmfile;
- char *startrec, *endrec;
- char cday[3], cyr[4], cdate[7];
- char *cstartdate;
- char *cenddate;
- char tmpline[80];
-
-/* check command line arguments */
- if( argc != 5 && argc != 6 ) {
- usage();
- exit(1);
- }
- fileio.infile = argv[1];
- startrec = argv[2];
- parmfile = argv[3];
- fileio.outfile = argv[4];
- if( argc == 6 )
- fileio.aerosol_list = argv[5];
-
- /* open parameter file and read line by line */
- if( (fid_parm = fopen( parmfile, "r" )) == NULL ){
- printf( "\nCan not open %s\n", parmfile );
- exit(1);
- }
-
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%d", &parms.ndatamax );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%d %d", &parms.x_length, &parms.y_length );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%d", &parms.kmax );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%f %f %f", &parms.xwin2, &parms.ywin2, &parms.zwin2 );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%f %f %f", &parms.xh, &parms.yh, &parms.zh );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%f %f %f", &parms.dx, &parms.dy, &parms.dz );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%f %f", &parms.xlo, &parms.xhi );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%f %f", &parms.ylo, &parms.yhi );
- fgets( fileio.mapformat, sizeof(fileio.mapformat), fid_parm );
- fgets( fileio.outformat, sizeof(fileio.outformat), fid_parm );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%d", &cloud_erosion );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%d", &sst_type );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%d", &quality_flag );
- fgets( tmpline, sizeof(tmpline), fid_parm );
- sscanf( tmpline, "%d", &aerosol );
-
- fileio.mapformat[ strlen(fileio.mapformat)-1 ] = '\0';
- fileio.outformat[ strlen(fileio.outformat)-1 ] = '\0';
-
- fclose( fid_parm );
-
- if( strcmp(fileio.mapformat, "interleave") != 0 &&
- strcmp(fileio.mapformat, "sequential") != 0 ) {
- printf( "check parameter file for interleave or sequential maps !\n" );
- exit(1);
- }
- if( strcmp(fileio.outformat, "formatted") != 0 &&
- strcmp(fileio.outformat, "unformatted") != 0 ) {
- printf( "check parameter file for formatted or unformatted output !\n" );
- exit(1);
- }
-
- /* Calculate number of interpolation "steps" based on
- long and lat ranges divided by interpolation interval.
- Origin is at lon=-180, lat=90 */
- parms.imax = floorf((parms.xhi + 180) / parms.dx) - floorf((parms.xlo + 180) / parms.dx );
- parms.jmax = floorf((parms.yhi - 90) / parms.dy) - floorf((parms.ylo - 90) / parms.dy );
- printf( "imax, jmax, kmax: %d %d %d \n",
- parms.imax, parms.jmax, parms.kmax );
-
- parms.imin = 0;
- parms.jmin = 0;
- parms.kmin = 0;
-
- /* calculate degrees per grid point */
- parms.dxd = 360. / (float)parms.x_length;
- parms.dyd = 180. / (float)parms.y_length;
-
- /* find columns and rows for subsetting into hdf image array
- remember @ hdfarray(0,0) --> lon=-180, lat=90 */
- hdf.icolLeft = floorf( (parms.xlo + 180.) / parms.dxd );
- hdf.icolRight = floorf( ((parms.xhi + 180.) / parms.dxd) - 1. );
- hdf.irowUpper = floorf( fabsf(parms.yhi - 90.) / parms.dyd );
- hdf.irowLower = floorf( (fabsf(parms.ylo - 90.) / parms.dyd) - 1. );
-
- /* printf( "array corners: %d %d %d %d \n", hdf.icolLeft, hdf.irowUpper,
- hdf.icolRight ,hdf.irowLower); */
-
- /* set global interpolation flag if necessary */
- if( parms.xlo == -180. && parms.xhi == 180. ) {
- all_zones = 1; /* true */
- printf( "all_zones is true: interpolating across -180 West\n" );
- }
-
- if( cloud_erosion )
- printf( "performing cloud erosion filtering on SST data \n" );
- if( sst_type )
- printf( "working with 'all pixel' data: quality flag is %d \n", quality_flag );
- if( aerosol )
- printf( "performing aerosol correction \n" );
-
- /* allocate space for arrays */
- calloc_bytes = 0;
- if( (temp = fmal2d( parms.x_length, parms.y_length )) == NULL )
- printf( "not enough memory for array temp\n" ), exit(-1);
- else calloc_bytes = parms.x_length * parms.y_length * sizeof(float);
-
- if( (in_data = cmal2d( parms.x_length, parms.y_length )) == NULL )
- printf( "not enough memory for array in_data\n" ), exit(-1);
- else calloc_bytes += parms.x_length * parms.y_length * sizeof(char);
-
- if( (quality_data = cmal2d( parms.x_length, parms.y_length )) == NULL )
- printf( "not enough memory for array quality_data\n" ), exit(-1);
- else calloc_bytes += parms.x_length * parms.y_length * sizeof(char);
-
- if( (sl = fmal3d( parms.imax, parms.jmax, parms.kmax )) == NULL )
- printf( "not enough memory for array sl\n" ), exit(-1);
- else calloc_bytes += parms.imax * parms.jmax * parms.kmax * sizeof(float);
-
- if( (vsum = fmal4d( 2, parms.imax, parms.jmax, parms.kmax )) == NULL )
- printf( "not enough memory for array vsum\n" ), exit(-1);
- else calloc_bytes += parms.imax * parms.jmax * parms.kmax * 2 * sizeof(float);
-
- if( (sst_seq = (float *)calloc( parms.imax, sizeof(float) )) == NULL )
- printf( "not enough memory for array sst_seq\n" ), exit(-1);
- else calloc_bytes += parms.imax * sizeof(float);
-
- if( (sst_inter = (float *)calloc( parms.kmax, sizeof(float) )) == NULL )
- printf( "not enough memory for array sst_inter\n" ), exit(-1);
- else calloc_bytes += parms.kmax * sizeof(float);
-
- if( (x = (float *)calloc( parms.ndatamax, sizeof(float) )) == NULL )
- printf( "not enough memory for array x\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(float);
-
- if( (y = (float *)calloc( parms.ndatamax, sizeof(float) )) == NULL )
- printf( "not enough memory for array y\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(float);
-
- if( (z = (float *)calloc( parms.ndatamax, sizeof(float) )) == NULL )
- printf( "not enough memory for array z\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(float);
-
- if( (f = (float *)calloc( parms.ndatamax, sizeof(float) )) == NULL )
- printf( "not enough memory for array f\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(float);
-
- /* Extend array xx by long_extend to handle crossing
- longitude 180 during global interpolation.
- Extend by 4*search radius (plus 1 in xx dimension
- for -180 West crossing point */
- if( all_zones ) {
- long_extend = nint( 4 * parms.xwin2 / parms.dx );
- /* make sure long_extend is even for equal number of
- bins on either side of -180 W */
- if( long_extend % 2 != 0 )
- long_extend++;
- if( (xx = (float *)calloc( parms.imax + long_extend + 1, sizeof(float) )) == NULL )
- printf( "not enough memory for array xx\n" ), exit(-1);
- else calloc_bytes += (parms.imax + long_extend + 1) * sizeof(float);
- } else {
- if( (xx = (float *)calloc( parms.imax, sizeof(float) )) == NULL )
- printf( "not enough memory for array xx\n" ), exit(-1);
- else calloc_bytes += parms.imax * sizeof(float);
- }
-
- if( (yy = (float *)calloc( parms.jmax, sizeof(float) )) == NULL)
- printf( "not enough memory for array yy\n" ), exit(-1);
- else calloc_bytes += parms.jmax * sizeof(float);
-
- if( (zz = (float *)calloc( parms.kmax, sizeof(float) )) == NULL )
- printf( "not enough memory for array zz\n" ), exit(-1);
- else calloc_bytes += parms.kmax * sizeof(float);
-
- if( (ixmin = (int *)calloc( parms.ndatamax, sizeof(int) )) == NULL )
- printf( "not enough memory for array ixmin\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(int);
-
- if( (ixmax = (int *)calloc( parms.ndatamax, sizeof(int) )) == NULL )
- printf( "not enough memory for array ixmax\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(int);
-
- if( (iymin = (int *)calloc( parms.ndatamax, sizeof(int) )) == NULL )
- printf( "not enough memory for array iymin\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(int);
-
- if( (iymax = (int *)calloc( parms.ndatamax, sizeof(int) )) == NULL )
- printf( "not enough memory for array iymax\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(int);
-
- if( (izmin = (int *)calloc( parms.ndatamax, sizeof(int) )) == NULL )
- printf( "not enough memory for array izmin\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(int);
-
- if( (izmax = (int *)calloc( parms.ndatamax, sizeof(int) )) == NULL )
- printf( "not enough memory for array izmax\n" ), exit(-1);
- else calloc_bytes += parms.ndatamax * sizeof(int);
-
-
- printf( "done calloc arrays\n" );
- printf( "total bytes used: %d (%5.1f Mb)\n\n", calloc_bytes, calloc_bytes/1e6 );
-
-
- /* Determine zlo which corresponds to record number in infile for
- first map center and number of days since 1 Jan 1985 */
- fileio.istart = atoi( startrec );
-
- /* Open input file list and read first record */
- if( (fid_infile = fopen( fileio.infile, "r" )) == NULL ){
- printf( "\nCan not open %s\n", fileio.infile );
- exit(1);
- }
-
- /* position to proper record in file */
- fseek( fid_infile, (fileio.istart-1)*RECSIZE, SEEK_SET );
- fgets( datafile, sizeof(datafile), fid_infile );
- fclose( fid_infile );
- basefile = basename( &datafile[0] );
-
- strcpy( cday, " " ); /* initialize strings */
- strcpy( cyr, " " );
- /* e.g, parse year and jul day from 1985004h54dd-gdm.hdf */
- strncpy( cyr, &basefile[0], 4 );
- iyr = atoi( cyr );
- strncpy( cday, &basefile[4], 3 );
- iday = atoi( cday );
-
- /* set parms.zlo: days since 1 Jan 1985 */
- parms.zlo = (float)iday+(float)(iyr-1985)*365.;
- if(iyr > 1988) parms.zlo = parms.zlo + 1.;
- if(iyr > 1992) parms.zlo = parms.zlo + 1.;
- if(iyr > 1996) parms.zlo = parms.zlo + 1.;
- if(iyr > 2000) parms.zlo = parms.zlo + 1.;
- if(iyr > 2004) parms.zlo = parms.zlo + 1.;
- if(iyr > 2008) parms.zlo = parms.zlo + 1.;
- printf( " zlo is %f\n", parms.zlo );
- printf( " first map centered on day: %d in year: %d\n", iday, iyr );
-
- /* Set the first center dates */
- cstartdate = strcat( cdate, cyr );
- cstartdate = strcat( cdate, cday );
- date.startdate = atoi( cstartdate );
- printf( " start date is %d \n", date.startdate );
-
- /* reopen file to determine last map center date */
- fopen( fileio.infile, "r" );
- fileio.iend = fileio.istart + (parms.kmax-1) * parms.dz;
- fseek(fid_infile, (fileio.iend-1)*RECSIZE, SEEK_SET );
- fgets( datafile, sizeof(datafile), fid_infile );
- fclose( fid_infile );
- basefile = basename( &datafile[0] );
-
- /* parse year and jul day */
- strcpy( cdate, "" );
- strncpy( cyr, &basefile[0], 4 );
- strncpy( cday, &basefile[4], 3 );
- cenddate = strcat( cdate, cyr );
- cenddate = strcat( cdate, cday );
- date.enddate = atoi( cenddate );
- printf( " end date is %d\n", date.enddate );
-
- /* Now determine temporal interpolation (record) ranges */
- /* set istart and iend so the first and last maps will span a complete
- temporal interpolation range if possible. e.g., if first map is centered
- on 1 Jan 1986 (this is date of first record to read from the
- input datafile [fileio.istart]), with a search radius of 5 days (zwin2),
- set istart back five days to 27 Dec 1985 and iend to 6 Jan 1986 */
-
- /* set initial step back and forward size to temporal search radius */
- rec_step = (int) parms.zwin2;
-
- /* reopen input datafile */
- if( (fid_infile = fopen( fileio.infile, "r" )) == NULL ){
- printf( "\nCan not open %s\n", fileio.infile );
- exit(1);
- }
-
- /* determine max bytes in infile */
- fseek( fid_infile, 0L, SEEK_END );
- max_infile_size = ftell( fid_infile );
-
- /* step back into infile; use return of fseek() to determine if
- file pointer is set before beginning of file */
- while( fseek(fid_infile, (fileio.istart-1-rec_step)*RECSIZE, SEEK_SET) != 0 )
- rec_step--;
- fileio.istart -= rec_step;
-
- /* step forward into infile */
- rec_step = parms.zwin2;
- fseek( fid_infile, (fileio.iend-1+rec_step)*RECSIZE, SEEK_SET );
-
- /* if the file pointer is set beyond the EOF: step back */
- while( ftell(fid_infile) >= max_infile_size ) {
- rec_step--;
- fseek( fid_infile, (fileio.iend-1+rec_step)*RECSIZE, SEEK_SET );
- }
- fileio.iend += rec_step;
-
- printf( " records: istart %d and iend %d \n\n", fileio.istart, fileio.iend );
- fclose( fid_infile );
-
- /* call interp() */
- interp();
-
- free(temp), free(sl), free(vsum), free(sst_seq), free(sst_inter);
- free(x), free(y), free(z), free(f), free(xx), free(yy), free(zz);
- free(ixmin), free(ixmax), free(iymin), free(iymax);
- free(izmin), free(izmax), free(in_data), free(quality_data);
- }
-
-/* functions to allocate space for float arrays */
-/* 2d dimension */
-float **fmal2d( int ix, int iy ){
-
- float **ival;
- int i;
-
- ival = (float **)calloc( ix, sizeof(float *) );
- for( i=0; i < ix; i++ )
- ival[i] = (float *)calloc( iy, sizeof(float) );
-
- return(ival);
-}
-
- /* 3d dimension */
- float ***fmal3d( int ix, int iy, int iz ){
-
- float ***ival;
- int i, j;
-
- ival = (float ***)calloc( ix, sizeof(float **) );
- for( i=0; i < ix; i++ )
- ival[i] = (float **)calloc( iy, sizeof(float *) );
-
- for( i=0; i < ix; i++ )
- for( j=0; j < iy; j++ )
- ival[i][j] = (float *)calloc( iz, sizeof(float) );
-
- return(ival);
- }
-
-/* 4d dimension */
-float ****fmal4d( int ix, int iy, int iz, int ia ){
-
- float ****ival;
- int i, j, k;
-
- ival = (float ****)calloc( ix, sizeof(float ***) );
- for( i=0; i < ix; i++ )
- ival[i] = (float ***)calloc( iy, sizeof(float **) );
- for( i=0; i < ix; i++ )
- for( j=0; j < iy; j++)
- ival[i][j] = (float **)calloc( iz, sizeof(float *) );
- for( i=0; i < ix; i++ )
- for( j=0; j < iy; j++)
- for( k=0; k < iz; k++)
- ival[i][j][k] = (float *)calloc( ia, sizeof(float) );
-
- return(ival);
-}
-
-/* function to allocate space for char array */
-/* 2d dimension */
-char **cmal2d( int ix, int iy ) {
-
- char **ival;
- int i;
-
- ival = (char **)calloc( ix, sizeof(char *) );
- for( i=0; i < ix; i++ )
- ival[i] = (char *)calloc( iy, sizeof(char) );
-
- return(ival);
-}
-
-
-
-usage() {
- printf( " ~~ gaussian interpolation of pathfinder SST data (C version) ~~\n\n " );
- printf( "USAGE: gaussinterp <infile> <beg_rec> <parmfile> <outfile>\n" );
- printf( " -- infile: file containing list to process\n" );
- printf( " -- beg_rec: first record (file) in infile processed\n" );
- printf( " -- parmfile: input parameter file (e.g., interp.parms)\n" );
- printf( " -- outfile: interpolated output file (ascii or binary)\n" );
- printf( "\n e.g., gaussinterp list.dat 100 interp.parms output.dat\n" );
- printf( "[process records of list.dat starting from record 100; interpolated \
-result is output.dat]\n" );
- printf( "note: see interp.parms for example format of parameter file\n" );
- printf( " see gaussinterp.readme for more general information\n" );
- return(0);
-}
diff --git a/climatology/clim/orig/Fortran/armstrong_interp_code.tar b/climatology/clim/orig/Fortran/armstrong_interp_code.tar
deleted file mode 100755
index 4e125b3..0000000
Binary files a/climatology/clim/orig/Fortran/armstrong_interp_code.tar and /dev/null differ
diff --git a/climatology/clim/orig/Fortran/binsum.f b/climatology/clim/orig/Fortran/binsum.f
deleted file mode 100644
index 96b65af..0000000
--- a/climatology/clim/orig/Fortran/binsum.f
+++ /dev/null
@@ -1,64 +0,0 @@
-c $Revision: 2.4 $
-c subprogram binsum
-
-c-- calculate interpolation weights and vsum
-
- subroutine binsum( nf, x, y, z, f,
- & xx, yy, zz, vsum,
- & ixmin, ixmax, iymin,
- & iymax, izmin, izmax,
- & imax, jmax, kmax, ndatamax )
-
- real*4 x(ndatamax),y(ndatamax),z(ndatamax),f(ndatamax)
- real*4 xx(imax),yy(jmax),zz(kmax)
- real*4 vsum(2,imax,jmax,kmax)
- integer*4 ixmin(ndatamax),ixmax(ndatamax)
- integer*4 iymin(ndatamax),iymax(ndatamax)
- integer*4 izmin(ndatamax),izmax(ndatamax)
-
- common /parms/ imin,jmin,kmin,
- & xwin2,ywin2,zwin2,
- & xh,yh,zh,
- & dx,dy,dz,
- & xlo,xhi,ylo,yhi,zlo,
- & dxd,dyd
-
- data xfac/-0.6931/
-
- do n=1,nf
- ixmin(n)=max(nint((x(n)-xwin2-xlo+0.5*dx)/dx),1)
- ixmax(n)=min(nint((x(n)+xwin2-xlo+0.5*dx)/dx),imax)
- iymin(n)=max(nint((y(n)-ywin2-ylo+0.5*dy)/dy),1)
- iymax(n)=min(nint((y(n)+ywin2-ylo+0.5*dy)/dy),jmax)
- izmin(n)=max(nint((z(n)-zwin2-zlo+0.5*dz)/dz),1)
- izmax(n)=min(nint((z(n)+zwin2-zlo+0.5*dz)/dz),kmax)
-c print *, x(n),y(n),z(n), f(n)
-c print *,' ixmin, ixmax', ixmin(n), ixmax(n)
-c print *,' iymin, iymax', iymin(n), iymax(n)
-c print *,' izmin, izmax', izmin(n), izmax(n)
- enddo
-
-
-
- do n=1,nf
- do kk=izmin(n),izmax(n)
- do jj=iymin(n),iymax(n)
- do ii=ixmin(n),ixmax(n)
-
-c- - this is the algorithm coded for weights
-
- fac=exp( xfac*(((x(n)-xx(ii))/xh)**2
- & + ((y(n)-yy(jj))/yh)**2
- & + ((z(n)-zz(kk))/zh)**2) )
-
-c print *, 'x, xx, y, yy, z, zz, fac f',
-c & x(n), xx(ii), y(n), yy(jj), z(n), zz(kk), fac, f(n)
-
- vsum(1,ii,jj,kk)=vsum(1,ii,jj,kk)+f(n)*fac
- vsum(2,ii,jj,kk)=vsum(2,ii,jj,kk)+fac
- enddo
- enddo
- enddo
- enddo
- return
- end
diff --git a/climatology/clim/orig/Fortran/interp.f b/climatology/clim/orig/Fortran/interp.f
deleted file mode 100644
index 8977798..0000000
--- a/climatology/clim/orig/Fortran/interp.f
+++ /dev/null
@@ -1,302 +0,0 @@
-#include "interp.h"
-c $Revision: 2.12 $
-c subprogram interp.f
-c reads hdf file, extract data, call bisum(), calculates
-c and writes out interpolated maps
-
-
- subroutine interp( temp, sst_inter, sst_seq, sl,
- & x, y, z, f,
- & xx, yy, zz, vsum, ixmin, ixmax,
- & iymin, iymax, izmin, izmax, ndatamax,
- & x_length, y_length, imax, jmax, kmax )
-
- integer x_length,y_length
- integer dummy
-
-c-- dimension the arrays
- real*4 temp(x_length,y_length)
- real*4 sl(imax,jmax,kmax)
- real*4 sst_inter(kmax), sst_seq(imax)
- real*4 x(ndatamax),y(ndatamax),z(ndatamax),f(ndatamax)
- real*4 xx(imax),yy(jmax),zz(kmax)
- real*4 vsum(2,imax,jmax,kmax)
- real*4 ULlon, ULlat, LRlon, LRlat
- integer*4 ixmin(ndatamax),ixmax(ndatamax)
- integer*4 iymin(ndatamax),iymax(ndatamax)
- integer*4 izmin(ndatamax),izmax(ndatamax)
-
-c-- hdf data array must be hardwired, no dync alloc :(
- byte in_data( XSIZE, YSIZE )
-
- character*80 plabel
- character*50 infile
- character*30 outfile
- character*150 datafile
- character*30 basename
- character*35 tmpfile
- character*10 mapformat
- character*15 outformat
- character*3 cday
- character*4 cyr
- character*200 zcatcmd
-
- common /parms/ imin,jmin,kmin,
- & xwin2,ywin2,zwin2,
- & xh,yh,zh,
- & dx,dy,dz,
- & xlo,xhi,ylo,yhi,zlo,
- & dxd,dyd
-
- common /fileio/ infile,outfile,istart,
- & iend,mapformat,outformat
-
- common /hdfindex/ icolLeft,icolRight,irowUpper,irowLower
-
- common /basefile/ basename
-
-c-- open output file
- open(unit=20,status='new',form=outformat,file=outfile,
- & err=1000,iostat=ierr )
- plabel='sst interpolation'
-
-c-- initialize vsum to '0's
- do k1=1,2
- do k2=1,kmax
- do k3=1,jmax
- do k4=1,imax
- vsum(k1,k4,k3,k2)=0.
- enddo
- enddo
- enddo
- enddo
-
-c-- initialize arrays of long, lat and time intervals
- do ii=imin,imax
- xx(ii)=xlo+float(ii-1)*dx ! Long to interpolate to
- enddo
- do jj=jmin,jmax
- yy(jj)=ylo+float(jj-1)*dy ! Lat to interpolate to
- enddo
- do kk=kmin,kmax
- zz(kk)=zlo+float(kk-1)*dz ! Time to interpolate to
- enddo
-
- do n=1,ndatamax
- x(n)=0.0
- y(n)=0.0
- z(n)=0.0
- f(n)=0.0
- ixmin(n) = 0
- ixmax(n) = 0
- iymin(n) = 0
- iymax(n) = 0
- izmin(n) = 0
- izmax(n) = 0
- enddo
-
-c-- slope (cal) and offset to convert DN to SST
- cal= 0.15
- offset = 3.0
-
-c-- Open input file list
- open(unit=15,status='old',file=infile,access='direct',
- & recl=RECSIZE,form='formatted',err=1500,iostat=ierr)
-
-c-- read a infile one record at a time and process . . .
- do k=istart,iend
- ipts=0
- read( 15,'(a)',rec=k ) datafile
-c print *, datafile
-
- icompress = 0
-c-- hack to get trailing 'Z' (if compressed)
-c do islash = 150, 1, -1
-c if ( datafile(islash:islash) .eq. 'Z' ) then
-c icompress = 1
-c goto 101
-c endif
-c enddo
-c101 continue
-
-c-- 12/27/99: C call for basename implemented
-c-- variable basename returned via common statement
- call getbase( datafile )
- print *,'\n file: ', basename
-
-c-- cyr and cday parsed from basename
-c-- (e.g., parsed from '1988332h54dd-gdm.hdf')
- cday=basename( 5:7 )
- cyr=basename( 1:4 )
-
-c-- if unix compressed, zcat to a temporary filename
- if( basename(22:22) .eq. 'Z' ) then
- icompress = 1
- tmpfile = '/tmp/' // basename(1:20)
- zcatcmd = 'zcat ' // datafile( 1:len(datafile)-1 )
- & // ' > ' // tmpfile
- call system( zcatcmd )
- datafile = tmpfile
- endif
-
-c-- convert iday character to integer
- read(cday,'(i3)') iday
- read(cyr,'(i4)') iyr
-c write(6,*)'cday = ',cday
-c write(6,*)'cyr = ',cyr
-
-c***> HDF call: d8gimg()
- retn=d8gimg(datafile,in_data,x_length,
- & y_length,dummy)
-
-c-- read hdf DN values and convert to SST
- do i=icolLeft, icolRight
- do j=irowUpper, irowLower
- xlat=-89.75+dyd*float(j-1)
-
-c-- center output in Pacific Ocean
-c if( i .lt. x_length ) ix = i-(x_length/2)
-c if( i .le. (x_length/2) ) ix = i+(x_length/2)
-
-c-- center output in Atlantic (default)
- ix = i
-
-c-- convert signed byte to float
-c if ( in_data(i,j) .lt. 0 .and. abs(xlat) .lt. 70. ) then
- if ( in_data(i,j).lt. 0 ) then
- temp(ix,j)=float( in_data(i,j) ) + 256
- else
- temp(ix,j)=float( in_data(i,j) )
- endif
-
-
-
-C MULTIPLY THE PATHFINDER DIGITAL NUMBER BY THE CALIBRATION NUMBER (0.15)
-C AND ADD THE OFFSET (-3.0) TO GET DEGREES CELSIUS
-
- if ( temp(ix,j).gt.0 ) then
- ipts=ipts+1
- f(ipts)=( cal*temp(ix,j) ) - offset
- x(ipts) = ( dxd*float(ix-1) ) - 180. + dxd/2
- y(ipts) = ( 90. - (dyd*float(j-1)) ) - dyd/2
- z(ipts)=float(iday)+float(iyr-1985)*365.
- if(iyr.gt.1988) z(ipts)=z(ipts)+1
- if(iyr.gt.1992) z(ipts)=z(ipts)+1
- if(iyr.gt.1996) z(ipts)=z(ipts)+1
- if(iyr.gt.2000) z(ipts)=z(ipts)+1
- if(iyr.gt.2004) z(ipts)=z(ipts)+1
- endif
- enddo
- enddo
-
- nfnew=ipts
- print *, ' no of pts in file ',' = ', nfnew
-
-c-- calculate interpolation weights and vsum
-c-- arrays passed directly... a common statement
-c-- does not seem to work.
- call binsum( nfnew, x, y, z, f,
- & xx, yy, zz, vsum,
- & ixmin, ixmax, iymin,
- & iymax, izmin, izmax,
- & imax, jmax, kmax, ndatamax )
-
- if( icompress .eq. 1 ) call system( 'rm -f ' // tmpfile )
-c-- ..... read next hdf file from infile
- enddo
-
-c-- all input files processed; calculate interpolated SST
- do kk=1,kmax
- do jj=1,jmax
- do ii=1,imax
- sl(ii,jj,kk)=0.
- if (vsum(2,ii,jj,kk).gt.0) then
- sl(ii,jj,kk)=vsum(1,ii,jj,kk)/vsum(2,ii,jj,kk)
- endif
- enddo
- enddo
- enddo
-
-c-- write output as map "interleaved" or map "sequential"
-c-- "interleaved" is the original implementation
-c-- both formats preceded by header
-
-c-- "interleaved" refers to each row in the output
-c-- file representing a unique lon, lat position with columns
-c-- of associated sst values (maps)
-c-- i.e. row one: lon1 lat1 sst1 sst2 sst3....lastsst
-c-- row two: lon2 lat1 sst1 sst2 sst3....lastsst
-
-c-- "sequential" refers to each map written as rows and
-c-- columns of lat, lon with the array element representing the
-c-- sst at that geo-position for that map.
-c-- each map is written sequentially in the file
-
-c-- geo-positions of UL and LR corners
- ULlon = -(180 - ( ((icolLeft-1) * dxd) + dxd/2 ))
- ULlat = (90 - ( ((irowUpper-1) * dyd) + dyd/2 ))
- LRlon = -(180 - ( (icolRight * dxd) - dxd/2 ))
- LRlat = (90 - ( (irowLower * dyd) - dyd/2 ))
-
-c-- version number, "f" refers to fortran version
- version = 'f2.9'
-
-c-- write the 3 record header
- if( outformat .eq. 'formatted' ) then
- write(20,'(a)'), plabel
- write(20,*) imax,jmax,kmax
- write(20,*) dx,dy,dz
- write(20,*) ULlon,ULlat,LRlon,LRlat
- elseif( outformat .eq. 'unformatted' ) then
- write(20) imax,jmax,kmax
- write(20) dx,dy,dz
- write(20) ULlon,ULlat,LRlon,LRlat
- endif
-
- if( mapformat .eq. 'interleave' ) then
- print *, '\n map output is interleave'
- do jj=1,jmax
- do ii=1,imax
- do kk=1,kmax
- sst_inter(kk)=sl(ii,jj,kk)
- enddo
- if( outformat .eq. 'formatted' ) then
- write(20,*) ii,jj,sst_inter
- else
- write(20) ( sst_inter(i), i=1,kmax )
- endif
- enddo
- enddo
-
- else if( mapformat .eq. 'sequential' ) then
- print *, '\n map output is sequential'
- do kk=1,kmax
- do jj=1,jmax
- do ii=1,imax
- sst_seq(ii)=sl(ii,jj,kk)
- enddo
- if( outformat .eq. 'formatted' ) then
- write(20,*) jj,kk,sst_seq
- else
- write(20) ( sst_seq(i), i=1,imax )
- endif
- enddo
- enddo
- endif
-
- print *, '\ndone. . . '
- close( 15,status='keep',err=2000,iostat=ierr )
- close( 20,status='keep',err=2500,iostat=ierr )
- stop
-
- 1000 print *, 'Error opening output: ', outfile, 'error num: ', ierr
- goto 102
- 1500 print *, 'Error opening input: ', infile, 'error num: ', ierr
- goto 102
- 2000 print *, 'Error closing input: ', infile, 'error num: ', ierr
- goto 102
- 2500 print *, 'Error closing output: ', outfile, 'error num: ', ierr
- goto 102
- 102 continue
-
- end
diff --git a/climatology/clim/orig/Fortran/makefile b/climatology/clim/orig/Fortran/makefile
deleted file mode 100644
index a624096..0000000
--- a/climatology/clim/orig/Fortran/makefile
+++ /dev/null
@@ -1,46 +0,0 @@
-# makefile for guassinterp.
-# choose f77 or f90 compiler
-# and appropriate flags
-
-# f77 compiler
-FC = f77
-FFLAGS = -g
-#FFLAGS = -O -n32
-DEFINE = -DLANG_F77
-HDF = /usr/local/hdf
-#HDF = /usr/local/hdf4.1r3-n32
-
-# f90 compiler
-#FC = f90
-#FFLAGS = -n32 -bytereclen -cpp -extend_source
-#DEFINE = -DLANG_F90
-
-PGM = gaussinterp
-LIBS = -L$(HDF)/lib -ldf -ljpeg -lz -lgen
-INCLUDE = -Wf,-I/usr/local/include/hdf
-BIN = /sst/vol7/PATHFINDER/bin/sgi
-
-SRC = setupinterp.f interp.f binsum.f getbase.c passbase.f
-OBJ = setupinterp.o interp.o binsum.o getbase.o passbase.o
-
-$(PGM): $(OBJ)
- $(FC) $(FFLAGS) -o $(PGM) $(OBJ) $(LIBS)
-
-setupinterp.o:
- $(FC) $(FFLAGS) $(DEFINE) -c setupinterp.f
-interp.o:
-# $(FC) $(DEFINE) -c interp.f
- $(FC) $(FFLAGS) $(DEFINE) -c interp.f
-binsum.o:
- $(FC) $(FFLAGS) $(DEFINE) -c binsum.f
-passbase.o:
- $(FC) $(FFLAGS) $(DEFINE) -c passbase.f
-getbase.o:
- $(CC) $(FFLAGS) -c getbase.c
-
-install: $(PGM)
- cp $(PGM) $(BIN)
-
-clean:
- rm -f $(OBJ) $(PGM)
-
diff --git a/climatology/clim/orig/Fortran/passbase.f b/climatology/clim/orig/Fortran/passbase.f
deleted file mode 100644
index 79e2d8e..0000000
--- a/climatology/clim/orig/Fortran/passbase.f
+++ /dev/null
@@ -1,9 +0,0 @@
- subroutine passbase( filename )
-
- character*25 filename
- character*25 basename
- common /basefile/ basename
-
- basename = filename
- end
-
diff --git a/climatology/clim/orig/Fortran/setupinterp.f b/climatology/clim/orig/Fortran/setupinterp.f
deleted file mode 100644
index 8eed0b7..0000000
--- a/climatology/clim/orig/Fortran/setupinterp.f
+++ /dev/null
@@ -1,291 +0,0 @@
-#include "interp.h"
-c $Revision: 2.12 $
-c NAME gaussinterp
-c 7 Dec 99 earmstrong NASA/JPL
-c
-c DESCRIPTION
-c Gaussian interpolation program modified to map SST (HDF) data.
-c Modified from intersstdbg7day3.f (jvazquez' prog).
-c
-c SYNOPSIS
-c setupinterp() allocates space for arrays, calls interp() which does the
-c interpolation
-c
-c USAGE
-c % gaussinterp infile beg_rec end_rec outfile parmfile
-
-c **** Original program comments (intersstdbg7day3.f) ***
-c DESCRIPTION: Maps data when given the required information. Program
-c originally written by raghu. Version 7 reads the gridded data set.
-c Currently maps TMR water vapor.
-c
-c COMMENTS:
-c This programs wraps around in longitude for 0 to 360 maps.
-c If different maps size is needed in longitude w/o wraparound
-c some lines of code need to be modified in the file.
-c
-c
-c CHANGES:
-c 2/6/97 - change fint from integer*4 to integer*2; holds residuals
-c 2/19/97 - change output to direct access file, change sl() to int*2
-c 8/8/97 - change i/o so that it reads data directly from the residual
-c data files
-c 9/9/97 - change main program so that i/o part are subroutines
-c 9/10/97 - add header,time,lat,lon and #of maps to output file
-c 9/11/97 - add version and output file specification in runtime
-c argument list
-c **** end Orginal comments ****
-
-c 12/01/99 - major modifications (ema).
-c - added command line read for input, output filenames
-c and records numbers of input file read
-c - binsum is now a subprogram
-c - mapping params (interp.h) read in with include statement
-c - removed some superfluous do loops
-c 12/06/99 - dynamic allocation of arrays added
-c - read input parms from arg 5 (see 'interp.parms'
-c for an example of format)
-c - hdf data array size still hardcoded in interp.h
-c 12/10/99 - variable zlo set equal to command line arg for first
-c record number in infile to process (istart). zlo
-c removed from parameter file (interp.parms)
-c 12/13/99 - imax, jmax calculated from xlo, xhi, ylo, and yhi
-c imin, jmin, kmin set to 1
-c 12/16/99 - added options in parameter file for sequential
-c or interleaved maps, and formatted or unformatted
-c output file
-c 12/27/99 - added basename C call. program can now handle
-c compressed datafiles.
-c 1/4/00 - added subsetting of hdf array based on lon/lat
-c bounds. zlo now read from parameter file.
-c 1/4/00 - zlo found from parse of first datafile.
-c geo-positions of UL, LR written to output header
-
- program setupinterp
-
- integer ndatamax, x_length, y_length
- integer temp_malloc, sst_inter_malloc
- integer sst_seq_malloc, sl_malloc
- integer x_malloc, y_malloc, z_malloc, f_malloc
- integer xx_malloc, yy_malloc, zz_malloc, vsum_malloc
- integer ixmin_malloc, ixmax_malloc
- integer iymin_malloc, iymax_malloc
- integer izmin_malloc, izmax_malloc
- real*4 float_size
- integer*4 int_size
-
- character*80 plabel
- character*50 infile
- character*30 outfile
- character*60 parmfile
- character*10 mapformat
- character*15 outformat
- character*6 startrec
- character*6 endrec
- character*3 cday
- character*4 cyr
- character*150 datafile
- character*30 basename
-
- common /parms/imin,jmin,kmin,
- & xwin2,ywin2,zwin2,
- & xh,yh,zh,
- & dx,dy,dz,
- & xlo,xhi,ylo,yhi,zlo,
- & dxd,dyd
-
- common /fileio/ infile,outfile,istart,
- & iend,mapformat,outformat
- common /hdfindex/ icolLeft,icolRight,irowUpper,irowLower
-
- common /basefile/ basename
-
-c ...check command line arguments
- call getarg( 1, infile )
- call getarg( 2, startrec )
- call getarg( 3, endrec )
- call getarg( 4, parmfile )
- call getarg( 5, outfile )
-
- inum_args = iargc()
- if ( inum_args .ne. 5 ) then
- call usage()
- stop
- endif
-
- read( startrec, '(i)' ) istart
- read( endrec, '(i)' ) iend
-
- if ( iend .lt. istart ) then
- write(6,*) '\n Error: end record number
- & smaller than beginning record ! \n'
- stop
- endif
-
-c-- zlo determined by reading first record and parsing the datafile
-c-- for year and julian day using getbase() (C call)
-
-c-- Open input file list
- open( unit=ifile,status='old',file=infile,access='direct',
- & recl=RECSIZE,form='formatted',err=2000,iostat=ierr )
-
- read( ifile, '(a)', rec=istart ) datafile
- close( ifile, status='keep', err=2500, iostat=ierr )
-
- call getbase( datafile ) ! returns basename
- cday=basename( 5:7 )
- cyr=basename( 1:4 )
- read( cday,'(i3)' ) iday
- read( cyr,'(i4)' ) iyr
- zlo = float(iday)+float(iyr-1985)*365.
- if(iyr.gt.1988) zlo = zlo + 1.
- if(iyr.gt.1992) zlo = zlo + 1.
- if(iyr.gt.1996) zlo = zlo + 1.
- if(iyr.gt.2000) zlo = zlo + 1.
- if(iyr.gt.2004) zlo = zlo + 1.
- print *, 'zlo is ', zlo
-
-c-- read in the parameter file
- open( unit=iparm,status='old',form='formatted',file=parmfile,
- & err=1000,iostat=ierr )
-
- read( unit=iparm, fmt=* ) ndatamax
- read( unit=iparm, fmt=* ) x_length, y_length
- read( unit=iparm, fmt=* ) kmax
- read( unit=iparm, fmt=* ) xwin2, ywin2, zwin2
- read( unit=iparm, fmt=* ) xh, yh, zh
- read( unit=iparm, fmt=* ) dx, dy, dz
- read( unit=iparm, fmt=* ) xlo, xhi
- read( unit=iparm, fmt=* ) ylo, yhi
-c read( unit=iparm, fmt=* ) zlo
-c read( unit=iparm, fmt=* ) dxd, dyd
- read( unit=iparm, fmt='(a)' ) mapformat
- read( unit=iparm, fmt='(a)' ) outformat
- close( iparm,status='keep',err=1500,iostat=ierr )
-
-c write(*,*) ndatamax
-c write(*,*) x_length, y_length
-c write(*,*) kmax
-c write(*,*) xwin2,ywin2, zwin2
-c write(*,*) xh, yh, zh
-c write(*,*) dx, dy, dz
-c write(*,*) xlo, xhi
-c write(*,*) ylo, yhi
-c write(*,*) zlo
-cc write(*,*) dxd, dyd
-c write(*,*) mapformat
-c write(*,*) outformat
-
- if( (mapformat .ne. 'interleave') .and.
- & (mapformat .ne. 'sequential') ) then
- print *, 'check parameter file for interleave or sequential maps !'
- stop
- endif
- if( (outformat .ne. 'formatted') .and.
- & (outformat .ne. 'unformatted') ) then
- print *, 'check parameter file for formatted or unformatted output !'
- stop
- endif
-
-
-c-- calculate max number of interpolation "steps" based on
-c-- long and lat ranges divided by interpolation interval
-c imax = ( (xhi - xlo) + 1 ) / dx
-c jmax = ( abs(yhi - ylo) + 1 ) / dy
- imax = ( abs(xhi - xlo) ) / dx
- jmax = ( abs(yhi - ylo) ) / dy
-
- print *, 'imax, jmax, kmax', imax,jmax,kmax
-
- imin = 1
- jmin = 1
- kmin = 1
-
-c-- calculate degrees per grid point
- dxd = 360. / float(x_length)
- dyd = 180. / float(y_length)
-
-c-- find columns and rows for subsetting into hdf image array
-c-- remember @ hdfarray(1,1) --> lon=-180, lat=90
- icolLeft = nint( ((xlo + 180) / dxd) + 1 )
- icolRight = nint( (xhi + 180) / dxd )
- irowUpper = nint( (abs(yhi - 90) / dyd) + 1 )
- irowLower = nint( (abs(ylo - 90) / dyd) )
-
-c print *, 'icolLeft, icolRight,irowUpper, irowLower',
-c & icolLeft, icolRight, irowUpper, irowLower
-
-c-- 4 bytes for floats and ints
- float_size = 4
- int_size = 4
-
-c-- allocate space for arrays
- itotsize = x_length*y_length*float_size
- temp_malloc = malloc( %val(itotsize) )
- itotsize = kmax*float_size
- sst_inter_malloc = malloc( %val(itotsize) )
- itotsize = imax*float_size
- sst_seq_malloc = malloc( %val(itotsize) )
- itotsize = imax*jmax*kmax*float_size
- sl_malloc = malloc( %val(itotsize) )
- itotsize = ndatamax*float_size
- x_malloc = malloc( %val(itotsize) )
- y_malloc = malloc( %val(itotsize) )
- z_malloc = malloc( %val(itotsize) )
- f_malloc = malloc( %val(itotsize) )
- itotsize = imax*float_size
- xx_malloc = malloc( %val(itotsize) )
- itotsize = jmax*float_size
- yy_malloc = malloc( %val(itotsize) )
- itotsize = kmax*float_size
- zz_malloc = malloc( %val(itotsize) )
- itotsize = 2*imax*jmax*kmax*float_size
- vsum_malloc = malloc( %val(itotsize) )
- itotsize = ndatamax*int_size
- ixmin_malloc = malloc( %val(itotsize) )
- ixmax_malloc = malloc( %val(itotsize) )
- iymin_malloc = malloc( %val(itotsize) )
- iymax_malloc = malloc( %val(itotsize) )
- izmin_malloc = malloc( %val(itotsize) )
- izmax_malloc = malloc( %val(itotsize) )
-
-c print *, 'done malloc arrays\n'
-
-c--- call interp() with 'space' variables passed call by value
- call interp( %val(temp_malloc),
- & %val(sst_inter_malloc), %val(sst_seq_malloc),
- & %val(sl_malloc), %val(x_malloc), %val(y_malloc),
- & %val(z_malloc), %val(f_malloc), %val(xx_malloc),
- & %val(yy_malloc), %val(zz_malloc),
- & %val(vsum_malloc), %val(ixmin_malloc), %val(ixmax_malloc),
- & %val(iymin_malloc), %val(iymax_malloc), %val(izmin_malloc),
- & %val(izmax_malloc), ndatamax, x_length, y_length,
- & imax, jmax, kmax )
-
-c-- how to free memory? dunno
-
-
- 1000 print *, 'Error opening parameter file: ', parmfile, 'error num: ', ierr
- 1500 print *, 'Error closing parameter file: ', parmfile, 'error num: ', ierr
- 2000 print *, 'Error opening input: ', infile, 'error num: ', ierr
- 2500 print *, 'Error closing input: ', infile, 'error num: ', ierr
-
- end
-
- subroutine usage()
- print *, '\n ~~ gaussian interpolation of pathfinder SST data
- & (f77 version) ~~\n '
- print *, 'USAGE: % gaussinterp infile beg_rec end_rec parmfile outfile\n'
- print *, ' -- infile: file containing list to process
- & (ascii; fixed length records)'
- print *, ' -- beg_rec: first record (file) in infile processed'
- print *, ' -- end_rec: last record (file) in infile processed'
- print *, ' -- parmfile: input parameter file (e.g., interp.parms)'
- print *, ' -- outfile: interpolated output file (ascii or binary)'
- print *, '\n e.g., % gaussinterp list.dat 100 600 interp.parms output.dat '
- print *, '[process records 100 to 600 of list.dat; interpolated
- & result is output.dat]\n'
- print *, 'note: see interp.parms for example format of parameter file'
- print *, ' see gaussinterp.readme for more general information'
- print *, ' this executable compiled for image x and y size:', XSIZE, YSIZE
- end
diff --git a/climatology/clim/pixelStats.py b/climatology/clim/pixelStats.py
deleted file mode 100644
index bddb165..0000000
--- a/climatology/clim/pixelStats.py
+++ /dev/null
@@ -1,232 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-pixelStats.py
-
-Compute a multi-epoch (multi-day) statistics for each lat/lon pixel read from daily Level-3 grids.
-
-Also do statistics roll-ups from daily to monthly, monthly to seasonal, seasonal to yearly,
-yearly to multi-year, and multi-year to total N-year period.
-
-Simple code to be run using Spark or Dpark.
-
-"""
-
-import sys, os, urllib.request, urllib.parse, urllib.error, re, time
-import numpy as N
-import matplotlib
-from functools import reduce
-matplotlib.use('Agg')
-import matplotlib.pylab as M
-from netCDF4 import Dataset, default_fillvals
-
-from .variables import getVariables, close
-from .split import splitByMonth
-from .cache import retrieveFile, CachePath
-
-#from pyspark import SparkContext # both imported below when needed
-#import dpark
-
-Modes = ['sequential', 'dpark', 'spark']
-
-Accumulators = ['count', 'sum', 'sumsq', 'min', 'max']
-Stats = ['count', 'mean', 'stddev', 'min', 'max']
-
-GroupByKeys = ['month', 'season', 'year', '3-year', 'total']
-
-TimeFromFilenameDOY = {'get': ('year', 'doy'), 'regex': re.compile(r'\/A(....)(...)')}
-
-
-def pixelStats(urls, variable, nPartitions, timeFromFilename=TimeFromFilenameDOY, groupByKeys=GroupByKeys, accumulators=Accumulators,
- cachePath=CachePath, mode='dpark', modes=Modes):
- '''Compute a global (or regional) pixel mean field in parallel, given a list of URL's pointing to netCDF files.'''
- baseKey = groupByKeys[0]
- if baseKey == 'month':
- urlsByKey = splitByMonth(urls, timeFromFilename)
- else:
- print('pixelStats: Unrecognized groupByKey "%s". Must be in %s' % (baseKey, str(groupByKeys)), file=sys.stderr)
- sys.exit(1)
-
- if mode == 'sequential':
- accum = [accumulate(u, variable, accumulators) for u in urlsByKey]
- merged = reduce(combine, accum)
- stats = statsFromAccumulators(merged)
-
- elif mode == 'dpark':
- import dpark
- urls = dpark.parallelize(urlsByKey, nPartitions) # returns RDD of URL lists
- accum = urls.map(lambda urls: accumulate(urls, variable, accumulators)) # returns RDD of stats accumulators
- merged = accum.reduce(combine) # merged accumulators on head node
- stats = statsFromAccumulators(merged) # compute final stats from accumulators
-
- elif mode == 'spark':
- from pyspark import SparkContext
- sc = SparkContext(appName="PixelStats")
- urls = sc.parallelize(urlsByKey, nPartitions) # returns RDD of URL lists
- accum = urls.map(lambda urls: accumulate(urls, variable, accumulators)) # returns RDD of stats accumulators
- merged = accum.reduce(combine) # merged accumulators on head node
- stats = statsFromAccumulators(merged) # compute final stats from accumulators
-
- else:
- stats = None
- if mode not in modes:
- print('pixelStats: Unrecognized mode "%s". Must be in %s' % (mode, str(modes)), file=sys.stderr)
- sys.exit(1)
- return stats
-
-
-def accumulate(urls, variable, accumulators, cachePath=CachePath):
- '''Accumulate data into statistics accumulators like count, sum, sumsq, min, max, M3, M4, etc.'''
- keys, urls = urls
- accum = {}
- for i, url in enumerate(urls):
- try:
- path = retrieveFile(url, cachePath)
- fn = os.path.split(path)[1]
- except:
- print('accumulate: Error, continuing without file %s' % url, file=sys.stderr)
- continue
-
- try:
- var, fh = getVariables(path, [variable], arrayOnly=True, set_auto_mask=True) # return dict of variable objects by name
- v = var[variable] # masked array
- close(fh)
- except:
- print('accumulate: Error, cannot read variable %s from file %s' % (variable, path), file=sys.stderr)
- continue
-
- if i == 0:
- for k in accumulators:
- if k == 'min': accum[k] = default_fillvals['f8'] * N.ones(v.shape, dtype=N.float64)
- elif k == 'max': accum[k] = -default_fillvals['f8'] * N.ones(v.shape, dtype=N.float64)
- elif k == 'count': accum[k] = N.zeros(v.shape, dtype=N.int64)
- else:
- accum[k] = N.zeros(v.shape, dtype=N.float64)
-
- if 'count' in accumulators:
- accum['count'] += ~v.mask
- if 'min' in accumulators:
- accum['min'] = N.ma.minimum(accum['min'], v)
- if 'max' in accumulators:
- accum['max'] = N.ma.maximum(accum['max'], v)
-
- v = N.ma.filled(v, 0.)
- if 'sum' in accumulators:
- accum['sum'] += v
- if 'sumsq' in accumulators:
- accum['sumsq'] += v*v
- return (keys, accum)
-
-
-def combine(a, b):
- '''Combine accumulators by summing.'''
- keys, a = a
- b = b[1]
- for k in list(a.keys()):
- if k != 'min' and k != 'max':
- a[k] += b[k]
- if 'min' in accumulators:
- a['min'] = N.ma.minimum(a['min'], b['min'])
- if 'max' in accumulators:
- a['max'] = N.ma.maximum(a['max'], b['max'])
- return (('total',), a)
-
-
-def statsFromAccumulators(accum):
- '''Compute final statistics from accumulators.'''
- keys, accum = accum
- # Mask all of the accumulator arrays
- accum['count'] = N.ma.masked_equal(accum['count'], 0, copy=False)
- mask = accum['count'].mask
- for k in accum:
- if k != 'count':
- accum[k] = N.ma.array(accum[k], copy=False, mask=mask)
-
- # Compute stats (masked)
- stats = {}
- if 'count' in accum:
- stats['count'] = accum['count']
- if 'min' in accum:
- stats['min'] = accum['min']
- if 'max' in accum:
- stats['max'] = accum['max']
- if 'sum' in accum:
- stats['mean'] = accum['sum'] / accum['count']
- if 'sumsq' in accum:
- stats['stddev'] = N.sqrt(accum['sumsq'] / (accum['count'].astype(N.float32) - 1))
- return (keys, stats)
-
-
-def writeStats(urls, variable, stats, outFile, copyToHdfsPath=None, format='NETCDF4', cachePath=CachePath):
- '''Write out stats arrays to netCDF with some attributes.
- '''
- keys, stats = stats
- dout = Dataset(outFile, 'w', format=format)
- print('Writing %s ...' % outFile, file=sys.stderr)
- dout.setncattr('variable', variable)
- dout.setncattr('urls', str(urls))
- dout.setncattr('level', str(keys))
-
- inFile = retrieveFile(urls[0], cachePath)
- din = Dataset(inFile, 'r')
- try:
- coordinates = din.variables[variable].getncattr('coordinates')
- coordinates = coordinates.split()
- except:
- coordinates = ('lat', 'lon') # kludge: FIX ME
-
- # Add dimensions and variables, copying data
- coordDim = [dout.createDimension(coord, din.variables[coord].shape[0]) for coord in coordinates] # here lat, lon, alt, etc.
- for coord in coordinates:
- var = dout.createVariable(coord, din.variables[coord].dtype, (coord,))
- var[:] = din.variables[coord][:]
-
- # Add stats variables
- for k,v in list(stats.items()):
- var = dout.createVariable(k, stats[k].dtype, coordinates)
- var[:] = v[:]
-
- din.close()
- dout.close()
- return outFile
-
-
-def totalStats(args):
- urlFile = args[0]
- with open(urlFile, 'r') as f:
- urls = [line.strip() for line in f]
- variable = args[1]
- mode = args[2]
- nPartitions = int(args[3])
- outFile = args[4]
- stats = pixelStats(urls, variable, nPartitions, mode=mode)
- outFile = writeStats(urls, variable, stats, outFile)
- return outFile
-
-def main(args):
- return totalStats(args)
-
-if __name__ == '__main__':
- print(main(sys.argv[1:]))
-
-
-# python pixelStats.py urls_sst_daynight_2003_3days.txt sst sequential 1 modis_sst_stats_test.nc
-# python pixelStats.py urls_sst_daynight_2003_4months.txt sst sequential 1 modis_sst_stats_test.nc
-# python pixelStats.py urls_sst_daynight_2003_4months.txt sst dpark 4 modis_sst_stats_test.nc
-# python pixelStats.py urls_sst_daynight_2003_4months.txt sst spark 4 modis_sst_stats_test.nc
-
-# python pixelStats.py urls_sst_daynight_2003_2015.txt sst dpark 16 modis_sst_stats.nc
-# python pixelStats.py urls_sst_daynight_2003_2015.txt sst spark 16 modis_sst_stats.nc
-
diff --git a/climatology/clim/plotlib.py b/climatology/clim/plotlib.py
deleted file mode 100644
index a54f8a9..0000000
--- a/climatology/clim/plotlib.py
+++ /dev/null
@@ -1,857 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#!/bin/env python
-
-import sys, os, math, types, time, datetime
-from urllib.request import urlopen
-from urllib.request import urlretrieve
-from urllib.parse import urlparse
-#import Numeric as N
-import numpy as N
-import numpy.ma as MA
-#import matplotlib.numerix.ma as MA
-import matplotlib
-matplotlib.use('Agg')
-import matplotlib.pylab as M
-from mpl_toolkits.basemap import Basemap
-#import hdfeos
-
-def echo2(*s): sys.stderr.write(' '.join(map(str, s)) + '\n')
-def warn(*s): echo2('plotlib:', *s)
-def die(*s): warn('Error,', *s); sys.exit()
-
-CmdOptions = {'MCommand': ['title', 'xlabel', 'ylabel', 'xlim', 'ylim', 'show'],
- 'plot': ['label', 'linewidth', 'legend', 'axis'],
- 'map.plot': ['label', 'linewidth', 'axis'],
- 'map.scatter': ['norm', 'alpha', 'linewidths', 'faceted', 'hold'],
- 'savefig': ['dpi', 'orientation']
- }
-
-def imageMap(lons, lats, vals, vmin=None, vmax=None,
- imageWidth=None, imageHeight=None, outFile=None,
- projection='cyl', cmap=M.cm.jet, makeFigure=False,
- borders=[0., -90., 360., 90.], autoBorders=True, borderSlop=10.,
- meridians=[0, 360, 60], parallels=[-60, 90, 30],
- **options
- ):
- lons = normalizeLons(lons)
- if vmin == 'auto': vmin = None
- if vmax == 'auto': vmax = None
- if imageWidth is not None: makeFigure = True
- if projection is None or projection == '': projection = 'cyl'
- if cmap is None or cmap == '': cmap = M.cm.jet
- if isinstance(cmap, bytes) and cmap != '':
- try:
- cmap = eval('M.cm.' + cmap)
- except:
- cmap = M.cm.jet
-
-# ensureItems(options, {'xlabel': 'Longitude (deg)', 'ylabel': 'Latitude (deg)', \
- ensureItems(options, { \
- 'title': 'An Image Map', 'dpi': 100,
- 'imageWidth': imageWidth or 1024, 'imageHeight': imageHeight or 768})
- if autoBorders:
- borders = [min(lons), min(lats), max(lons), max(lats)]
- borders = roundBorders(borders, borderSlop)
-
- m = Basemap(borders[0], borders[1], borders[2], borders[3], \
- projection=projection, lon_0=N.average([lons[0], lons[-1]]))
-
- if makeFigure:
- dpi = float(options['dpi'])
- width = float(imageWidth) / dpi
- if imageHeight is None:
- height = width * m.aspect
- else:
- height = float(imageHeight) / dpi
- f = M.figure(figsize=(width,height)).add_axes([0.1,0.1,0.8,0.8], frameon=True)
-
- if vmin is not None or vmax is not None:
- if vmin is None:
- vmin = min(vals)
- else:
- vmin = float(vmin)
- if vmax is None:
- vmax = max(vals)
- else:
- vmax = float(vmax)
- vrange = (vmax - vmin) / 255.
- levels = N.arange(vmin, vmax, vrange/30.)
- else:
- levels = 30
-
- x, y = m(*M.meshgrid(lons,lats))
- c = m.contourf(x, y, vals, levels, cmap=cmap, colors=None)
-
- m.drawcoastlines()
- m.drawmeridians(list(range(meridians[0], meridians[1], meridians[2])), labels=[0,0,0,1])
- m.drawparallels(list(range(parallels[0], parallels[1], parallels[2])), labels=[1,1,1,1])
- M.colorbar(c)
- evalKeywordCmds(options)
- if outFile:
- M.savefig(outFile, **validCmdOptions(options, 'savefig'))
-
-
-# all of these calls below are handled by the evalKeywordCmds line above
-# M.xlim(0,360)
-# M.xlabel(xlabel)
-# M.ylabel(ylabel)
-# M.title(title)
-# M.show()
-
-
-def imageMap2(lons, lats, vals, vmin=None, vmax=None,
- imageWidth=None, imageHeight=None, outFile=None,
- projection='cyl', cmap=M.cm.jet, makeFigure=False,
- borders=[0., -90., 360., 90.], autoBorders=True, borderSlop=10.,
- meridians=[0, 360, 60], parallels=[-60, 90, 30],
- **options
- ):
-# lons = normalizeLons(lons)
- if vmin == 'auto': vmin = None
- if vmax == 'auto': vmax = None
- if imageWidth is not None: makeFigure = True
- if projection is None or projection == '': projection = 'cyl'
- if cmap is None or cmap == '': cmap = M.cm.jet
- if isinstance(cmap, bytes) and cmap != '':
- try:
- cmap = eval('M.cm.' + cmap)
- except:
- cmap = M.cm.jet
-
-# ensureItems(options, {'xlabel': 'Longitude (deg)', 'ylabel': 'Latitude (deg)', \
- ensureItems(options, { \
- 'title': 'An Image Map', 'dpi': 100,
- 'imageWidth': imageWidth or 1024, 'imageHeight': imageHeight or 768})
- if autoBorders:
- borders = [min(min(lons)), min(min(lats)), max(max(lons)), max(max(lats))]
- borders = roundBorders(borders, borderSlop)
-
- m = Basemap(borders[0], borders[1], borders[2], borders[3], \
- projection=projection, lon_0=N.average([lons.flat[0], lons.flat[-1]]))
-
- if makeFigure:
- dpi = float(options['dpi'])
- width = float(imageWidth) / dpi
- if imageHeight is None:
- height = width * m.aspect
- else:
- height = float(imageHeight) / dpi
- f = M.figure(figsize=(width,height)).add_axes([0.1,0.1,0.8,0.8], frameon=True)
-
- if vmin is not None or vmax is not None:
- if vmin is None:
- vmin = min(min(vals))
- else:
- vmin = float(vmin)
- if vmax is None:
- vmax = max(max(vals))
- else:
- vmax = float(vmax)
- vrange = vmax - vmin
- levels = N.arange(vmin, vmax, vrange/30.)
- else:
- levels = 30
-
- c = m.contourf(lons, lats, vals, levels, cmap=cmap, colors=None)
- m.drawcoastlines()
- m.drawmeridians(list(range(meridians[0], meridians[1], meridians[2])), labels=[0,0,0,1])
- m.drawparallels(list(range(parallels[0], parallels[1], parallels[2])), labels=[1,1,1,1])
- M.colorbar(c, orientation='horizontal')
- evalKeywordCmds(options)
- if outFile: M.savefig(outFile, **validCmdOptions(options, 'savefig'))
-
-
-def image2(vals, vmin=None, vmax=None, outFile=None,
- imageWidth=None, imageHeight=None, upOrDown='upper',
- cmap=M.cm.jet, makeFigure=False, **options
- ):
- M.clf()
- M.axes([0, 0, 1, 1])
- if vmin == 'auto': vmin = None
- if vmax == 'auto': vmax = None
- if imageWidth is not None: makeFigure = True
- if cmap is None or cmap == '': cmap = M.cm.jet
- if isinstance(cmap, bytes) and cmap != '':
- try:
- cmap = eval('M.cm.' + cmap)
- except:
- cmap = M.cm.jet
-
- if makeFigure:
- dpi = float(options['dpi'])
- width = float(imageWidth) / dpi
- height = float(imageHeight) / dpi
- f = M.figure(figsize=(width,height)).add_axes([0.1,0.1,0.8,0.8], frameon=True)
-
- if vmin is not None or vmax is not None:
- if vmin is None:
- vmin = min(min(vals))
- else:
- vmin = float(vmin)
- if vmax is None:
- vmax = max(max(vals))
- else:
- vmax = float(vmax)
- vrange = vmax - vmin
- levels = N.arange(vmin, vmax, vrange/30.)
- else:
- levels = 30
-
- M.contourf(vals, levels, cmap=cmap, origin=upOrDown)
- evalKeywordCmds(options)
- if outFile: M.savefig(outFile, **validCmdOptions(options, 'savefig'))
-
-
-def marksOnMap(lons, lats, vals=None, vmin=None, vmax=None,
- imageWidth=None, imageHeight=None, outFile=None,
- projection='cyl', cmap=M.cm.jet,
- sizes=80, colors='b', marker='o', makeFigure=False,
- times=None, timeDelta=None,
- borders=[0., -90., 360., 90.], autoBorders=True, borderSlop=10.,
- meridians=[0, 360, 60], parallels=[-60, 90, 30],
- **options
- ):
- lons = normalizeLons(lons)
- if imageWidth is not None: makeFigure = True
- if projection is None: projection = 'cyl'
-
-# ensureItems(options, {'xlabel': 'Longitude (deg)', 'ylabel': 'Latitude (deg)',
- ensureItems(options, { \
- 'title': 'Markers on a Map', 'dpi': 100,
- 'imageWidth': imageWidth or 1024, 'imageHeight': imageHeight or 768})
- if autoBorders:
- borders = [min(lons), min(lats), max(lons), max(lats)]
- borders = roundBorders(borders, borderSlop)
-
- m = Basemap(borders[0], borders[1], borders[2], borders[3], \
- projection=projection, lon_0=N.average([lons[0], lons[-1]]))
-
- if makeFigure:
- dpi = float(options['dpi'])
- width = float(imageWidth) / dpi
- if imageHeight is None:
- height = width * m.aspect
- else:
- height = float(imageHeight) / dpi
- f = M.figure(figsize=(width,height)).add_axes([0.1,0.1,0.8,0.8],frameon=True)
-
- if vals is not None:
- if vmin is None or vmin == 'auto':
- vmin = min(vals)
- warn('auto scale min is %f' % vmin)
- else:
- vmin = float(vmin)
- if vmax is None or vmax == 'auto':
- vmax = max(vals)
- else:
- vmax = float(vmax)
- warn('auto scale max is %f' % vmax)
- vrange = (vmax - vmin) / 255.
- colors = N.array([(val - vmin)/vrange for val in vals])
-
- if timeDelta is not None:
- if times is not None:
- times = list(map(mkTime, times))
- timeDelta = float(timeDelta) * 60.
- start = roundTime(times[0], 'down', timeDelta)
- end = roundTime(times[-1], 'up', timeDelta)
- plotTimes = arange(start, end+timeDelta, timeDelta)
- else:
- timeDelta = None
-
-
- g = m.scatter(lons, lats, s=sizes, c=colors, marker=marker, cmap=cmap,
- vmin=vmin, vmax=vmax, **validCmdOptions(options, 'map.scatter'))
-
- m.drawcoastlines()
- m.drawmeridians(list(range(meridians[0], meridians[1], meridians[2])), labels=[0,0,0,1])
- m.drawparallels(list(range(parallels[0], parallels[1], parallels[2])), labels=[1,1,1,1])
- M.colorbar(g)
- evalKeywordCmds(options)
- if outFile: M.savefig(outFile, **validCmdOptions(options, 'savefig'))
-
-
-def plotSwathVar(granules, variable, scaleFactor, title, outFile, filterMin=None, filterMax=None,
- scaleMin=None, scaleMax=None, imageWidth=None, imageHeight=None,
- plotType='map', projection='cyl', markerSize=10, **options):
- if filterMin == 'auto': filterMin = None
- if filterMax == 'auto': filterMax = None
-# files = [localize(url) for url in granules if url != 'None']
- files = granules
- imageFiles = []
-
- for i, file in enumerate(files):
- print('plotSwathVar: Reading %s: %s' % (file, variable))
- localFile = localize(file, retrieve=False)
- if i == 0:
- swath = hdfeos.swaths(file)[0]
-# geoFields = hdfeos.swath_geo_fields(file, swath)
- lat = hdfeos.swath_field_read(file, swath, 'Latitude')
- lon = hdfeos.swath_field_read(file, swath, 'Longitude')
-### time = hdfeos.swath_field_read(file, swath, 'Time')
-### pressure = hdfeos.swath_field_read(file, swath, '???')
-
- if N.minimum.reduce(lon.flat) < -360. or N.minimum.reduce(lat.flat) < -90.:
- useImageMap = False # have missing values in lat/lon coord variables
- else:
- useImageMap = True
-
- dataFields = hdfeos.swath_data_fields(file, swath)
- if '[' not in variable:
- varName = variable
- else:
- varName, slice = variable.split('[')
- if varName not in dataFields:
- die('%s not a variable in %s' % (variable, file))
- if '[' not in variable:
- var = hdfeos.swath_field_read(file, swath, variable) * float(scaleFactor)
- else:
- vals = hdfeos.swath_field_read(file, swath, varName)
- var = eval('['.join(('vals', slice)))
- var = var * float(scaleFactor)
-
- print('plotSwathVar: Variable range: %f -> %f' % (min(min(var)), max(max(var))))
- if plotType != 'map' or not useImageMap:
- lat = lat.flat; lon = lon.flat; var = var.flat
-
- if filterMin is not None or filterMax is not None:
- if filterMin is not None and filterMax is None:
- cond = N.greater(var, float(filterMin))
- elif filterMin is None and filterMax is not None:
- cond = N.less(var, float(filterMax))
- else:
- cond = N.logical_and(N.greater(var, float(filterMin)),
- N.less(var, float(filterMax)))
- if plotType == 'map' and useImageMap:
- lat = MA.masked_where(cond, lat, copy=0)
- lon = MA.masked_where(cond, lon, copy=0)
- var = MA.masked_where(cond, var, copy=0)
- else:
- lat = N.compress(cond, lat.flat)
- lon = N.compress(cond, lon.flat)
- var = N.compress(cond, var.flat)
-
- if plotType == 'map':
- imageFile = localFile + '_map.png'
- if useImageMap:
- imageMap2(lon, lat, var, scaleMin, scaleMax, imageWidth, imageHeight,
- imageFile, projection, autoBorders=False, title=title+' '+file,
- **options)
- else:
- marksOnMap(lon, lat, var, scaleMin, scaleMax,
- imageWidth, imageHeight, imageFile, projection,
- autoBorders=True, title=title+' '+file,
- sizes=markerSize*markerSize, **options)
- elif plotType == 'hist':
- imageFile = localFile + '_aot_hist.png'
- hist(var, 50, imageFile)
- else:
- die("plotSwathVar: plotType must be 'map' or 'hist'")
-
- imageFiles.append(imageFile)
- return makeMovie(imageFiles, outFile)
-
-
-def imageSwathVar(granules, variable, scaleFactor, title, outFile, filterMin=None, filterMax=None,
- scaleMin=None, scaleMax=None, imageWidth=None, imageHeight=None,
- plotType='map', projection='cyl', markerSize=10, **options):
- if filterMin == 'auto': filterMin = None
- if filterMax == 'auto': filterMax = None
-# files = [localize(url) for url in granules if url != 'None']
- files = granules
- imageFiles = []; lonLatBounds = []
-
- for i, file in enumerate(files):
- print('imageSwathVar: Reading %s: %s' % (file, variable))
- localFile = localize(file, retrieve=False)
- if i == 0:
- swath = hdfeos.swaths(file)[0]
-# geoFields = hdfeos.swath_geo_fields(file, swath)
- lat = hdfeos.swath_field_read(file, swath, 'Latitude')
- lon = hdfeos.swath_field_read(file, swath, 'Longitude')
-### time = hdfeos.swath_field_read(file, swath, 'Time')
-### pressure = hdfeos.swath_field_read(file, swath, '???')
-
- if N.minimum.reduce(lon.flat) < -360. or N.minimum.reduce(lat.flat) < -90.:
- useImageMap = False # have missing values in lat/lon coord variables
- else:
- useImageMap = True
-
- dataFields = hdfeos.swath_data_fields(file, swath)
- if '[' not in variable:
- varName = variable
- else:
- varName, slice = variable.split('[')
- if varName not in dataFields:
- die('%s not a variable in %s' % (variable, file))
- if '[' not in variable:
- var = hdfeos.swath_field_read(file, swath, variable) * float(scaleFactor)
- else:
- vals = hdfeos.swath_field_read(file, swath, varName)
- var = eval('['.join(('vals', slice)))
- var = var * float(scaleFactor)
-
- print('imageSwathVar: Variable range: %f -> %f' % (min(min(var)), max(max(var))))
- if plotType != 'map' or not useImageMap:
- lat = lat.flat; lon = lon.flat; var = var.flat
-
- if filterMin is not None or filterMax is not None:
- if filterMin is not None and filterMax is None:
- cond = N.greater(var, float(filterMin))
- elif filterMin is None and filterMax is not None:
- cond = N.less(var, float(filterMax))
- else:
- cond = N.logical_and(N.greater(var, float(filterMin)),
- N.less(var, float(filterMax)))
- if plotType == 'map' and useImageMap:
- lat = MA.masked_where(cond, lat, copy=0)
- lon = MA.masked_where(cond, lon, copy=0)
- var = MA.masked_where(cond, var, copy=0)
- else:
- lat = N.compress(cond, lat.flat)
- lon = N.compress(cond, lon.flat)
- var = N.compress(cond, var.flat)
-
- lonLatBound = (min(min(lon)), min(min(lat)), max(max(lon)), max(max(lat)))
- lonLatBounds.append(lonLatBound)
-
- if plotType == 'map':
- imageFile = localFile + '_image.png'
- if useImageMap:
- upOrDown = 'upper'
- if lat[0, 0] < lat[-1, 0]: upOrDown = 'lower'
- if lon[0, 0] > lon[0, -1]: var = fliplr(var)
- image2(var, scaleMin, scaleMax, imageFile, upOrDown=upOrDown, **options)
-# plainImage2(var, imageFile)
- else:
- marksOnMap(lon, lat, var, scaleMin, scaleMax,
- imageWidth, imageHeight, imageFile, projection,
- autoBorders=True, title=title+' '+file,
- sizes=markerSize*markerSize, **options)
- elif plotType == 'hist':
- imageFile = localFile + '_aot_hist.png'
- hist(var, 50, imageFile)
- else:
- die("plotSwathVar: plotType must be 'map' or 'hist'")
-
- imageFiles.append(imageFile)
- print("imageSwathVar results:", imageFiles)
- return (imageFiles, lonLatBounds)
-
-def fliplr(a):
- b = N.array(a)
- for i in range(b.shape[0]):
- b[i,:] = a[i,::-1]
- return b
-
-def plainImage2(var, imageFile):
- M.clf()
- M.figimage(var)
- M.savefig(imageFile)
-
-
-def maskNcVar(inNcFile, outNcFile, outVars=[], writeMode='include',
- conditionVar=None, filterMin=None, filterMax=None,
- varToMask=None, maskValue=-9999.):
- nc = NC(inNcFile)
- if conditionVar is not None:
- try:
- condVar = nc.variables[conditionVar]
- except:
- die('maskNcVar: Variable %s not in ncFile %s' % (conditionVar, inNcFile))
- if varToMask is None: die('maskNcVar: Must specify variable to mask with condition.')
- try:
- var = nc.variables[varToMask]
- except:
- die('maskNcVar: Variable %s not in ncFile %s' % (varToMask, inNcFile))
-
- if filterMin is not None or filterMax is not None:
- if filterMin is not None and filterMax is None:
- cond = N.greater(var, float(filterMin))
- elif filterMin is None and filterMax is not None:
- cond = N.less(var, float(filterMax))
- else:
- cond = N.logical_and(N.greater(var, float(filterMin)),
- N.less(var, float(filterMax)))
- var = N.putmask(var, cond, float(maskValue))
- outVars = list( set(outVars).add(conditionVar).add(varToMask) )
- return subsetNcFile(inNcFile, outVars, outNcFile)
-
-def subsetNcFile(inNcFile, outNcFile, outVars, writeMode='include'):
- if writeMode == '' or writeMode == 'auto': writeMode = 'include'
- inf = NC(inNcFile)
- outf = NC(outNcFile, 'w')
- return outNcFile
-
-
-def hist(x, bins, outFile=None, **options):
- if outFile: M.clf()
- M.hist(x, bins, **options)
- if outFile: M.savefig(outFile, **validCmdOptions(options, 'savefig'))
-
-def localize(url, retrieve=True):
- scheme, netloc, path, params, query, frag = urlparse(url)
- dir, file = os.path.split(path)
- if retrieve: urlretrieve(url, file)
- return file
-
-def makeMovie(files, outFile):
- if len(files) > 1:
- outMovieFile = os.path.splitext(outFile)[0] + '.mpg'
- cmd = 'convert ' + ' '.join(files) + ' ' + outMovieFile
- os.system(cmd)
- warn('Wrote movie ' + outMovieFile)
- return outMovieFile
- else:
- os.rename(files[0], outFile)
- return outFile
-
-def mkTime(timeStr):
- """Make a time object from a date/time string YYYY-MM-DD HH:MM:SS"""
- from time import mktime, strptime
- return mktime( strptime(timeStr, '%Y %m %d %H %M %S') )
-
-def roundTime(time, resolution, direction):
- pass
-
-def roundBorders(borders, borderSlop=10.):
- b0 = roundBorder(borders[0], 'down', borderSlop, 0.)
- b1 = roundBorder(borders[1], 'down', borderSlop, -90.)
- b2 = roundBorder(borders[2], 'up', borderSlop, 360.)
- b3 = roundBorder(borders[3], 'up', borderSlop, 90.)
- return [b0, b1, b2, b3]
-
-def roundBorder(val, direction, step, end):
- if direction == 'up':
- rounder = math.ceil
- slop = step
- else:
- rounder = math.floor
- slop = -step
-### v = rounder(val/step) * step + slop
- v = rounder(val/step) * step
- if abs(v - end) < step+1.: v = end
- return v
-
-def normalizeLon(lon):
- if lon < 0.: return lon + 360.
- if lon > 360.: return lon - 360.
- return lon
-
-def normalizeLons(lons):
- return N.array([normalizeLon(lon) for lon in lons])
-
-
-def plotColumns(specs, groupBy=None, outFile=None, rmsDiffFrom=None, floatFormat=None,
- colors='bgrcmyk', markers='+x^svD<4>3', **options):
- if groupBy:
- plotColumnsGrouped(specs, groupBy, outFile, rmsDiffFrom, floatFormat,
- colors, markers, **options)
- else:
- plotColumnsSimple(specs, outFile, rmsDiffFrom, floatFormat,
- colors, markers, **options)
-
-
-def plotColumnsSimple(specs, outFile=None, rmsDiffFrom=None, floatFormat=None,
- colors='bgrcmyk', markers='+x^svD<4>3', **options):
- """Plot olumns of numbers from one or more data files.
- Each plot spec. contains a filename and a list of labelled columns:
- e.g., ('file1', 'xlabel:1,ylabel1:4,ylabel2:2,ylabel3:13)
- Bug: For the moment, only have 7 different colors and 10 different markers.
- """
- ensureItems(options, {'legend': True})
- ydataMaster = None
- for spec in specs:
- file, columns = spec # each spec is a (file, columnList) pair
- columns = columns.split(',') # each columnList is a comma-separated list of named columns
- # Each named column is a colon-separated pair or triple 'label:integer[:style]'
- # Column indices are one-based.
- # Styles are concatenated one-char flags like 'go' for green circles or
- # 'kx-' for black X's with a line.
- fields = N.array([list(map(floatOrMiss, line.split())) for line in open(file, 'r')])
- xcol = columns.pop(0) # first column in list is the x axis
- xlabel, xcol, xstyle = splitColumnSpec(xcol)
- xdata = fields[:,xcol-1]
- markIndex = 0
- for ycol in columns:
- ylabel, ycol, ystyle = splitColumnSpec(ycol)
- if ystyle is None: ystyle = colors[markIndex] + markers[markIndex]
- ydata = fields[:,ycol-1] # all other columns are multiple y plots
- if rmsDiffFrom:
- if ydataMaster is None:
- ydataMaster = ydata # kludge: must be first ycol in first file
- ylabelMaster = ylabel
- else:
- s = diffStats(ylabelMaster, ydataMaster, ylabel, ydata)
- print(s.format(floatFormat), file=sys.stderr)
- n, mean, sigma, min, max, rms = s.calc()
- ylabel = ylabel + ' ' + floatFormat % rms
- M.plot(xdata, ydata, ystyle, label=ylabel)
- markIndex += 1
- evalKeywordCmds(options)
- if outFile: M.savefig(outFile)
-
-
-def plotColumnsGrouped(specs, groupBy, outFile=None, rmsDiffFrom=None, floatFormat=None,
- colors='bgrcmyk', markers='+x^svD<4>3', **options):
- """Plot olumns of numbers from one or more data files.
- Each plot spec. contains a filename and a list of labelled columns:
- e.g., ('file1', 'xlabel:1,ylabel1:4,ylabel2:2,ylabel3:13)
- Bug: For the moment, only have 7 different colors and 10 different markers.
- """
- ensureItems(options, {'legend': True})
- ydataMaster = None
- for spec in specs:
- file, columns = spec # each spec is a (file, columnList) pair
- columns = columns.split(',') # each columnList is a comma-separated list of named columns
- # Each named column is a colon-separated pair or triple 'label:integer[:style]'
- # Column indices are one-based.
- # Styles are concatenated one-char flags like 'go' for green circles or
- # 'kx-' for black X's with a line.
- fields = N.array([list(map(floatOrMiss, line.split())) for line in open(file, 'r')])
- xcol = columns.pop(0) # first column in list is the x axis
- xlabel, xcol, xstyle = splitColumnSpec(xcol)
- xdata = fields[:,xcol-1]
- markIndex = 0
- for ycol in columns:
- ylabel, ycol, ystyle = splitColumnSpec(ycol)
- if ystyle is None: ystyle = colors[markIndex] + markers[markIndex]
- ydata = fields[:,ycol-1] # all other columns are multiple y plots
- if rmsDiffFrom:
- if ydataMaster is None:
- ydataMaster = ydata # kludge: must be first ycol in first file
- ylabelMaster = ylabel
- else:
- s = diffStats(ylabelMaster, ydataMaster, ylabel, ydata)
- print(s.format(floatFormat), file=sys.stderr)
- n, mean, sigma, min, max, rms = s.calc()
- ylabel = ylabel + ' ' + floatFormat % rms
- M.plot(xdata, ydata, ystyle, label=ylabel)
- markIndex += 1
- evalKeywordCmds(options)
- if outFile: M.savefig(outFile)
-
-
-def plotTllv(inFile, markerType='kx', outFile=None, groupBy=None, **options):
- """Plot the lat/lon locations of points from a time/lat/lon/value file."""
- fields = N.array([list(map(float, line.split())) for line in open(inFile, 'r')])
- lons = fields[:,2]; lats = fields[:,1]
- marksOnMap(lons, lats, markerType, outFile, \
- title='Lat/lon plot of '+inFile, **options)
-
-
-def plotVtecAndJasonTracks(gtcFiles, outFile=None, names=None, makeFigure=True, show=False, **options):
- """Plot GAIM climate and assim VTEC versus JASON using at least two 'gc' files.
- First file is usually climate file, and rest are assim files.
- """
- ensureItems(options, {'title': 'GAIM vs. JASON for '+gtcFiles[0], \
- 'xlabel': 'Geographic Latitude (deg)', 'ylabel': 'VTEC (TECU)'})
- if 'show' in options:
- show = True
- del options['show']
- M.subplot(211)
- gtcFile = gtcFiles.pop(0)
- name = 'clim_'
- if names: name = names.pop(0)
- specs = [(gtcFile, 'latitude:2,jason:6,gim__:8,%s:13,iri__:10' % name)]
- name = 'assim'
- for i, gtcFile in enumerate(gtcFiles):
- label = name
- if len(gtcFiles) > 1: label += str(i+1)
- specs.append( (gtcFile, 'latitude:2,%s:13' % label) )
- plotColumns(specs, rmsDiffFrom='jason', floatFormat='%5.1f', **options)
- M.legend()
-
- M.subplot(212)
- options.update({'title': 'JASON Track Plot', 'xlabel': 'Longitude (deg)', 'ylabel': 'Latitude (deg)'})
- fields = N.array([list(map(floatOrMiss, line.split())) for line in open(gtcFiles[0], 'r')])
- lons = fields[:,2]; lats = fields[:,1]
- marksOnMap(lons, lats, show=show, **options)
- if outFile: M.savefig(outFile)
-
-
-def diffStats(name1, vals1, name2, vals2):
- """Compute RMS difference between two Numeric vectors."""
- from Stats import Stats
- label = name2 + ' - ' + name1
- diff = vals2 - vals1
- return Stats().label(label).addm(diff)
-
-def ensureItems(d1, d2):
- for key in list(d2.keys()):
- if key not in d1: d1[key] = d2[key]
-
-def splitColumnSpec(s):
- """Split column spec 'label:integer[:style]' into its 2 or 3 parts."""
- items = s.split(':')
- n = len(items)
- if n < 2:
- die('plotlib: Bad column spec. %s' % s)
- elif n == 2:
- items.append(None)
- items[1] = int(items[1])
- return items
-
-def floatOrMiss(val, missingValue=-999.):
- try: val = float(val)
- except: val = missingValue
- return val
-
-def floatOrStr(val):
- try: val = float(val)
- except: pass
- return val
-
-def evalKeywordCmds(options, cmdOptions=CmdOptions):
- for option in options:
- if option in cmdOptions['MCommand']:
- args = options[option]
- if args:
- if args is True:
- args = ''
- else:
- args = "'" + args + "'"
- if option in cmdOptions:
- args += dict2kwargs( validCmdOptions(options, cmdOptions[option]) )
- try:
- eval('M.' + option + '(%s)' % args)
- except:
- die('failed eval of keyword command option failed: %s=%s' % (option, args))
-# else:
-# warn('Invalid keyword option specified" %s=%s' % (option, args))
-
-def validCmdOptions(options, cmd, possibleOptions=CmdOptions):
- return dict([(option, options[option]) for option in list(options.keys())
- if option in possibleOptions[cmd]])
-
-def dict2kwargs(d):
- args = [',%s=%s' % (kw, d[kw]) for kw in d]
- return ', '.join(args)
-
-def csv2columns(csvFile, columns):
- """Given a CSV file and a comma-separated list of desired column names and
- types (name:type), return an array of vectors containing type-converted data.
-
- Type should be the name of string-to-val type-conversion function such as float, int, or str.
- If type is missing, then float conversion is assumed.
- """
- import csv
- names = []; types = []; cols = []
- for column in columns.split(','):
- if column.find(':') > 0:
- name, type = column.split(':')
- else:
- name = column; type = 'float'
- names.append(name.strip())
- types.append( eval(type.strip()) ) # get type conversion function from type string
- cols.append([])
-
- print(csvFile)
- for fields in csv.DictReader(urlopen(csvFile).readlines(), skipinitialspace=True):
- tmpColVals = []
- try:
- for i, type in enumerate(types): tmpColVals.append( type(fields[names[i]]) )
- except Exception as e:
- print("Got exception coercing values: %s" % e)
- continue
- for i in range(len(types)): cols[i].append(tmpColVals[i])
- return [N.array(col) for col in cols]
-
-def csv2marksOnMap(csvFile, columns, title, vmin=None, vmax=None,
- imageWidth=None, imageHeight=None, mapFile=None, projection='cyl'):
- if mapFile is None: mapFile = csvFile + '.map.png'
- lons, lats, vals = csv2columns(csvFile, columns)
- marksOnMap(lons, lats, vals, vmin, vmax, imageWidth, imageHeight,
- mapFile, projection, title=title)
- return mapFile
-
-def imageSlice(dataFile, lonSlice, latSlice, varSlice, title, vmin=None, vmax=None,
- imageWidth=None, imageHeight=None, imageFile=None,
- projection='cyl', colormap=M.cm.jet):
-# if dataFile == []: dataFile = 'http://laits.gmu.edu/NWGISS_Temp_Data/WCSfCays5.hdf'
- if imageFile is None: imageFile = dataFile + '.image.png'
-
- print(dataFile)
- print(imageFile)
- tmpFile, headers = urlretrieve(dataFile)
-# tmpFile = 'WCSfCays5.hdf'
- try:
- from Scientific.IO.NetCDF import NetCDFFile as NC
- nc = NC(tmpFile)
- fileType = 'nc'
- except:
- try:
- import hdfeos
- grids = hdfeos.grids(tmpFile)
- fileType = 'hdf_grid'
- grid = grids[0]
- except:
- try:
- swaths = hdfeos.swaths(tmpFile)
- fileType = 'hdf_swath'
- swath = swaths[0]
- except:
- raise RuntimeError('imageSlice: Can only slice & image netCDF or HDF grid/swath files.')
- if fileType == 'nc':
- lons = evalSlice(nc, lonSlice)
- lats = evalSlice(nc, latSlice)
- vals = evalSlice(nc, varSlice)
- elif fileType == 'hdf_grid':
- print(grids)
- fields = hdfeos.grid_fields(tmpFile, grid)
- print(fields)
- field = fields[0]
- dims = hdfeos.grid_field_dims(tmpFile, grid, field)
- print(dims)
- lons = hdfeos.grid_field_read(tmpFile, grid, lonSlice)
- lats = hdfeos.grid_field_read(tmpFile, grid, latSlice)
- vals = hdfeos.grid_field_read(tmpFile, grid, field)
- elif fileType == 'hdf_swath':
- print(swaths)
- print(hdfeos.swath_geo_fields(tmpFile, swath))
- print(hdfeos.swath_data_fields(tmpFile, swath))
- lons = hdfeos.get_swath_field(tmpFile, swath, 'Longitude') # assume HDFEOS conventions
- lats = hdfeos.get_swath_field(tmpFile, swath, 'Latitude')
- vals = hdfeos.get_swath_field(tmpFile, swath, varSlice)
-
- imageMap(lons, lats, vals, vmin, vmax, imageWidth, imageHeight, imageFile,
- title=title, projection=projection, cmap=colormap)
- return imageFile
-
-def evalSlice(nc, varSlice):
- if varSlice.find('[') > 0:
- varName, slice = varSlice.split('[')
- vals = nc.variables[varName]
- vals = eval('['.join(('vals', slice)))
- else:
- vals = nc.variables[varSlice][:]
- return vals
-
-
-if __name__ == '__main__':
- from sys import argv
-# lons = N.arange(0, 361, 2, N.Float)
-# lats = N.arange(-90, 91, 1, N.Float)
-# data = N.fromfunction( lambda x,y: x+y, (len(lats), len(lons)))
-
- outFile = 'out.png'
-# imageMap(lons, lats, data, outFile)
-# marksOnMap(lons, lats, 'bx', outFile)
-# plotVtecAndJasonTracks([argv[1], argv[2]], outFile, show=True, legend=True)
-
- me = argv.pop(0)
- args = list(map(floatOrStr, argv))
- imageSlice(*args)
diff --git a/climatology/clim/reroot.py b/climatology/clim/reroot.py
deleted file mode 100644
index ac723cf..0000000
--- a/climatology/clim/reroot.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-'''
- reroot.py -- Change the root of the URL for a list of files
-
-'''
-
-import sys, os
-import urllib.parse
-
-AIRS_DAP = 'http://airspar1.gesdisc.eosdis.nasa.gov/opendap/Aqua_AIRS'
-AIRS_FTP = 'ftp://airsl2.gesdisc.eosdis.nasa.gov/ftp/data/s4pa/Aqua_AIRS'
-# matchStart for this case is 'Aqua_AIRS'
-
-
-def reroot(url, root=AIRS_DAP, matchStart='Aqua_AIRS'):
- protocol, netloc, path, params, query, fragment = urllib.parse.urlparse(url)
- start = root[:root.index(matchStart)]
- rest = path[path.index(matchStart):-1]
- return start + rest
-
-
-def main(args):
-# root = args[0]
-# matchStart = args[1]
- root = AIRS_DAP
- matchStart = 'Aqua_AIRS'
- for url in sys.stdin:
- print(reroot(url, root, matchStart))
-
-
-if __name__ == '__main__':
- main(sys.argv[1:])
diff --git a/climatology/clim/setup.py b/climatology/clim/setup.py
deleted file mode 100644
index acde78d..0000000
--- a/climatology/clim/setup.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from setuptools import setup
-
-setup(name='clim',
- version='0.1dev0')
-
-
-
diff --git a/climatology/clim/sort.py b/climatology/clim/sort.py
deleted file mode 100644
index 32794e2..0000000
--- a/climatology/clim/sort.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# sort.py -- Utility routines to sort URL lists into N-day groups for computing climatologies.
-#
-
-import sys, os, urllib.parse
-
-
-def sortByKeys(urls, getKeysFn):
- '''Extract keys (e.g. DOY and year) from filename and sort by the keys.'''
- keyed = []
- for url in urls:
- if url is None: continue
- url = url.strip()
- if url == '': continue
- keyed.append( (getKeysFn(url), url) )
-
- keyed.sort()
- sort = [u[1] for u in keyed] # remove keys
- return sort
-
-
-def main(args):
- from .datasets import ModisSst
- urlFile = args[0]
- urls = open(urlFile, 'r').readlines()
- urlsSorted = sortByKeys(urls, ModisSst.getKeys)
- print('\n'.join(urlsSorted))
-
-
-if __name__ == '__main__':
- main(sys.argv[1:])
-
-
-# Get URL's for MODIS SST daily 4km netCDF files
-# wls is a ls command for the web. Understands FTP & HTTP root URL's and traverses directories. Can also retrieve all of the matching files.
-
-# python wls.py --wildcard 'A*sst*.nc' ftp://podaac.jpl.nasa.gov/OceanTemperature/modis/L3/aqua/11um/v2014.0/4km/daily > urls
-
-# Sort by DOY, year, N/D
-# python sort.py < urls > urls_sorted
-
-# Now have URL list that is in proper order to compute N-day climatologies.
-
diff --git a/climatology/clim/sparkTest.py b/climatology/clim/sparkTest.py
deleted file mode 100644
index 9aeba01..0000000
--- a/climatology/clim/sparkTest.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# sparkTest.py -- word count example to test installation
-#
-
-from pyspark import SparkContext
-
-sc = SparkContext(appName="PythonWordCount")
-#lines = sc.textFile("file:///home/code/climatology-gaussianInterp/README.md", 8)
-lines = sc.textFile("file:///usr/share/dict/words", 128)
-counts = lines.flatMap(lambda x: x.split(' ')) \
- .map(lambda x: (x, 1)) \
- .reduceByKey(lambda x,y: x + y)
-
-output = counts.collect()
-#for (word, count) in output:
-# print("%s: %i" % (word, count))
-
diff --git a/climatology/clim/spatialFilter.py b/climatology/clim/spatialFilter.py
deleted file mode 100644
index a6a036c..0000000
--- a/climatology/clim/spatialFilter.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# spatialFilter routine -- Apply a fixed spatial filter (smoother) in lat/lon and then average over times/grids
-#
-# Calls into optimized routine in Fortran (spatialFilter_f.f).
-#
-
-import numpy as N, time
-from spatialFilter_f import spatialfilter_f
-
-
-def spatialFilter(var, # bundle of input arrays: masked variable, coordinates
- varNames, # list of names in order: primary variable, coordinates in order lat, lon, time
- spatialFilter, # 3x3 filter numpy array of integers
- normalization, # normalization factor for filter (integer)
- missingValue=-9999., # value to mark missing values in interp result
- verbose=1, # integer to set verbosity level
- optimization='fortran'): # Mode of optimization, using 'fortran' or 'cython'
- '''Apply a fixed spatial filter (smoother) in lat/lon and then average over times/grids.
- '''
- # Prepare numpy arrays
- v = var[varNames[0]][:] # real*4 in Fortran code, is v.dtype correct?
- vmask = N.ma.getmask(v).astype('int8')[:] # integer*1, convert bool mask to one-byte integer for Fortran
- vtime = var[varNames[1]][:] # integer*4 in Fortran
- lat = var[varNames[2]][:] # real*4
- lon = var[varNames[3]][:] # real*4
-
- if optimization == 'fortran':
- vinterp, vcount, status = \
- spatialfilter_f(v, vmask, vtime, lat, lon,
- spatialFilter, normalization, missingValue, verbose)
- else:
- pass
-
- vinterp = N.ma.masked_where(vcount == 0, vinterp)
- return (vinterp, vcount, status)
-
diff --git a/climatology/clim/spatialFilter_f.f b/climatology/clim/spatialFilter_f.f
deleted file mode 100644
index 1b705b4..0000000
--- a/climatology/clim/spatialFilter_f.f
+++ /dev/null
@@ -1,121 +0,0 @@
-c
-c spatialFilter routine -- Apply a fixed spatial filter (smoother) in lat/lon and then average over times/grids
-c
-c Designed to be called from python using f2py.
-c
-c
-c Low Pass Filter = 1/9 * [1 1 1 ! normalization = 9
-c 1 1 1
-c 1 1 1]
-c
-c Gaussian Filter = 1/16 * [1 2 1 ! normalization = 16
-c 2 4 2
-c 1 2 1]
-c
-
- subroutine spatialFilter_f(var, mask,
- & time, lat, lon,
- & filter, normalization,
- & missingValue,
- & verbose,
- & vinterp, vcount, status,
- & ntime, nlat, nlon)
-
- implicit none
- integer*4 ntime, nlat, nlon
-c ! prepared 3D array of variable data over time, lon, & lat
- real*4 var(ntime, nlat, nlon)
-c ! variable to be interpolated, over ntime epochs
- integer*1 mask(ntime, nlat, nlon)
-c ! pixel quality mask
- integer*4 time(ntime)
-c ! time epochs to gaussian-interpolate over
- real*4 lat(nlat)
- ! latitude corrdinate vector
- real*4 lon(nlon)
- ! longitude corrdinate vector
-cf2py intent(in) var, mask, time, lat, lon !! annotations for f2py processor
-
- integer*4 filter(3, 3)
-c ! 3x3 filter coefficients as integers
- integer*4 normalization
-c ! Normalization factor for the filter, divide by sum of integers
-cf2py intent(in) filter, normalization
-
- real*4 missingValue
-c ! value to mark missing values in interp result
- integer*4 verbose
-c ! integer to set verbosity level
-cf2py intent(in) missingValue, verbose
-
- real*4 vinterp(nlat, nlon)
-c ! interpolated variable using gaussians, missing values not counted
- integer*4 vcount(nlat, nlon)
-c ! count of good data, might be zero after masking
- integer*4 status
-c ! negative status indicates error
-cf2py intent(out) vinterp, vcount, status
-
- integer*4 iin, jin, kin
- integer*4 i, j, fac, count
- real*4 val, sum
-
- write(6, *) 'Echoing inputs ...'
- write(6, *) 'ntime, nlat, nlon:', ntime, nlat, nlon
- write(6, *) 'filter:', filter
- write(6, *) 'normalization', normalization
- write(6, *) 'missingValue:', missingValue
-
- status = 0
-
- if (verbose .gt. 3) then
- write(6, *) 'time:', time
- write(6, *) 'lat:', lat
- write(6, *) 'lon:', lon
-c write(6, *) 'mask(3):', mask(3,:,:)
- write(6, *) 'var(3):', var(3,:,:)
- end if
-
- do i = 1, nlat
- if (verbose .gt. 1) write(6, *) lat(i)
- do j = 1, nlon
- vinterp(i,j) = 0.0
- vcount(i,j) = 0.0
- if (verbose .gt. 3) then
- write(6, *) '(i,j) = ', i, j
- write(6, *) '(lat,lon) = ', lat(i), lon(j)
- end if
-
- do kin = 1, ntime
- sum = 0.0
- count = 0
- do iin = -1, +1
- if (i+iin .lt. 1 .or. i+iin .gt. nlat) cycle
- do jin = -1, +1
- if (j+jin .lt. 1 .or. j+jin .gt. nlon) cycle
-
- if (mask(kin,iin,jin) .eq. 0) then
- fac = filter(iin+2, jin+2)
- val = var(kin,iin,jin)
- sum = sum + fac * val
- count = count + fac
- end if
- end do
- end do
- if (count .gt. 0) then
-c ! filter for (i,j) pixel isn't empty
- vinterp(i,j) = vinterp(i,j) + sum / normalization
- vcount(i,j) = vcount(i,j) + 1
- end if
- end do
- if (vcount(i,j) .gt. 0) then
- vinterp(i,j) = vinterp(i,j) / vcount(i,j)
-c ! compute mean over number of non-empty times/grids
- else
- vinterp(i,j) = missingValue
- end if
- end do
- end do
- return
- end
-
diff --git a/climatology/clim/spatialFilter_f.mk b/climatology/clim/spatialFilter_f.mk
deleted file mode 100644
index 9150b5b..0000000
--- a/climatology/clim/spatialFilter_f.mk
+++ /dev/null
@@ -1 +0,0 @@
-f2py -c -m spatialFilter_f spatialFilter_f.f
diff --git a/climatology/clim/split.py b/climatology/clim/split.py
deleted file mode 100755
index 9307957..0000000
--- a/climatology/clim/split.py
+++ /dev/null
@@ -1,212 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#!/usr/bin/env python
-
-"""
-split.py == Some utility functions to split lists of URL's into chunks or time periods.
-
-"""
-
-import sys, os, re, json
-import datetime
-
-
-def fixedSplit(seq, n):
- '''Split a sequence into fixed-length chunks of length N. Last chunk is different length.'''
- chunk = []
- for i, s in enumerate(seq):
- chunk.extend(s)
- if (i+1) % n == 0:
- yield chunk
- chunk = []
- if len(chunk) > 0: yield chunk
-
-
-def splitByMonth(seq, timeFromFilename={'get': 'doy', 'regex': re.compile(r'\/A.(....)(...)')}, transformer=None, keyed=True):
- '''Split URL's into months using regex to extract information from the filename. Return list or keyed dictionary.'''
- if timeFromFilename['get'][1] == 'doy':
- transformer = lambda keys: (keys[0], doy2month(*keys))
- urlsByMonth = [ku for ku in splitByKeys(seq, timeFromFilename['regex'], transformer, keyed)]
- return urlsByMonth
-
-
-def splitByKeys(seq, regex, transformer=None, keyed=True):
- '''Split a sequence into chunks by a key.
-The key is extracted from the string by matching a regular expression to the string and returning the matched groups.
- '''
- regex = re.compile(regex)
- chunk = []
- for i, s in enumerate(seq):
- s = s.strip()
- if i == 0:
- keys = extractKeys(s, regex, transformer)
- keys1 = extractKeys(s, regex, transformer)
- if keys1 != keys:
- if keyed:
- if len(keys) == 1:
- try:
- intKey = int(keys[0])
- yield (intKey, chunk)
- except:
- yield (keys, chunk)
- else:
- yield (keys, chunk)
- else:
- yield chunk
- chunk = [s]
- keys = keys1
- else:
- chunk.append(s)
- if len(chunk) > 0:
- if keyed:
- if len(keys) == 1:
- try:
- intKey = int(keys[0])
- yield (intKey, chunk)
- except:
- yield (keys, chunk)
- else:
- yield chunk
-
-
-def extractKeys(s, regex, transformer=None):
- '''Extract keys from a string by matching a regular expression to the string and returning the matched groups. Transformer functions alter the keys.'''
- regex = re.compile(regex)
- mat = regex.search(s)
- if not mat:
- print('extractKeys: Fatal error, regex %s does not match %s' % (regex.pattern, s), file=sys.stderr)
- sys.exit(1)
- else:
- keys = mat.groups()
- if transformer is not None:
- keys = transformer(keys)
- return keys
-
-
-def splitByNDays(seq, n, regex, transformer=None, keyed=True):
- '''Split URL's into N-day chunks.'''
- daily = [s for s in splitByKeys(seq, regex, transformer, keyed)]
- for chunk in fixedSplit(daily, n):
- yield chunk
-
-def splitByNDaysKeyed(seq, n, regex, transformer=None, keyed=True):
- '''Split URL's into N-day chunks.'''
- daily = [s for s in splitByKeys(seq, regex, transformer, keyed)] # url groups keyed by DOY first
- for chunk in daily:
- keys, chunk = chunk
- try:
- key = int(keys[0])
- except:
- key = int(keys)
- i = (int((key-1)/n)) * n + 1
- yield (i, chunk)
-
-def groupByKeys(seq):
- '''Merge multiple keys into a single key by appending lists.'''
- seq = [s for s in seq]
- merge = {}
- for s in seq:
- key, chunk = s
- if key not in merge:
- merge[key] = chunk
- else:
- merge[key].extend(chunk) # extend returns None, that blows
- result = []
- for k in sorted(merge.keys()):
- result.append((k, merge[k]))
- return result
-
-
-def windowSplit(seq, nEpochs, nWindow):
- '''Split a sequence (e.g. of daily files/urls) into nWindow-long chunks for climatology averaging.
-The length of the window will usually be longer than the nEpochs the average is good for.
- '''
- pass
-
-
-# Tests follow.
-
-def test1(args):
- n = int(args[0])
- fn = args[1]
- with open(fn, 'r') as f:
- for chunk in fixedSplit(f, n):
- print(' '.join(chunk))
-
-def test2(args):
- regex = args[0]
- regex = re.compile(regex)
- fn = args[1]
- with open(fn, 'r') as f:
- for chunk in splitByKey(f, regex):
- print(' '.join(chunk))
-
-def test3(args):
- '''Broken!'''
- nDays = int(args[0])
- regex = args[1]
- regex = re.compile(regex)
- fn = args[2]
- with open(fn, 'r') as f:
- for chunk in splitByNDays(f, nDays, regex):
- print(chunk)
-
-def test4(args):
- '''Correct!'''
- nDays = int(args[0])
- regex = args[1]
- regex = re.compile(regex)
- fn = args[2]
- with open(fn, 'r') as f:
- for chunk in splitByNDays(f, nDays, regex):
- print()
- print('\n'.join(chunk))
- print(len(chunk))
-
-def test5(args):
- '''Generate multi-line JSON for pyspark.'''
- nDays = int(args[0])
- regex = args[1]
- fn = args[2]
- with open(fn, 'r') as f:
- for chunk in splitByNDays(f, nDays, regex):
- print(json.dumps(chunk))
-
-def test6(args):
- '''Generate keyed split by month for spark.'''
- regex = args[0]
- fn = args[1]
- with open(fn, 'r') as f:
- for chunk in splitByMonth(f, {'get': 'doy', 'regex': re.compile(regex)}):
- print(chunk)
-
-
-def main(args):
-# test1(args)
-# test2(args)
-# test3(args)
-# test4(args)
-# test5(args)
- test6(args)
-
-if __name__ == '__main__':
- import sys
- main(sys.argv[1:])
-
-
-# python split.py 5 '(...).L3m' urls_sst_daynight_2003_2015_sorted.txt
-
-# python split.py '\/A(....)(...)' urls_sst_daynight_2003_4months.txt
-# python split.py '\/A(....)(...)' urls_sst_daynight_2003_2015.txt
diff --git a/climatology/clim/test/__init__.py b/climatology/clim/test/__init__.py
deleted file mode 100644
index 6acb5d1..0000000
--- a/climatology/clim/test/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/climatology/clim/test/ccmpTest.py b/climatology/clim/test/ccmpTest.py
deleted file mode 100644
index c5159e0..0000000
--- a/climatology/clim/test/ccmpTest.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import unittest
-import ClimatologySpark2
-
-
-class CCMPTest(unittest.TestCase):
- def cmmp_test(self):
- dsName = 'CCMPWind'
- nEpochs = '1'
- nWindow = '1'
- averager = 'pixelMean'
- sparkConfig = 'multicore,4,4'
- outHdfsPath = 'cache/clim'
-
- ClimatologySpark2.main([dsName, nEpochs, nWindow, averager, sparkConfig, outHdfsPath])
diff --git a/climatology/clim/timePartitions.py b/climatology/clim/timePartitions.py
deleted file mode 100755
index 46ced1c..0000000
--- a/climatology/clim/timePartitions.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
- timePartitions.py
-
- Routines to partition time ranges into segments, and time-ordered
- file URL's into time segments.
-
-"""
-
-import os, sys
-
-
-def partitionFilesByKey(paths, path2keyFn):
- """Partition a list of files (paths) into groups by a key.
-The key is computed from the file path by the passed-in 'path2key' function.
-
-For example, to group files by day or month, the key function could return a string
-date like 'YYYY/MM/DD' or a month as 'YYYY/MM'.
- """
- key = path2keyFn(paths[0])
- groupedPaths = []
- for path in paths:
- if path.strip() == '': continue
- nextKey = path2keyFn(path)
- if nextKey != key:
- yield (key, groupedPaths)
- key = nextKey
- groupedPaths = [path]
- else:
- groupedPaths.append(path)
- if len(groupedPaths) > 0: yield (key, groupedPaths)
-
-
diff --git a/climatology/clim/util/__init__.py b/climatology/clim/util/__init__.py
deleted file mode 100644
index 6acb5d1..0000000
--- a/climatology/clim/util/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/climatology/clim/util/array.py b/climatology/clim/util/array.py
deleted file mode 100755
index 1c68a20..0000000
--- a/climatology/clim/util/array.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#!/bin/env python
-
-"""
-array.py -- Simple utilities for arrays: slice, compress by mask, bundle up, etc.
-"""
-
-import sys, os
-import numpy as N
-#import pynio as NC
-
-
-def sliceArrays(i, j, *arrays):
- """Slice all input arrays using the supplied indices, a[i:j], and return them in the
-same order as input.
- """
- return [a[i:j] for a in arrays]
-
-def compressArrays(mask, *arrays):
- """Compress all input arrays using the supplied mask, a.compress(mask), and return them
-in the same order as input.
- """
- return [a.compress(mask) for a in arrays]
-
-
-class BadNameError(RuntimeError): pass
-class DimensionError(RuntimeError): pass
-class VariableError(RuntimeError): pass
-
-ReservedKeywords = ['dimensions', 'variables', 'attributes']
-
-
-class BundleOfArrays(object):
- """Simple holder class for a bundle of arrays (variables). Each variable is a vector
-or array over the supplied dimensions, or a scalar value. Its purpose is to synchronize
-modification of many arrays, such as by slice or compress, and to provide persistence for
-a bundle of variables to a file (e.g. netCDF).
- """
- def __init__(self, dimensions=None, **kwargs):
- """Init object with array dimensions and scalar values."""
- self.dimensions = {}; self.variables = {}; self.attributes = {}
- self.createAttributes(**kwargs)
- if dimensions is None: dimensions = ('Record', -1) # default dim name for vectors
- self.createDims(dimensions) # of unlimited dimension (-1)
-
- def createAttribute(self, name, val):
- self.checkForBadName(name)
- self.attributes[name] = val
- setattr(self, name, val)
- return self
-
- def createAttributes(self, **kwargs):
- for key, val in kwargs.items():
- self.createAttribute(key, val)
-
- def createDim(self, name, size):
- """Create a dimension to be used in the variable arrays."""
- self.checkForBadName(name)
- if type(size) != int: raise DimensionError('Size of dimension must be an integer')
- self.dimensions[name] = size
-
- def createDims(self, *dims):
- """Create multiple dimensions from list of (name, size) tuples."""
- for dim in dims:
- if type(dim) == tuple:
- name, val = dim
- else:
- name = dim.name; val = dim
- self.createDim(name, val)
- return self
-
- def createVar(self, name, val, copy=False):
- """Create a variable array. If the value is None, the variable is not created.
-By default, the array is NOT copied since a copy may have just been made by slicing, etc.
-If you need to make a copy of the incoming val array, use copy=True.
- """
- self.checkForBadName(name)
- if val is not None:
- try:
- n = len(val)
- if copy:
- try:
- val = val.copy() # use copy method if it exists, e.g. numpy array
- except:
- val = val[:] # copy array by slicing
- except:
- raise VariableError('Variable must be a list, vector, array, or None.')
- self.variables[name] = val
- setattr(self, name, val)
-
- def createVars(self, *vars):
- """Create multiple variables from list of (name, value) tuples."""
- for var in vars:
- if type(var) == list or type(var) == tuple:
- name, val = var
- else:
- name = var.name; val = var
- self.createVar(name, val)
- return self
-
- def slice(self, i, j):
- """Slice all of the variable arrays as a[i:j], and return new array bundle."""
- out = self.shallowCopy()
- for key in self.variables:
- val = self.variables[key][i:j]
- out.createVar(key, val, copy=False)
- return out
-
- def compress(self, mask):
- """Compress all of the variable arrays using a mask, a.compress(mask)[i:j], and return
-a new array bundle.
- """
- out = self.shallowCopy()
- for key in self.variables:
- s = self.variables[key].shape
- a = self.variables[key]
- if len(s) == 1:
- val = a.compress(mask)
- else:
- # Can't use N.compress() from old numpy version because
- # it flattens the additional dimensions (bug).
- # Temporarily, do it by lists in python (slower)
- val = N.array([a[i] for i, b in enumerate(mask) if b])
- out.createVar(key, val, copy=True)
- return out
-
- def shallowCopy(self):
- """Shallow copy of the object, retaining all attributes, dimensions, and variables.
-*** Note: This routine does NOT copy the arrays themselves, but slice & compress do. ***
- """
- out = BundleOfArrays()
- out.attributes = self.attributes.copy()
- # Must use copy() here or both bundles will point to same attr/dim/var dictionaries.
- # It is a shallow copy, but this is OK since attr/dim values are immutable.
- for key, val in out.attributes.items(): setattr(out, key, val)
- out.dimensions = self.dimensions.copy()
- out.variables = self.variables.copy()
- # Again, shallow copy is OK, referred-to arrays are copied when slice/compress called
- for key, val in out.variables.items(): setattr(out, key, val)
- return out
-
- def checkForBadName(self, name):
- """Ensure that name is not in reserved attribute list.
-Raises exception BadNameError.
- """
- if name in ReservedKeywords:
- raise BadNameError('Attribute name, "%s", is in reserved list %s' % (name, \
- str(ReservedKeywords)))
-# try:
-# k = getattr(self, name)
-# raise BadNameError('Attribute %s already exists.' % name)
-# except:
-# pass
-
- def __repr__(self):
- return 'Attributes: %s\nDimensions: %s\nVariables: %s' % tuple(
- map(str, (self.attributes, self.dimensions, list(self.variables.keys()))))
-
-
-def test(args):
- n = 300
- vars = ['obsL', 'errL', 'obsP', 'errP']
- obsL = N.array(list(range(n)))+1.; errL = N.ones(n)
- obsP = N.array(list(range(n)))+1.; errP = N.ones(n)
- obs = BundleOfArrays(name='test', nRecords=n).createVars(*list(zip(vars, (obsL, errL, obsP, errP))))
- print(obs)
- print(len(obs.obsL))
- a = obs.shallowCopy()
- print(a)
- print(len(a.obsL))
- b = obs.slice(100, 200)
- print(b)
- print(len(b.obsL))
- print(len(a.obsL))
- print(len(obs.obsL))
-
-def main(args):
- test(args)
-
-if __name__ == '__main__':
- main(sys.argv[1:])
diff --git a/climatology/clim/util/introspect.py b/climatology/clim/util/introspect.py
deleted file mode 100755
index 481d856..0000000
--- a/climatology/clim/util/introspect.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#!/bin/env python
-
-"""
-introspect.py -- A few introspection utilities, most importantly myArgs().
-"""
-
-import sys, os
-from inspect import currentframe, getargvalues, getmodule
-
-
-def callingFrame():
- """Return calling frame info."""
- return currentframe().f_back
-
-def myArgs():
- """Return the arguments of the executing function (the caller of myArgs)."""
- return getargvalues(currentframe().f_back)[3]
-
-
-def test():
- def func1(a, b, c):
- print(currentframe())
- print(getmodule(callingFrame())) # getmodule doesn't work on frame objects
- args = myArgs()
- print(args)
- return a + b + c
-
- return func1(1, 2, 3)
-
-
-def main(args):
- print(test())
-
-if __name__ == '__main__':
- main(sys.argv[1:])
diff --git a/climatology/clim/util/plot.py b/climatology/clim/util/plot.py
deleted file mode 100644
index 9aa97ee..0000000
--- a/climatology/clim/util/plot.py
+++ /dev/null
@@ -1,147 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-plot.py -- Simple plotting utilitites
-"""
-
-import sys, os
-from matplotlib.figure import Figure
-from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
-
-def echo2(*s): sys.stderr.write(' '.join(map(str, s)) + '\n')
-def echo2n(*s): sys.stderr.write('\n'.join(map(str, s)) + '\n')
-def warn(*s): echo2('plot:', *s)
-def die(*s): warn('Error,', *s); sys.exit()
-
-
-def makeMovie(plotFiles, outFile, deletePlots=True):
- """Make MPG movie from a series of still images."""
- if type(plotFiles) == dict:
- plotFiles = [plotFiles[k] for k in sorted(plotFiles.keys())]
- cmd = 'convert ' + ' '.join(plotFiles) + ' ' + outFile
- warn(cmd)
- os.system(cmd)
- warn('Wrote movie ' + outFile)
- if deletePlots:
- for f in plotFiles: os.unlink(f)
- return outFile
-
-def createPlot():
- fig = Figure()
- canvas = FigureCanvas(fig)
- ax = fig.add_subplot(1,1,1)
- return (fig, canvas, ax)
-
-def labelPlot(fix, canvas, ax, title,
- xlabel=None,
- ylabel=None,
- xlim=None,
- ylim=None):
-# ax.set_title(title)
- if xlabel is not None: ax.set_xlabel(xlabel)
- if ylabel is not None: ax.set_ylabel(ylabel)
- if xlim is not None: ax.set_xlim(xlim)
- if ylim is not None: ax.set_ylim(ylim)
-
-
-def createTwoPlots(figure=None, canvas=None, vertical=True):
- if figure is None:
- fig = Figure()
- else:
- fig = figure
- if canvas is None: canvas = FigureCanvas(fig)
- if vertical:
- ax1 = fig.add_subplot(2,1,1)
- ax2 = fig.add_subplot(2,1,2)
- else:
- ax1 = fig.add_subplot(1,2,1)
- ax2 = fig.add_subplot(1,2,2)
- return (fig, canvas, ax1, ax2)
-
-def labelTwoPlots(fig, canvas, ax1, ax2,
- title = None,
- title1 = None,
- xlabel1 = None, ylabel1 = None,
- xlim1 = None, ylim1 = None,
- title2 = None,
- xlabel2 = None, ylabel2 = None,
- xlim2 = None, ylim2 = None,
- ):
- if title1 is None: title1 = title
- if title1 is not None: ax1.set_title(title1)
- if xlabel1 is not None: ax1.set_xlabel(xlabel1)
- if ylabel1 is not None: ax1.set_ylabel(ylabel1)
- if xlim1 is not None: ax1.set_xlim(xlim1)
- if ylim1 is not None: ax1.set_ylim(ylim1)
- if title2 is not None: ax2.set_title(title2)
- if xlabel2 is not None: ax2.set_xlabel(xlabel2)
- if ylabel2 is not None: ax2.set_ylabel(ylabel2)
- if xlim2 is not None: ax2.set_xlim(xlim2)
- if ylim2 is not None: ax2.set_ylim(ylim2)
-
-
-def createFourPlots(figure=None, canvas=None):
- if figure is None:
- fig = Figure()
- else:
- fig = figure
- if canvas is None: canvas = FigureCanvas(fig)
- ax1 = fig.add_subplot(2,2,1)
- ax2 = fig.add_subplot(2,2,2)
- ax3 = fig.add_subplot(2,2,3)
- ax4 = fig.add_subplot(2,2,4)
- return (fig, canvas, ax1, ax2, ax3, ax4)
-
-def labelFourPlots(fig, canvas, ax1, ax2, ax3, ax4,
- file, sat,
- title = None,
- title1 = None,
- xlabel1 = None, ylabel1 = None,
- xlim1 = None, ylim1 = None,
- title2 = None,
- xlabel2 = None, ylabel2 = None,
- xlim2 = None, ylim2 = None,
- title3 = None,
- xlabel3 = None, ylabel3 = None,
- xlim3 = None, ylim3 = None,
- title4 = None,
- xlabel4 = None, ylabel4 = None,
- xlim4 = None, ylim4 = None,
- ):
- if title is None: title = file + ': ' + sat
-# fig.set_title(title)
- if title1 is None: title1 = title
- if title1 is not None: ax1.set_title(title1)
- if xlabel1 is not None: ax1.set_xlabel(xlabel1)
- if ylabel1 is not None: ax1.set_ylabel(ylabel1)
- if xlim1 is not None: ax1.set_xlim(xlim1)
- if ylim1 is not None: ax1.set_ylim(ylim1)
- if title2 is not None: ax2.set_title(title2)
- if xlabel2 is not None: ax2.set_xlabel(xlabel2)
- if ylabel2 is not None: ax2.set_ylabel(ylabel2)
- if xlim2 is not None: ax2.set_xlim(xlim2)
- if ylim2 is not None: ax2.set_ylim(ylim2)
- if title3 is not None: ax3.set_title(title3)
- if xlabel3 is not None: ax3.set_xlabel(xlabel3)
- if ylabel3 is not None: ax3.set_ylabel(ylabel3)
- if xlim3 is not None: ax3.set_xlim(xlim3)
- if ylim3 is not None: ax3.set_ylim(ylim3)
- if title4 is not None: ax4.set_title(title4)
- if xlabel4 is not None: ax4.set_xlabel(xlabel4)
- if ylabel4 is not None: ax4.set_ylabel(ylabel4)
- if xlim4 is not None: ax4.set_xlim(xlim4)
- if ylim4 is not None: ax4.set_ylim(ylim4)
-
diff --git a/climatology/clim/util/stats.py b/climatology/clim/util/stats.py
deleted file mode 100755
index c17f7c8..0000000
--- a/climatology/clim/util/stats.py
+++ /dev/null
@@ -1,232 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import sys
-from math import sqrt
-from collections import namedtuple
-
-import numpy as NP
-import scipy.stats
-
-
-def mad( l ):
- """Compute the median absolute deviation (a robust measure of spread) of the list of
- values *l*."""
- median = NP.median(l)
- return NP.median([abs(x - median) for x in l])
-
-
-def robust_std(l, alpha=1/scipy.stats.norm.ppf(0.75)):
- """Compute a robust estimate of the standard deviation (by default, for normally
- distrbuted samples)."""
- return alpha * mad(l)
-
-
-def filter_outliers( time_series, n_std=6, indices=False ):
- """Filter outliers (those samples a distance of *n_std* robust standard deviations
- from the median) and return."""
- med = NP.median( time_series )
- std = robust_std( time_series )
- lhs = list(zip( *[ (i, x) for i, x in enumerate( time_series ) if abs(x - med) < ( std * n_std ) ] ))
- if len( lhs ) == 0:
- # No outliers detected, return the full time series
- return time_series
- I, out = lhs
- if isinstance( time_series, NP.ndarray ):
- out = NP.array( out )
- if indices:
- return out, I
- else:
- return out
-
-
-####################################################################################################
-
-#!/usr/bin/env python
-#
-# Stats.py -- Simple statistics class: computes mean, stddev, min, max, rms.
-#
-# Author: Brian Wilson
-# @(#) Stats.py 1.0 2003/11/24
-# 1.1 2010/12/14 --- rewrote the add method so that
-# higher moments are c alculated correctly
-# (Mark D. Butala)
-#
-# Implemented by saving five accumulators:
-# no of points, mean, sum of squares of diffs from mean, min, and max.
-# Methods:
-# add -- add a data point to the accumulating stats
-# calc -- compute the five statistics: n, mean, std dev, min, max, rms
-# label -- set the label for printing
-# format -- set the float format for printing
-# __repr__ -- generates one-line string version of statistics for easy printing
-# reset -- zero the accumulators
-# addm -- add an array of data points to the accumulators (add multiple)
-#
-# See tests at end of file for example usage.
-#
-
-StatsCalc = namedtuple('StatsCalc', 'n mean stddev min max rms skewness kurtosis')
-
-
-class Stats(object):
- """Simple statistics class that computes mean, std dev, min, max, and rms."""
- __slots__ = ('count', 'mean', 'stddev', 'min', 'max', 'rms', 'skewness', 'kurtosis',
- 'rms2', 'M2', 'M3', 'M4', 'labelStr', 'formatStr', 'missingValue')
-
- def __init__(self, missingValue=-9999., label=None, format=None):
- """Create Stats object, optionally set print label and float format string."""
- self.reset(missingValue)
- self.missingValue = missingValue
- self.labelStr = label
- self.formatStr = format
-
- def add(self, val):
- """Add one data point to the accumulators."""
- self.count += 1
- n = self.count
- if n == 1:
- self.mean = 0.
- self.M2 = 0.
- self.rms2 = 0.
- self.M3 = 0.
- self.M4 = 0.
- self.min = val
- self.max = val
- else:
- self.min = min(self.min, val)
- self.max = max(self.max, val)
-
- delta = val - self.mean # use devation from mean to prevent roundoff/overflow problems
- delta_n = delta / float(n)
- delta_n2 = delta_n * delta_n
- self.mean += delta_n
- self.rms2 += (val**2 - self.rms2) / float(n)
- term = delta * delta_n * (n-1)
- self.M4 += term * delta_n2 * (n*n - 3*n + 3) + 6 * delta_n2 * self.M2 - 4 * delta_n * self.M3
- self.M3 += term * delta_n * (n - 2) - 3 * delta_n * self.M2
- self.M2 += term
- return self
-
- def calc(self):
- """Calculate the statistics for the data added so far.
- Returns tuple of six values: n, mean, stddev, min, max, rms.
- """
- n = self.count
- if (n >= 2):
- M2 = self.M2
- stddev = sqrt(M2 / float(n - 1))
- rms = sqrt(self.rms2)
- self.stddev = stddev
- self.rms = rms
- self.skewness = sqrt(n) * self.M3 / (M2 * sqrt(M2))
- self.kurtosis = (n * self.M4) / (M2 * M2) - 3
- return StatsCalc(self.count, self.mean, self.stddev, self.min, self.max, self.rms, self.skewness, self.kurtosis)
-
- def label(self, str):
- """Label the statistics for printing."""
- self.labelStr = str
- return self
-
- def format(self, str):
- """Set the float format to be used in printing stats."""
- self.formatStr = str
- return self
-
- def __repr__(self):
- """One-line stats representation for simple printing."""
- if (self.labelStr == None or self.labelStr == ""): self.labelStr = "Stats"
- line = self.labelStr + ": "
- if self.formatStr:
- a = [self.formatStr for i in range(7)]
- a.insert(0, '%d')
- format = ' '.join(a)
- line += format % self.calc()
- else:
- line += "N=%d mean=%f stddev=%f min=%f max=%f rms=%f skewness=%f kurtosis=%f" % self.calc()
- return line
-
- def reset(self, missingValue):
- """Reset the accumulators to start over."""
- self.count = 0
- self.mean = missingValue
- self.stddev = missingValue
- self.min = missingValue
- self.max = missingValue
- self.rms = missingValue
- self.skewness = missingValue
- self.kurtosis = missingValue
- self.M2 = 0.
- self.rms2 = 0.
- self.M3 = 0.
- self.M4 = 0.
- self.labelStr = None
- self.formatStr = None
- return self
-
- def addm(self, seq):
- """Add multiple - add a sequence of data points all at once."""
- for val in seq:
- self.add(val)
- return self
-
-
-####################################################################################################
-
-
-def main(args):
- fn = args[0]
- try:
- if fn == '-':
- fid = sys.stdin
- else:
- fid = open(fn, 'r')
- stats = Stats()
- stats.addm( (float(x) for x in fid) )
- print(stats)
- finally:
- if fid is not sys.stdin:
- fid.close()
-
-
-if __name__ == '__main__':
- main(sys.argv[1:])
-
-
-'''
- def test():
- """
->>> print Stats()
-Stats: 0 0.000000 0.000000 0.000000 0.000000 0.000000
-
->>> def f(s):
-... for v in [2.3, 4.5, 1.8, 6.2, 3.5]: s.add(v)
-... s.label('test2')
-... return s
->>> print f( Stats() )
-test2: 5 3.660000 1.468279 1.800000 6.200000 3.888480
-
->>> print Stats().label('test3').addm([2.3, 4.5, 1.8, 6.2, 3.5])
-test3: 5 3.660000 1.468279 1.800000 6.200000 3.888480
-
->>> print Stats('test4').format('%5.2f').addm([2.3, 4.5, 1.8, 6.2, 3.5])
-test4: 5 3.66 1.47 1.80 6.20 3.89
-
->>> print Stats('test5', '%4.1f').addm([2.3, 4.5, 1.8, 6.2, 3.5])
-test5: 5 3.7 1.5 1.8 6.2 3.9
- """
-
- import doctest
- doctest.testmod()
-'''
diff --git a/climatology/clim/util/timeJ2000.py b/climatology/clim/util/timeJ2000.py
deleted file mode 100755
index 5e20576..0000000
--- a/climatology/clim/util/timeJ2000.py
+++ /dev/null
@@ -1,383 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#!/bin/env python
-
-"""
-timeJ2000.py -- Date & Time class based on native Python datetime, time, and calendar
- libraries. Represents a Date/Time as seconds past J2000 epoch
- and provides various format conversions and date delta arithmetic.
- Also includes some new smart functions that perform desired
- transformations on a Do The Right Thing basis.
-"""
-
-import sys, datetime, calendar, time, types
-
-##CONSTANTS
-J2000_1970_EPOCH = 946684800 + 12*60*60 #2000/01/01,12:00:00 in seconds past 1970
-LATEST_TIME = 9999999999 #Highest (latest) time in J2000 to care about... useful for initializations
-EARLIEST_TIME = -9999999999 #Lowest (earlist) time in J2000 to care about... useful for initializations
-
-def echo (str ): sys.stdout.write(str + "\n")
-def err (str ): sys.stderr.write(str + "\n")
-def warn (str ): err("---WARNING, IONOTIME: "+str)
-def die (str, status=1): err("***ERROR: "+str); sys.exit(status)
-
-##BASE TRANSFORMATIONS
-def ensureYYYY(y):
- if y>99: return y
- if y>50: return 1900+y
- return 2000+y
-
-def ensureYY(y):
- return y%100
-
-#transforms an hms string to a float hours
-def hms_to_hours(str):
- return float(str[0:2])+float(str[2:4])/60.0+float(str[4:6])/360.0
-
-def J2000_to_list(sec=0.0):
- #check for fractional seconds
- frac=0.0
- if sec > int(sec):
- frac=sec-int(sec)
- sec =int(sec)
- callist=list(time.gmtime(sec+J2000_1970_EPOCH))
- #add back in fractional seconds if present
- if frac > 0.0:
- callist[5]=callist[5]+frac
- return callist[0:6]
-def list_to_J2000(inlist):
- #check for fractional seconds and remove
- clist=[0,0,0,0,0,0.0] #default to zeros everywhere
- clist[:len(inlist)]=inlist
- ss=clist[5]
- frac=0.0
- if ss > int(ss):
- frac=ss-int(ss)
- clist[5]=int(ss)
- #transform, adding fractional seconds afterwards
- return calendar.timegm(clist)-J2000_1970_EPOCH+frac
-
-##INTELLIGENT FUNCTIONS
-def valid_formats():
- return ('J2000', #int or float bare number
- 'HHMMSS', #string
- 'YYMMDD', #string
- 'YYYYMMDD', #string
- 'YYMMDDHHMMSS', #string .
- 'YYYYMMDDHHMMSS', #string .
- 'YYYYMMDD_HHMMSS', #string .
- 'YYMMDD_HHMMSS', #string .
- 'DOY', #string
- 'HOD',"HOURSINDAY", #string hours of day
- 'MOD',"MINUTESINDAY", #string minutes of day
- 'SOD',"SECONDSINDAY", #string seconds of day
- 'YYDOY', #string
- 'LIST', #list(y,m,d,h,m,s)
- 'HMS', #string
- 'YMD', #string
- 'YMDHMS', #string
- 'GAIMSTRING', #string yyyy/mm/dd,hh:mm:ss.frac
- 'TENETHOURLY', #string siteDOYlmm.yy.tenet
- 'LOCALHMS', #string HHMMSS.F adjusted for local time (requires longitude in deg)
- 'HOURLETTER'#, #string a where a(a,x) for each hour of day
-# 'RINEX' #string
- )
-
-def to_J2000(input,format=None):
- sec=0 #internal representation
- if format: format=format.upper()
-
- #assume J2000 seconds for any bare number
- if isinstance(input,int) or isinstance(input,float) or isinstance(input,int) or format=='J2000': return float(input)
- #if it's a list, simple... will be interpretted as y,m,d,hh,mm,ss with 0's in any unspecified slot
- elif isinstance(input,list) or isinstance(input,tuple): return list_to_J2000(input)
- #if it's a string, could be many things
- elif isinstance(input,bytes):
- #strip off any fractional second information first
- p=input.find('.')
- frac=0.0
- if p>=0:
- if input.find('tenet') < 0:
- frac=float(input[p:])
- input =input[:p]
- #Autoguess format based on length or user-specified request
- if len(input)==len('siteDOYlmm.yy.tenet') and format=="TENETHOURLY":
- (doy,hl,mm,y)=(int(input[4:7]),input[7:8],int(input[8:10]),int(input[11:13]))
- (yyyy,m,d)=J2000_to_list(list_to_J2000((ensureYYYY(int(y)),1,doy)))[0:3]
- return list_to_J2000((yyyy,m,d,ord(hl)-ord('a'),mm,0))
-
- if format=="DOY":
- return list_to_J2000((2000,1,int(input)))
-
- if format in ("HOD","HOURSINDAY"):
- return list_to_J2000((2000,1,1,int(input),0,0))
-
- if format in ("MOD","MINUTESINDAY"):
- return list_to_J2000((2000,1,1,0,int(input),0))
-
- if format in ("SOD","SECONDSINDAY"):
- return list_to_J2000((2000,1,1,0,0,int(input)))
-
- if format=="YYDOY":
- return list_to_J2000((ensureYYYY(int(input[0:2])),1,int(input[2:])))
-
- if len(input)==len('a') or format=='HOURLETTER':
- return list_to_J2000((2000,1,1,ord(input)-ord('a'),0,0))
- if len(input)==len('YYYY/MM/DD,HH:MM:SS') or format=='GAIMSTRING' or format=='ISO':
- return list_to_J2000((int(input[0:4]),
- int(input[5:7]),
- int(input[8:10]),
- int(input[11:13]),
- int(input[14:16]),
- int(input[17:19])+frac))
- if len(input)==len('YYYYMMDD_HHMMSS') or format=='YYYYMMDD_HHMMSS':
- return list_to_J2000((int(input[0:4]),
- int(input[4:6]),
- int(input[6:8]),
- int(input[9:11]),
- int(input[11:13]),
- int(input[13:15])+frac))
-
- if len(input)==len('YYMMDD_HHMMSS') or format=='YYMMDD_HHMMSS':
- return list_to_J2000((ensureYYYY(int(input[0:2])),
- int(input[2:4]),
- int(input[4:6]),
- int(input[7:9]),
- int(input[9:11]),
- int(input[11:13])+frac))
-
- if len(input)==len('YYYYMMDDHHMMSS') or format=='YYYYMMDDHHMMSS':
- return list_to_J2000((int(input[0:4]),
- int(input[4:6]),
- int(input[6:8]),
- int(input[8:10]),
- int(input[10:12]),
- int(input[12:14])+frac))
-
- if len(input)==len('YYMMDDHHMMSS') or format=='YYMMDDHHMMSS' or format=="YMDHMS":
- return list_to_J2000((ensureYYYY(int(input[0:2])),
- int(input[2:4]),
- int(input[4:6]),
- int(input[6:8]),
- int(input[8:10]),
- int(input[10:12])+frac))
-
- if len(input)==len('YYYYMMDD') or format=='YYYYMMDD':
- return list_to_J2000((int(input[0:4]),
- int(input[4:6]),
- int(input[6:8])))
-
- if len(input)==len('HHMMSS') and format in ('HHMMSS','HMS'):
- return list_to_J2000((2000,1,1,
- int(input[0:2]),
- int(input[2:4]),
- int(input[4:6])+frac))
-
- if len(input)==len('YYMMDD') or format in ('YYMMDD','YMD'):
- return list_to_J2000((ensureYYYY(int(input[0:2])),
- int(input[2:4]),
- int(input[4:6])))
-
- die("Unknown string format",input)
- die("Unknown input type to to_J2000:",input)
-
-def from_J2000(sec=0,format="YYYYMMDD_HHMMSS",aux=None):
- #aux contains spare information, thusfar only used for site id's for filenames or longitude for localtime
- format=format.upper()
- if format == "J2000" : return sec
- (y,m,d,hh,mm,ss)=J2000_to_list(sec)
- f=""
- if ss > int(ss): f=("%f"%(ss-int(ss))).strip('0') #remove leading and trailing 0
- if format == "LIST" : return [y,m,d,hh,mm,ss]
- if format == "HOURLETTER" : return chr(hh+ord('a'))
- if format in("HOURSINDAY","HOD") : return hh+mm/60.0+ss/60.0/60.0
- if format in("MINUTESINDAY","MOD") : return hh*60+mm+ss/60.0
- if format in("SECONDSINDAY","SOD") : return (hh*60+mm)*60+ss
- if format in("HHMMSS","HMS") : return "%02d%02d%02d"%(hh,mm,ss)+f
- if format in("YYMMDD","YMD") : return "%02d%02d%02d"%(ensureYY(y),m,d)
- if format == "YYYYMMDD" : return "%04d%02d%02d"%(y,m,d)
- if format in("YYMMDDHHMMSS","YMDHMS"): return "%02d%02d%02d%02d%02d%02d"%(ensureYY(y),m,d,hh,mm,ss)+f
- if format == "YYYYMMDDHHMMSS" : return "%04d%02d%02d%02d%02d%02d"%(y,m,d,hh,mm,ss)+f
- if format == "YYMMDD_HHMMSS" : return "%02d%02d%02d_%02d%02d%02d"%(ensureYY(y),m,d,hh,mm,ss)+f
- if format == "YYYYMMDD_HHMMSS" : return "%04d%02d%02d_%02d%02d%02d"%(y,m,d,hh,mm,ss)+f
- if format == "GAIMSTRING" : return "%04d/%02d/%02d,%02d:%02d:%02d"%(y,m,d,hh,mm,ss)+f
- if format == "ISO" : return "%04d-%02d-%02dT%02d:%02d:%02dZ"%(y,m,d,hh,mm,ss)+f
- doy = time.gmtime(sec+J2000_1970_EPOCH)[7] #fetch doy
- if format == "DOY" : return "%03d"%(doy)
- if format == "YYDOY" : return "%02d%03d"%(ensureYY(y),doy)
- if format == "TENETHOURLY" :
- if not aux: aux="site"
- return "%4s%03d%1s%02d.%02d.tenet"%(aux,doy,chr(ord('a')+hh),mm,ensureYY(y))
- if format == "LOCALHMS" :
- if not aux: aux=0
- localtime = hh + aux/360.0*24.0 #in this case, aux is longitude in deg
- while (localtime < 0): localtime+=+24
- while (localtime >= 24): localtime-= 24
- return "%02d%02d%02d"%(localtime,mm,ss)+f
- die("Unrecognized format string in from_J2000 "+format)
-
-class IonoTime:
- "Handles conversions between times and dates for all variety of ionospheric time interests"
- #internal representation is seconds past J2000
- def __init__(self,input=None):
- self.sec = 0
- self.set(input)
- def set(self,input=None,format=None):
- if not input: return self
- if isinstance(input,IonoTime):
- self.sec=input.sec
- else:
- self.sec = to_J2000(input,format)
- return self
- def to(self,format=None,aux=None):
- if not format: return self.sec
- return from_J2000(self.sec,format,aux)
- def now(self):
- self.sec = to_J2000(time.localtime()[0:6])
- return self
- def nowUTC(self):
- self.sec = to_J2000(time.gmtime()[0:6])
- return self
- def addSeconds(self,s):
- self.sec+=s
- return self
- def addMinutes(self,m):
- self.sec+=m*60.0
- return self
- def addHours (self,h):
- self.sec+=h*60.0*60.0
- return self
- def addDays (self,d):
- self.sec+=d*60.0*60.0*24.0
- return self
- def addMonths (self,mi):
- (y,m,d,hh,mm,ss)=from_J2000(self.sec,"LIST")
- m+=mi
- while m > 12:
- y=y+1
- m-=12
- while m < 1:
- y=y-1
- m+=12
- self.sec=to_J2000((y,m,d,hh,mm,ss))
- return self
- def addYears (self,yi):
- (y,m,d,hh,mm,ss)=from_J2000(self.sec,"LIST")
- self.sec=to_J2000((y+yi,m,d,hh,mm,ss))
- return self
- def copy (self):
- n=IonoTime(self.sec)
- return n
- def makemidnight(self):
- (y,m,d,hh,mm,ss)=from_J2000(self.sec,"LIST")
- self.sec=to_J2000((y,m,d))
- return self
- def floor(self,interval): #round current object to a specified accuracy
- (y,m,d,hh,mm,ss)=from_J2000(self.sec,"LIST")
- interval=interval.lower()
- if interval.find('year' )>=0: self.sec=to_J2000((y, 1, 0, 0, 0, 0))
- elif interval.find('month' )>=0: self.sec=to_J2000((y, m, 1, 0, 0, 0))
- elif interval.find('day' )>=0: self.sec=to_J2000((y, m, d, 0, 0, 0))
- elif interval.find('hour' )>=0: self.sec=to_J2000((y, m, d, hh, 0, 0))
- elif interval.find('minute')>=0: self.sec=to_J2000((y, m, d, hh, mm, 0))
- elif interval.find('second')>=0: self.sec=to_J2000((y, m, d, hh, mm,int(ss)))
- else : die("IonoTime: Floor: Malformed interval: "+interval)
- return self
- def __sub__(self,other):
- return IonoTime(self.sec-other)
- def __add__(self,other):
- return IonoTime(self.sec+other)
-# def __iadd__(self,other):
-# return IonoTime(self.sec+other)
-# def __isub__(self,other):
-# return IonoTime(self.sec-other)
- def __cmp__(self,other):
- return cmp(self.sec,other.sec)
- def __coerce__(self,other):
- if isinstance(other,float) or isinstance(other,int) or isinstance(other,int):
- return (self.sec,other)
- if isinstance(other,bytes):
- return (from_J2000(self.sec,"YYYYMMDD_HHMMSS"),other)
- if isinstance(other,list) or isinstance(other,tuple):
- return (from_J2000(self.sec,"LIST"),other)
- def __repr__(self):
- return from_J2000(self.sec,"YYYYMMDD_HHMMSS")
-
-def test():
- print("Testing timeJ2000 routines:")
- print("Checking to_J2000")
- if not to_J2000("20040606" )==139752000 : die("FAILED YYYYMMDD test")
- if not to_J2000("040606" )==139752000 : die("FAILED YYMMDD test")
- if not to_J2000("20040606010101" )==139755661 : die("FAILED YYYYMMDDHHMMSS test")
- if not to_J2000("c" )==-36000.0 : die("FAILED HOURLETTER test")
- if not to_J2000("20040606010101.1" )==139755661.1 : die("FAILED YYYYMMDDHHMMSS.F test")
- if not to_J2000("20040606_010101" )==139755661 : die("FAILED YYYYMMDD_HHMMSS test")
- if not to_J2000("20040606_010101.1")==139755661.1 : die("FAILED YYYYMMDD_HHMMSS.F test")
- if not to_J2000("040606_010101" )==139755661 : die("FAILED YYMMDD_HHMMSS test")
- if not to_J2000("040606_010101.1" )==139755661.1 : die("FAILED YYMMDD_HHMMSS.F test")
- if not to_J2000("040606010101" )==139755661 : die("FAILED YYMMDDHHMMSS test")
- if not to_J2000("040606010101.1" )==139755661.1 : die("FAILED YYMMDDHHMMSS.F test")
- if not to_J2000("121212.1",'HHMMSS')==732.1 : die("FAILED HHMMSS test")
- if not to_J2000(10244201.1 )==10244201.1 : die("FAILED J2000 test")
- if not to_J2000((2004,6,6,1,1,1.1) )==139755661.1 : die("FAILED list test")
- if not to_J2000("103",'DOY' )==8769600 : die("FAILED DOY test")
- if not to_J2000("00103",'YYDOY' )==8769600 : die("FAILED YYDOY test")
- if not to_J2000("2004/06/06,01:01:01.1")==139755661.1: die("FAILED GAIMSTRING test")
- if not to_J2000("help158b01.04.tenet",'TENETHOURLY')==139755660.0 : die("FAILED TENETHOURLY test")
- print("Passed to_J2000")
-
- print("Checking from_J2000")
- if not from_J2000(139752000 ,"YYYYMMDD" )=="20040606" : die("FAILED YYYYMMDD test")
- if not from_J2000(139752000.1,"YYYYMMDD" )=="20040606" : die("FAILED YYYYMMDD test")
- if not from_J2000(139752000 ,"YYMMDD" )=="040606" : die("FAILED YYMMDD test")
- if not from_J2000(139752000.1,"YYMMDD" )=="040606" : die("FAILED YYMMDD test")
- if not from_J2000(139755661 ,"HOURLETTER" )=="b" : die("FAILED HOURLETTER test")
- if not from_J2000(139755661 ,"YYYYMMDDHHMMSS" )=="20040606010101" : die("FAILED YYYYMMDDHHMMSS test")
- if not from_J2000(139755661.1,"YYYYMMDDHHMMSS" )=="20040606010101.1" : die("FAILED YYYYMMDDHHMMSS.F test")
- if not from_J2000(139755661 ,"YYYYMMDD_HHMMSS")=="20040606_010101" : die("FAILED YYYYMMDD_HHMMSS test")
- if not from_J2000(139755661.1,"YYYYMMDD_HHMMSS")=="20040606_010101.1" : die("FAILED YYYYMMDD_HHMMSS.F test")
- if not from_J2000(139755661 ,"YYMMDD_HHMMSS" )=="040606_010101" : die("FAILED YYMMDD_HHMMSS test")
- if not from_J2000(139755661.1,"YYMMDD_HHMMSS" )=="040606_010101.1" : die("FAILED YYMMDD_HHMMSS.F test")
- if not from_J2000(139755661 ,"YYMMDDHHMMSS" )=="040606010101" : die("FAILED YYMMDDHHMMSS test")
- if not from_J2000(139755661.1,"YYMMDDHHMMSS" )=="040606010101.1" : die("FAILED YYMMDDHHMMSS.F test")
- if not from_J2000(732.1 ,"HHMMSS" )=="121212.1" : die("FAILED HHMMSS.F test")
- if not from_J2000(139752000.1,"J2000" )==139752000.1 : die("FAILED J2000 test")
-# (1,1.1) == (1,1.1000000001) ?!
-# if not from_J2000(139755661.1,"LIST" )==(2004,6,6,1,1,1.1) : die("FAILED LIST test")
- if not from_J2000(8769600 ,"DOY" )=="103" : die("FAILED DOY test")
- if not from_J2000(8769600 ,"YYDOY" )=="00103" : die("FAILED YYDOY test")
- if not from_J2000(139755661.1,"GAIMSTRING" )=="2004/06/06,01:01:01.1": die("FAILED GAIMSTRING test")
- if not from_J2000(139755661.1,"TENETHOURLY",'help')=="help158b01.04.tenet": die("FAILED TENETHOURLY test")
- print("Passed from_J2000")
-
- print("Testing IonoTime")
- if not IonoTime(0)+"a" =="20000101_120000a" : die("FAILED string coersion test")
- if not IonoTime(0)+1.0 ==1 : die("FAILED integer coersion test")
- if not IonoTime(0)+[1,2]==[2000,1,1,12,0,0,1,2] : die("FAILED list coersion test")
- if not IonoTime(0).addDays(2).addHours(2).addMinutes(2).addSeconds(2) == ((2*24+2)*60+2)*60+2: die("FAILED deltatime test")
- if not IonoTime(10) == IonoTime(10) : die("FAILED equivalence test")
- if not IonoTime(12) - IonoTime(10) == 2 : die("FAILED subtraction test")
- if not IonoTime(12) + IonoTime(10) == 22 : die("FAILED addition test")
- if not IonoTime(12).makemidnight().to('LOCALHMS',140) == "090000" : die("FAILED Midnight or LOCALHMS test")
- if not IonoTime(6576).floor('day').to('YYYYMMDDHHMMSS') == "20000101000000": die("FAILED floor test")
- print("Passed IonoTime")
-
-
-def main(args):
- test()
-
-if __name__ == "__main__":
- main(sys.argv[1:])
diff --git a/climatology/clim/util/warn.py b/climatology/clim/util/warn.py
deleted file mode 100644
index 40e9eb6..0000000
--- a/climatology/clim/util/warn.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# warn.py -- Utility routines to print warning & error messages like --
-# "module: error message"
-#
-try: __file__
-except: __file__ = 'warn.py' # ensure __file__ is set for warning messages
- # each module file will execute this code
-import sys, os
-from inspect import getmodule, currentframe
-
-def echo(*s):
- """Stringify & join any number of args and print resulting string to stdout"""
- sys.stdout.write(' '.join(map(str, s)) + '\n')
-
-def echon(*s):
- """Same as echo() except join with newlines."""
- sys.stdout.write('\n'.join(map(str, s)) + '\n')
-
-def echo2(*s):
- """Stringify & join any number of args and print resulting string to stderr"""
- sys.stderr.write(' '.join(map(str, s)) + '\n')
-
-def echo2n(*s):
- """Same as echo2() except join with newlines."""
- sys.stderr.write('\n'.join(map(str, s)) + '\n')
-
-def moduleName(file):
- """Extract a module name from the python source file name, with appended ':'."""
- return os.path.splitext(os.path.split(file)[1])[0] + ":"
-
-
-# Each module must define these functions so that the module name is the proper file.
-
-def warn(*s):
- """Print a warning message to stderr, identifying the module it came from."""
- echo2(moduleName(__file__)+':', *s)
-
-def die(ss, status=1):
- """Print a warning message to stderr, and die with a non-zero status value."""
- if type(ss) == str: ss = [ss]
- warn(*ss); sys.exit(status)
-
diff --git a/climatology/clim/util/wls.py b/climatology/clim/util/wls.py
deleted file mode 100755
index d28255f..0000000
--- a/climatology/clim/util/wls.py
+++ /dev/null
@@ -1,811 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#!/usr/bin/env python
-#-----------------------------------------------------------------------------
-# Name: filelist.py
-# Purpose: File listing class/functions.
-#
-# Author: Brian Wilson
-#
-# Created: Mon Apr 10 11:01:06 2006
-#-----------------------------------------------------------------------------
-#
-USAGE = """
-filelist.py [--help] [--bottomUp] [--directory] [--delete]
- [--fetchDir <outputDir>] [--fetchWitSubDirs]
- [--list] [--matchUrl] --quiet] [--regex '.*\.[cC]']
- [--size] [--topOnly] [--url]
- [--wildcard '*.txt.*'] [--xml] <topPaths ...>
-
-Recursively traverse and print (with full paths or URL's) all files
-under the topPath(s) that match ANY of one or more regular expressions
-and/or wildcard glob) strings. By default, it simply prints the matches,
-but one can also get their sizes, fetch them, or delete them.
-
-The topPaths can be a mixture of local and remote (ftp or http)
-paths, in which case a list of URL's is returned. If xml mode is
-turned on, then the output is an XML list.
-
-If no regex or wildcard patterns are specified, then ALL files
-are returned. If files are fetched, then the URL's are
-REWRITTEN to point to the local copies.
-
-"""
-# See the bottom of the file for exact switches and example of use.
-
-import sys, os, re, string, getopt, types, getpass
-import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, urllib.parse, time, shutil, socket, stat
-from fnmatch import fnmatchcase
-from ftplib import FTP
-#import dataenc
-
-def matchAnyThenConstrain(root, name, haveRegs, regs, haveWilds, wildCards,
- constraintFunction):
- """Return True if the file name matches any of the compiled regular
- expressions or any of the wildcard (glob) specs, and (if present) the
- constraintFunction returns True. The regex can be a pair of match &
- substitution patterns. The 'name' of the file might be altered by a
- regex substitution and/or the constraintFunction.
- """
- if not haveRegs and not haveWilds:
- if constraintFunction is not None:
- return constraintFunction(root, name)
- else:
- return (True, name)
- else:
- match = False
- if haveRegs:
- for reg in regs:
- pattern, subst = reg
- if pattern.search(name):
- match = True
- if subst:
- name = pattern.sub(subst, name)
- break
- if haveWilds and not match:
- for wild in wildCards:
- if fnmatchcase(name, wild):
- match = True
- break
- if match and constraintFunction is not None:
- match, name = constraintFunction(root, name)
- return (match, name)
-
-
-# Users call this function
-def filelist(urlPaths, regSpecs=[], wildCards=[], needCredentials=False, userCredentials=None,
- matchFunction=matchAnyThenConstrain, constraintFunction=None,
- matchUrl=False, walkDirectories=True,
- urlMode=True, xmlMode=True, quietMode=False, verboseMode=False, getFileInfo=False,
- fetchDir=None, fetchIfNewer=False, fetchWithSubDirs=False,
- directoryMode=False, listMode=False, deleteMode=False, topDown=True,
- stream=sys.stdout):
- """Recursively traverse and print (with full paths or URL's) all files
- under the topPath(s) that match one or more regular expressions and/or
- wildcard (glob) strings, and an optional constraint (T/F) function to
- further winnow the candidate matches. (The matchFunction can also be
- entirely replaced with custom logic.)
-
- By default, it simply generates the matches, but one can also fetch them,
- get their sizes, or delete them (if they are local files).
- Handles local directory paths and ftp/http URL's.
-
- Returns three file lists: matched, actually fetched, & destination names.
- """
- try:
- matchedFiles = [] # source files that match criteria
- fetchedFiles = [] # files that were actually fetched this run
- destinationFiles = [] # destination (local) file names (rewritten URL)
-
- topPaths = []
- for url in urlPaths:
- if url == '' or url == None: continue
- remote, protocol, netloc, path = remoteUrl(url)
- if not remote: url = os.path.abspath(url)
- if url[-1] == '/': url = url[:-1]
- topPaths.append(url)
-
- if needCredentials and userCredentials is None:
- userCredentials = promptForCredentials(topPaths)
-
- if fetchDir:
- workDir = os.path.join(fetchDir, '.tmp')
- # fetch into tmp directory & then rename so fetching is atomic
- try: os.mkdir(workDir)
- except: pass
- if not os.path.exists(workDir):
- die("filelist: Cannot write to fetch directory %s" % fetchDir)
-
- if isinstance(topPaths, bytes): topPaths = [topPaths]
- regSpecs = [s for s in regSpecs if s != '' and s != None]
- wildCards = [s for s in wildCards if s != '' and s != None]
-
- haveRegs = False; regs = []; haveWilds = False; haveMatchFunction = False
- if len(regSpecs) > 0:
- haveRegs = True
- regs = []
- for reg in regSpecs:
- (pattern, subst) = parse_re_with_subst(reg)
- regs.append( (re.compile(pattern), subst) )
- if len(wildCards) > 0:
- haveWilds = True
-
- prefix = ''
- extra = ''
- suffix = ''
- if deleteMode:
- suffix += ' deleted.'
- if '.' in topPaths:
- die("filelist: Recursively deleting from the dot (.) path is not safe. Shame.")
-
- if directoryMode: listMode = False
- if listMode: getFileInfo = True
- if quietMode: stream = None
- sumSizes = 0
- if xmlMode:
- matchedFiles.append('<files>')
- fetchedFiles.append('<files>')
- _output('<files>', destinationFiles, stream)
- prefix += ' <file>'
- suffix += '</file>'
-
- for top in topPaths:
- if verboseMode: warn('filelist: searching', top)
- topMatchCount = 0; topFetchCount = 0
-
- for root, dirs, files, infos in walk(top, userCredentials, walkDirectories, topDown):
- if verboseMode: warn('filelist: found files in', root)
- remote, protocol, netloc, path = remoteUrl(root)
- if directoryMode:
- contents = dirs
- else:
- contents = files
-
- for i in range(len(contents)):
- line = ''
- file = contents[i]
- try:
- info = infos[i]
- except:
- info = None
- if matchUrl:
- name = os.path.join(root, file)
- else:
- name = file
-
- match, newname = matchFunction(root, name, haveRegs, regs,
- haveWilds, wildCards, constraintFunction)
- if match:
- line = ''
- topMatchCount += 1
- fn = os.path.join(root, file)
-
- if getFileInfo or (fetchIfNewer and not remote):
- if remote:
- if info and getFileInfo:
- if listMode: line = info.line
- extra = ' ' + str(info.size) + ' ' + str(info.modTime)
- sumSizes += info.size
- else:
- st = os.stat(fn)
- line = ' '.join( map(str, \
- (st.st_mode, st.st_uid, st.st_gid, st.st_size, st.st_mtime, fn)))
- info = FileInfo(line, st.st_size, st.st_mtime, st.st_uid, st.st_gid, st.st_mode)
- if getFileInfo:
- extra = ' ' + str(info.size) + ' ' + str(info.modTime)
- sumSizes += info.size
-
- if not remote and urlMode: fn = makeFileUrl(fn)
- matchedFiles.append(prefix + fn + extra + suffix)
-
- if matchUrl:
- newfn = newname
- else:
- newfn = os.path.join(root, newname)
- newr, newp, newloc, newpath = remoteUrl(newfn)
- newfile = os.path.split(newpath)[1]
-
- if fetchDir:
- if fetchDir == '.': fetchDir = os.getcwd()
- if fetchWithSubDirs:
- destDir = os.path.join(fetchDir, newpath[1:])
- else:
- destDir = fetchDir
- destFile = os.path.join(destDir, newfile)
- tmpFile = os.path.join(workDir, newfile)
-
- if shouldFetch(remote, destFile, fetchIfNewer, info):
- if not quietMode:
- warn('filelist: Fetching ', fn)
- warn('filelist: Writing ', destFile)
- try:
- os.makedirs(destDir)
- except:
- # kludge, makedirs throws exception if any part of path exists
- pass
- if remote:
- urllib.request.urlretrieve(fn, tmpFile)
- else:
- shutil.copyfile(fn, tmpFile)
- os.rename(tmpFile, destFile) # atomic rename of file into destDir
-
- topFetchCount += 1
- fetchedFiles.append(prefix + fn + suffix)
- if getFileInfo: line = line + ' ' + destFile
-
- # now rewrite URL to point to local copy of file
- fn = destFile
- if not remote and urlMode: fn = makeFileUrl(fn)
-
- if not listMode:
- line = prefix + fn + extra + suffix
- _output(line, destinationFiles, stream)
- if deleteMode:
- if remote:
- die('filelist: Cannot delete remote files (yet)')
- else:
- os.unlink(fn)
-
- if verboseMode and fetchDir:
- warn('filelist: Matched %d files from %s' % (topMatchCount, top))
- warn('filelist: Fetched %d files from %s' % (topFetchCount, top))
- if fetchDir:
- for f in os.listdir(workDir): os.remove(os.path.join(workDir, f))
- os.rmdir(workDir)
-
- if xmlMode:
- matchedFiles.append('</files>')
- fetchedFiles.append('</files>')
- _output('<files>', destinationFiles, stream)
-
- if getFileInfo:
- if xmlMode:
- line = '<totalSize>%s</totalSize>' % sumSizes
- else:
- line = '#filelist: total size %s' % sumSizes
- matchedFiles.append(line)
- _output(line, destinationFiles, stream)
-
- except KeyboardInterrupt:
- if fetchDir:
- for f in os.listdir(workDir): os.remove(os.path.join(workDir, f))
- os.rmdir(workDir)
- die('filelist: Keyboard Interrupt')
-
- return (matchedFiles, fetchedFiles, destinationFiles)
-
-
-def shouldFetch(remote, destFile, fetchIfNewer, srcFileInfo):
- if remote:
- if os.path.exists(destFile):
- doFetch = False
- else:
- doFetch = True
- else:
- if os.path.exists(destFile):
- if fetchIfNewer:
- destModTime = os.path.getmtime(destFile)
- if destModTime < srcFileInfo.modTime:
- doFetch = True
- else:
- doFetch = False
- else:
- doFetch = False
- else:
- doFetch = True
- return doFetch
-
-def _output(line, lines, stream=None):
- """Internal function: Add line to output lines and optionally print to stream."""
- lines.append(line)
- if stream: print(line, file=stream)
-
-class FileInfo:
- """Holder class for those file info. elements that are consistent among local
- files (output of stat), ftp directories, http, etc. Minimum useful fields are
- modification time and size. Line contains usual string output of ls -l.
- """
- def __init__(self, line, size, modTime, userId=None, groupId=None, protectMode=None):
- self.line=line; self.size=size; self.modTime=modTime
- self.userId=userId; self.groupId=groupId; self.protectMode=protectMode
-
-class UserCredential(object):
- """Container for user credential info. like username, password, certificate, etc.
- """
- def __init__(self, username=None, password=None, validInterval=None, certificate=None):
- self.username = username
- self.password = password
- self.validInterval = validInterval # tuple of Ints (days, hours, minutes)
- if password is not None and validInterval is None:
- die('UserCredential: If password is present, validInterval is also required.')
- self.certificate = certificate
-
- def getPassword(self):
- pw = self._password
- if pw:
- pw, daynumber, timestamp = dataenc.pass_dec(pw)
- if dataenc.unexpired(daynumber, timestamp, self.validInterval):
- return pw
- else:
- return None
- else:
- return None
- def setPassword(self, pw):
- if pw and pw != '':
- self._password = dataenc.pass_enc(pw, daynumber=True, timestamp=True)
- else:
- self._password = pw
- password = property(getPassword, setPassword)
-
-class UserCredentials:
- """Contains dictionary of (url, credential) pairs and optionally an httpProxy.
- """
- def __init__(self, httpProxy=None, credentials={}):
- self.httpProxy = httpProxy
- self.credentials = credentials
- def add(self, url, credential):
- self.credentials[url] = credential; return self
- def forUrl(self, url):
- for key in self.credentials:
- if url.startswith(key):
- return self.credentials[key]
- return None
-
-def promptForCredentials(urls, httpProxy=None):
- if httpProxy == None:
- httpProxy = input('Enter HTTP proxy [none]: ')
- if httpProxy == '': httpProxy = None
- credentials = UserCredentials(httpProxy)
- localUserName = getpass.getuser()
- for url in urls:
- remote, protocol, netloc, path = remoteUrl(url)
- if remote:
- username, password, validInterval = promptForCredential(url, localUserName)
- credential = UserCredential(username, password, validInterval)
- credentials.add(url, credential)
- return credentials
-
-def promptForCredential(url, localUserName):
- remote, protocol, netloc, path = remoteUrl(url)
- if protocol == 'ftp':
- defaultUserName = 'anonymous'
- else:
- defaultUserName = localUserName
- username = input('Need credentials for URL %s\nUsername [%s]: ' \
- % (url, defaultUserName))
- if username == '': username = defaultUserName
- password = ''
- while password == '':
- password = getpass.getpass()
- validInterval = [0, 1, 0]
- if password != '':
- response = input('Enter valid time period for credential [(days, hours, minutes) = 0 1 0]: ')
- if response != '':
- validInterval = response.split()
- return (username, password, validInterval)
-
-class DirectoryWalker:
- """Recursively walk directories using the protocol specified in a URL.
- Sublclasses handle ftp, http, sftp, local file system, etc.
- """
- def __init__(self, userCredentials=None, retries=3, sleepTime=5):
- self.userCredentials = userCredentials
- self.retries = retries
- self.sleepTime = sleepTime
-
- def walk(self, top, walkDirectories=True):
- """Recursively walk directories on a remote site to retrieve file lists.
- """
- remote, protocol, netloc, path = remoteUrl(top)
- status, dir_listing = self.retrieveDirList(top)
- if status:
- if len(dir_listing) == 0:
- yield (top, [], [], [])
- else:
- (dirs, files, infos) = self.parseDirList(dir_listing, path)
- yield (top, dirs, files, infos)
-
- if walkDirectories:
- for dir in dirs:
- # Depth-first recursion
- for root, dirs, files, infos in self.walk(top + '/' + dir, walkDirectories):
- yield (root, dirs, files, infos)
- else:
- warn('DirectoryWalker: error, unable to retrieve directory listing at', top)
- yield (top, [], [], [])
-
- def retrieveDirList(self, url):
- """Retrieve directory listing as a list of text lines. Returns (status, dirList)."""
- pass
- def parseDirList(self, dirList, path=None):
- """Parse directory listing (text) and return three lists (dirs, files, fileInfos)."""
- pass
-
-class FtpDirectoryWalker(DirectoryWalker):
- """Recursively walk directories on an ftp site."""
- def __init__(self, userCredentials=None, retries=3, sleepTime=5):
- DirectoryWalker.__init__(self, userCredentials, retries, sleepTime)
-
- def retrieveDirList(self, url):
- """Retrieve a directory listing via ftp with retries.
- """
- remote, protocol, netloc, path = remoteUrl(url)
- credential = None
- if self.userCredentials:
- credential = self.userCredentials.forUrl(url)
- dir = ''; dir_list = []
- ftp = FTP()
- for i in range(self.retries):
- try:
- ftp.connect(netloc)
- if credential is None or \
- credential.username == 'anonymous' or \
- credential.username == '':
- ftp.login()
- else:
- ftp.login(credential.username, credential.password)
- ftp.cwd(path)
- ftp.retrlines('LIST', dir_list.append)
- ftp.quit()
- dir = '\n'.join(dir_list)
- return (True, dir)
- except:
- pass
- time.sleep(self.sleepTime)
- warn('FtpDirectoryWalker: connect retry to ', netloc, path)
- return (False, dir)
-
- def parseDirList(self, dir, path=None):
- """Parse long directory listing returned by ftp or (ls -l).
- Separate entries into directories and files.
- """
- dirs = []; files = []; infos = []
- for entry in dir.split('\n'):
- fields = entry.split()
- if len(fields) < 7: continue
- fn = fields[-1]
- if fn == '.' or fn == '..': continue
- if re.match('^d', fields[0])and fields[0][7] == 'r':
- dirs.append(fn)
- else:
- files.append(fn)
- info = FileInfo(entry, int(fields[4]), '-'.join(fields[5:8]), \
- fields[2], fields[3], fields[0])
- infos.append(info)
- return (dirs, files, infos)
-
-class DirListingParser(object):
- """Base class for directory listing parsers."""
- def __init__(self, regex):
- self.regex = regex
- self.compiledRegex = re.compile(self.regex)
-
- def parse(self, dir, listingHtml):
- """Return (dirs, files, infos)."""
- dirs = []; files = []; infos = []
- raise NotImplementedError("Override this method in sub class.")
-
-class ApacheDirListingParser(DirListingParser):
- """Parser class for apache."""
- def parse(self, dir, listingHtml):
- dirs = []; files = []; infos = []
- items = self.compiledRegex.findall(listingHtml)
- for item, itemName in items:
- if itemName.strip() == 'Parent Directory': continue
- if isinstance(item, str):
- name = item
- else:
- name, dateTime, size = item[:]
-
- if name.endswith('/'):
- type = 'd'
- dirs.append(name[:-1])
- else:
- type = '-'
- files.append(name)
- #not doing file info
- '''
- size = size.lower()
- if size.endswith('k'):
- size = int(size[:-1]) * 1024
- elif size.endswith('m'):
- size = int(size[:-1]) * 1024 * 1024
- else:
- size = -1
- line = '%s--------- 1 ? ? %15d %s %s' % (type, size, dateTime, name)
- info = FileInfo(line, size, dateTime)
- '''
- infos.append(None)
- return (dirs, files, infos)
-
-class CDAACDirListingParser(DirListingParser):
- """Parser class for CDAAC data server."""
- def parse(self, dir, listingHtml):
- dirs = []; files = []; infos = []
- items = self.compiledRegex.findall(listingHtml)
- for item, itemName in items:
- if itemName.strip() == 'Parent Directory': continue
- if isinstance(item, str):
- name = item
- else:
- name, dateTime, size = item[:]
- if name.endswith('/'):
- type = 'd'
- dirs.append(name)
- else:
- type = '-'
- files.append(name)
- #not doing file info
- '''
- size = size.lower()
- if size.endswith('k'):
- size = int(size[:-1]) * 1024
- elif size.endswith('m'):
- size = int(size[:-1]) * 1024 * 1024
- else:
- size = -1
- line = '%s--------- 1 ? ? %15d %s %s' % (type, size, dateTime, name)
- info = FileInfo(line, size, dateTime)
- '''
- infos.append(None)
- return (dirs, files, infos)
-
-class HttpDirectoryWalker(DirectoryWalker):
- """Recursively walk directories on an http (web) site to retrieve file lists.
- Handles many styles of HTML directory listings, but still very FRAGILE.
- """
-
- #list of directory listing parser plugins
- DIR_LIST_REGEX_PLUGINS = [
- #apache 2.0.55 directory listing
- ApacheDirListingParser(r'(?i)alt="\[.*?\]">\s*<A HREF="(?P<name>.*?)">(.*?)</A>'),
- #CDAAC (COSMIC Data)
- CDAACDirListingParser(r'(?i)<LI><A HREF="(?P<name>.*?)">(.*?)</A>'),
- ]
-
- def __init__(self, userCredentials=None, retries=3, sleepTime=5):
- DirectoryWalker.__init__(self, userCredentials, retries, sleepTime)
- if self.userCredentials:
- if self.userCredentials.httpProxy:
- os.environ['http_proxy'] = self.userCredentials.httpProxy
- # global kludge, default proxyHandler looks up proxy there
- passwordMgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
- for url, cred in self.userCredentials.credentials.items():
- passwordMgr.add_password(None, url, cred.username, cred.password)
- authHandler = urllib.request.HTTPBasicAuthHandler(passwordMgr)
- opener = urllib.request.build_opener(authHandler)
- else:
-# opener = urllib2.build_opener()
- opener = None
-# opener.add_headers = [('User-agent', 'Mozilla/5.0')]
- self.opener = opener
-
- def retrieveDirList(self, url):
- """Retrieve an HTML directory listing via http with retries.
- """
-### url = os.path.join(url, 'contents.html') ### hack for DAP servers at GES-DISC
- dir_listing = ''
- proxies = {}
- for i in range(self.retries):
- try:
- if self.opener:
- response = self.opener.open(url)
- else:
- response = urllib.request.urlopen(url)
- except IOError as e:
- if hasattr(e, 'reason'):
- warn('HttpDirectoryWalker: Error, failed to reach server because: %s' % e.reason)
- elif hasattr(e, 'code'):
- warn('HttpDirectoryWalker: Server could not fulfill request, error code %s' % e.code)
- else:
- dir_listing = response.read()
- return (True, dir_listing)
- time.sleep(self.sleepTime)
- warn('HttpDirectoryWalker: retrying ', url)
- return (False, dir_listing)
-
- reDirPath = re.compile(r'(?i)<H1>.*?Index of\s*?(\S+?)\s*?</H1>')
-
- def parseDirList(self, dir, path):
- """Parse fragile HTML directory listings returned by various HTTP servers,
- including Apache and OpenDAP. Separate entries into directories and files.
- """
- dirs = []; files = []; infos = []
- if path:
- match = HttpDirectoryWalker.reDirPath.search(dir)
- if not match:
- die('HttpDirectoryWalker: Cannot find directory name %s in HTML listing:\n%s' % (path, dir))
- dirName = match.group(1)
- if dirName not in path:
- warn('HttpDirectoryWalker: Directory name %s in HTML listing does not agree with path %s:\n%s' % (dirName, path, dir))
-
- # Try to find directory lines that contain file info
- reDirListWithStat = re.compile( \
- r'(?i)<A HREF=[\'"]*?(?P<name>[^\?].*?' + dirName + r'.*?)[\'"]*?>.*?</A>\s*(?P<dateTime>\S+ \S+)\s+?(?P<size>\S+)\s*?$')
- items = reDirListWithStat.findall(dir)
- # If not, then try to find simple directory lines
- if len(items) == 0:
... 1178 lines suppressed ...