You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by hu...@apache.org on 2013/06/26 00:15:24 UTC
svn commit: r1496657 - in
/incubator/climate/trunk/rcmet/src/main/python/rcmes: services/ storage/
utils/
Author: huikyole
Date: Tue Jun 25 22:15:23 2013
New Revision: 1496657
URL: http://svn.apache.org/r1496657
Log:
Review Board #11785 - Updating any modules using PyNio. Applying the latest changes
Modified:
incubator/climate/trunk/rcmet/src/main/python/rcmes/services/dataset_helpers.py
incubator/climate/trunk/rcmet/src/main/python/rcmes/services/decode_model_times.py
incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_latlon_var.py
incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_time_var.py
incubator/climate/trunk/rcmet/src/main/python/rcmes/services/list_vars_in_file.py
incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/db.py
incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/files.py
incubator/climate/trunk/rcmet/src/main/python/rcmes/utils/misc.py
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/services/dataset_helpers.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/services/dataset_helpers.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/services/dataset_helpers.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/services/dataset_helpers.py Tue Jun 25 22:15:23 2013
@@ -22,7 +22,6 @@
from bottle import request, route
import requests
-import Nio
@route('/getObsDatasets')
def getObservationDatasetData():
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/services/decode_model_times.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/services/decode_model_times.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/services/decode_model_times.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/services/decode_model_times.py Tue Jun 25 22:15:23 2013
@@ -104,9 +104,9 @@ def decode_model_times(filelist,timeVarN
import string
import math
import numpy
- import Nio
+ import netCDF4
- f = Nio.open_file(filelist[0])
+ f = netCDF4.Dataset(filelist[0], mode='r')
xtimes = f.variables[timeVarName]
timeFormat = xtimes.units
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_latlon_var.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_latlon_var.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_latlon_var.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_latlon_var.py Tue Jun 25 22:15:23 2013
@@ -49,7 +49,7 @@
"""
import sys
-import Nio
+import netCDF4
import bottle
from bottle import request
import json
@@ -60,11 +60,11 @@ import json
def find_latlon(filename):
success = 0
filename = filename.strip('"')
- f = Nio.open_file(filename)
- var_name_list = f.variables.keys()
+ f = netCDF4.Dataset(filename, mode='r')
# convert all variable names into lower case
- var_name_list_lower = [x.lower() for x in var_name_list]
+ var_name_list_lower = [key.encode().lower() for key in f.variables.keys()]
+ var_name_list = var_name_list_lower
# create a "set" from this list of names
varset = set(var_name_list_lower)
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_time_var.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_time_var.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_time_var.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/services/find_time_var.py Tue Jun 25 22:15:23 2013
@@ -45,7 +45,7 @@
import sys
import bottle
from bottle import request
-import Nio
+import netCDF4
import json
import decode_model_times as dmt
@@ -57,10 +57,10 @@ import decode_model_times as dmt
def list_time(filename):
filename = filename.strip('"')
success = 0
- f = Nio.open_file(filename)
- var_name_list = f.variables.keys()
+ f = netCDF4.Dataset(filename, mode='r')
# convert all variable names into lower case
- var_name_list_lower = [x.lower() for x in var_name_list]
+ var_name_list_lower = [key.encode().lower() for key in f.variables.keys()]
+ var_name_list = var_name_list_lower
# create a "set" from this list of names
varset = set(var_name_list_lower)
# Use "set" types for finding common variable name from in the file and from the list of possibilities
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/services/list_vars_in_file.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/services/list_vars_in_file.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/services/list_vars_in_file.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/services/list_vars_in_file.py Tue Jun 25 22:15:23 2013
@@ -40,7 +40,7 @@
"""
import sys
-import Nio
+import netCDF4
import bottle
from bottle import request
import json
@@ -53,13 +53,13 @@ def list_vars(filename):
filename = filename.strip('"')
print filename + ' is filename variable'
try:
- f = Nio.open_file(filename)
+ f = netCDF4.Dataset(filename, mode='r')
success = 1
except:
print 'Error_reading_file '+filename
if success: #make some json
- var_name_list = json.dumps({'variables':f.variables.keys() }, \
+ var_name_list = json.dumps({'variables':f.variables.keys().encode() }, \
sort_keys=True, indent=2)
if (request.query.callback):
return "%s(%s)" % (request.query.callback, var_name_list)
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/db.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/db.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/db.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/db.py Tue Jun 25 22:15:23 2013
@@ -178,7 +178,7 @@ def read_netcdf(netCD_fileName):
longitudes = netcdf.variables['lon'][:]
levels = netcdf.variables['lev'][:]
hours = netcdf.variables['time'][:]
- values = netcdf.variables['value'][:]
+ values = ma.array(netcdf.variables['value'][:])
# To get the base date
time_unit=netcdf.variables['time'].units.encode()
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/files.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/files.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/files.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/storage/files.py Tue Jun 25 22:15:23 2013
@@ -161,11 +161,11 @@ def read_data_from_file_list(filelist, m
# as no assumption made that same number of times in each file...
- for ifile in filelist:
+ for i,ifile in enumerate(filelist):
#print 'Loading data from file: ',filelist[i]
f = netCDF4.Dataset(ifile, mode='r')
- t2raw = f.variables[myvar][:]
+ t2raw = ma.array(f.variables[myvar][:])
timesraw = f.variables[timeVarName]
time = timesraw[:]
ntimes = len(time)
@@ -189,7 +189,6 @@ def read_data_from_file_list(filelist, m
timestore[timesaccu + np.arange(ntimes)] = time
timesaccu = timesaccu + ntimes
f.close()
- i += 1
print 'Data read in successfully with dimensions: ', t2store.shape
@@ -329,11 +328,10 @@ def read_data_from_one_file(ifile, myvar
varUnit = f.variables[myvar].units.encode().upper()
except:
varUnit = raw_input('Enter the model variable unit: \n> ').upper()
- t2raw = f.variables[myvar][:]
+ t2raw = ma.array(f.variables[myvar][:])
t2tmp = t2raw.squeeze()
if t2tmp.ndim == 2:
t2tmp = np.expand_dims(t2tmp, 0)
- t2tmp = t2tmp
f.close()
print ' success read_data_from_one_file: VarName=', myvar, ' Shape(Full)= ', t2tmp.shape, ' Unit= ', varUnit
timestore = process.decode_model_timesK(ifile, timeVarName, file_type)
@@ -484,7 +482,7 @@ def read_data_from_file_list_K(filelist,
for ifile in filelist:
#print 'Loading data from file: ',filelist[i]
f = netCDF4.Dataset(ifile, mode='r')
- t2raw = f.variables[myvar][:]
+ t2raw = ma.array(f.variables[myvar][:])
timesraw = f.variables[timeVarName]
time = timesraw[0:ntimes]
#ntimes=len(time)
Modified: incubator/climate/trunk/rcmet/src/main/python/rcmes/utils/misc.py
URL: http://svn.apache.org/viewvc/incubator/climate/trunk/rcmet/src/main/python/rcmes/utils/misc.py?rev=1496657&r1=1496656&r2=1496657&view=diff
==============================================================================
--- incubator/climate/trunk/rcmet/src/main/python/rcmes/utils/misc.py (original)
+++ incubator/climate/trunk/rcmet/src/main/python/rcmes/utils/misc.py Tue Jun 25 22:15:23 2013
@@ -476,7 +476,7 @@ def read_trmm_3b42_files(filelist, latMi
for ifile in filelist:
print 'Loading data from file: ', filelist[i]
f = netCDF4.Dataset(ifile, mode='r')
- t2raw = f.variables['hrf']
+ t2raw = ma.array(f.variables['hrf'][:])
# Read time from filename (NB. 'time' variable in netCDF always set to zero)
filename = os.path.basename(ifile) # strip off the filepath
@@ -571,7 +571,7 @@ def read_airs_lev3_files(filelist, myvar
for ifile in filelist:
print 'Loading data from file: ', filelist[i]
f = netCDF4.open_file(ifile, mode='r')
- t2raw = f.variables[myvar]
+ t2raw = ma.array(f.variables[myvar][:])
# Read time from filename (NB. 'time' variable in netCDF always set to zero)
filename = os.path.basename(ifile) # strip off the filepath
@@ -818,7 +818,7 @@ def read_eraint_surf_files(filelist, myv
for ifile in filelist:
print 'Loading data from file: ', filelist[i]
f = netCDF4.Dataset(ifile, mode='r')
- data = f.variables[myvar][:]
+ data = ma.array(f.variables[myvar][:])
scale = f.variables[myvar].scale_factor
offset = f.variables[myvar].add_offset
data = data * scale + offset