You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by fg...@apache.org on 2019/01/23 17:56:24 UTC
[incubator-sdap-ningesterpy] 01/29: initial commit
This is an automated email from the ASF dual-hosted git repository.
fgreg pushed a commit to branch v1.0.0-rc1
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ningesterpy.git
commit df17d9455f909afb4fb6b4b20215b0241b448cd7
Author: Frank Greguska <fr...@jpl.nasa.gov>
AuthorDate: Wed Dec 27 16:29:08 2017 -0600
initial commit
---
.gitignore | 9 +
ningesterpy/__init__.py | 5 +
ningesterpy/ningesterpy.py | 80 +++++
ningesterpy/processors/__init__.py | 66 +++++
ningesterpy/processors/callncpdq.py | 48 +++
ningesterpy/processors/callncra.py | 48 +++
ningesterpy/processors/computespeeddirfromuv.py | 68 +++++
ningesterpy/processors/emptytilefilter.py | 31 ++
ningesterpy/processors/kelvintocelsius.py | 21 ++
.../processors/normalizetimebeginningofmonth.py | 30 ++
ningesterpy/processors/processorchain.py | 72 +++++
ningesterpy/processors/regrid1x1.py | 136 +++++++++
ningesterpy/processors/subtract180longitude.py | 32 ++
ningesterpy/processors/tilereadingprocessor.py | 266 +++++++++++++++++
ningesterpy/processors/tilesummarizingprocessor.py | 93 ++++++
ningesterpy/processors/winddirspeedtouv.py | 90 ++++++
requirements.txt | 12 +
setup.py | 29 ++
tests/__init__.py | 4 +
tests/callncpdq_test.py | 53 ++++
tests/computespeeddirfromuv_test.py | 114 +++++++
tests/convert_iceshelf.py | 78 +++++
tests/datafiles/empty_mur.nc4 | Bin 0 -> 60937 bytes
tests/datafiles/not_empty_ascatb.nc4 | Bin 0 -> 78036 bytes
tests/datafiles/not_empty_avhrr.nc4 | Bin 0 -> 49511 bytes
tests/datafiles/not_empty_ccmp.nc | Bin 0 -> 206870 bytes
tests/datafiles/not_empty_measures_alt.nc | Bin 0 -> 45477 bytes
tests/datafiles/not_empty_mur.nc4 | Bin 0 -> 60907 bytes
tests/datafiles/not_empty_smap.h5 | Bin 0 -> 3000192 bytes
tests/datafiles/not_empty_wswm.nc | Bin 0 -> 33119 bytes
tests/datafiles/partial_empty_mur.nc4 | Bin 0 -> 84738 bytes
.../ascat_longitude_more_than_180.bin | Bin 0 -> 3858 bytes
.../ascatb_nonempty_nexustile.bin | Bin 0 -> 3515 bytes
.../dumped_nexustiles/avhrr_nonempty_nexustile.bin | Bin 0 -> 892 bytes
.../dumped_nexustiles/ccmp_nonempty_nexustile.bin | Bin 0 -> 27427 bytes
.../dumped_nexustiles/smap_nonempty_nexustile.bin | Bin 0 -> 1374 bytes
tests/hd5splitter.py | 123 ++++++++
tests/kelvintocelsius_test.py | 40 +++
tests/processorchain_test.py | 92 ++++++
tests/regrid1x1_test.py | 79 +++++
tests/subtract180longitude_test.py | 57 ++++
tests/tilereadingprocessor_test.py | 330 +++++++++++++++++++++
tests/tilesumarizingprocessor_test.py | 80 +++++
tests/winddirspeedtouv_test.py | 89 ++++++
44 files changed, 2275 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ba846b9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+*.pyc
+
+.idea
+
+.DS_Store
+
+*.egg-info
+build
+dist
diff --git a/ningesterpy/__init__.py b/ningesterpy/__init__.py
new file mode 100644
index 0000000..86f81aa
--- /dev/null
+++ b/ningesterpy/__init__.py
@@ -0,0 +1,5 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+from ningesterpy import ningesterpy
\ No newline at end of file
diff --git a/ningesterpy/ningesterpy.py b/ningesterpy/ningesterpy.py
new file mode 100644
index 0000000..e9c4fde
--- /dev/null
+++ b/ningesterpy/ningesterpy.py
@@ -0,0 +1,80 @@
+import logging
+import uuid
+
+import nexusproto.NexusContent_pb2 as nexusproto
+from flask import Flask, request, jsonify, Response
+from flask.json import JSONEncoder
+from flask_accept import accept
+from google.protobuf import json_format
+from werkzeug.exceptions import HTTPException, BadRequest
+from werkzeug.exceptions import default_exceptions
+
+from processors.processorchain import ProcessorChain, ProcessorNotFound, MissingProcessorArguments
+
+applog = logging.getLogger(__name__)
+app = Flask(__name__)
+
+
+class ProtobufJSONEncoder(JSONEncoder):
+ def default(self, obj):
+ try:
+ if isinstance(obj, nexusproto.NexusTile):
+ json_obj = json_format.MessageToJson(obj)
+ return json_obj
+ iterable = iter(obj)
+ except TypeError:
+ pass
+ else:
+ return list(iterable)
+ return JSONEncoder.default(self, obj)
+
+
+@app.route('/processorchain', methods=['POST'], )
+@accept('application/octet-stream', '*/*')
+def run_processor_chain():
+ try:
+ parameters = request.get_json()
+ except Exception as e:
+ raise BadRequest("Invalid JSON data") from e
+
+ try:
+ processor_list = parameters['processor_list']
+ except (KeyError, TypeError):
+ raise BadRequest(description="processor_list is required.")
+
+ try:
+ chain = ProcessorChain(processor_list)
+ except ProcessorNotFound as e:
+ raise BadRequest("Unknown processor requested: %s" % e.missing_processor) from e
+ except MissingProcessorArguments as e:
+ raise BadRequest(
+ "%s missing required configuration options: %s" % (e.processor, e.missing_processor_args)) from e
+
+ input_data = parameters['input_data']
+
+ result = next(chain.process(input_data), None)
+
+ if isinstance(result, nexusproto.NexusTile):
+ result = result.SerializeToString()
+
+ return Response(result, mimetype='application/octet-stream')
+
+
+def handle_error(e):
+ error_id = uuid.uuid4()
+
+ app.logger.exception("Exception %s" % error_id)
+ code = 500
+ message = "Internal server error"
+ if isinstance(e, HTTPException):
+ code = e.code
+ message = str(e)
+ return jsonify(message=message, error_id=error_id), code
+
+
+if __name__ == '__main__':
+ app.register_error_handler(Exception, handle_error)
+ for ex in default_exceptions:
+ app.register_error_handler(ex, handle_error)
+ app.json_encoder = ProtobufJSONEncoder
+ app.run()
diff --git a/ningesterpy/processors/__init__.py b/ningesterpy/processors/__init__.py
new file mode 100644
index 0000000..5bb9094
--- /dev/null
+++ b/ningesterpy/processors/__init__.py
@@ -0,0 +1,66 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+from collections import defaultdict
+
+import nexusproto.NexusContent_pb2 as nexusproto
+
+
+class Processor(object):
+ def __init__(self, *args, **kwargs):
+ self.environ = defaultdict(lambda: None)
+ for k, v in kwargs.items():
+ self.environ[k.upper()] = v
+ pass
+
+ def process(self, input_data):
+ raise NotImplementedError
+
+
+class NexusTileProcessor(Processor):
+ @staticmethod
+ def parse_input(input_data):
+ if isinstance(input_data, nexusproto.NexusTile):
+ return input_data
+ else:
+ return nexusproto.NexusTile.FromString(input_data)
+
+ def process(self, input_data):
+ nexus_tile = self.parse_input(input_data)
+
+ for data in self.process_nexus_tile(nexus_tile):
+ yield data
+
+ def process_nexus_tile(self, nexus_tile):
+ raise NotImplementedError
+
+# All installed processors need to be imported and added to the dict below
+
+from processors.callncpdq import CallNcpdq
+from processors.callncra import CallNcra
+from processors.computespeeddirfromuv import ComputeSpeedDirFromUV
+from processors.emptytilefilter import EmptyTileFilter
+from processors.kelvintocelsius import KelvinToCelsius
+from processors.normalizetimebeginningofmonth import NormalizeTimeBeginningOfMonth
+from processors.regrid1x1 import Regrid1x1
+from processors.subtract180longitude import Subtract180Longitude
+from processors.tilereadingprocessor import GridReadingProcessor, SwathReadingProcessor, TimeSeriesReadingProcessor
+from processors.tilesummarizingprocessor import TileSummarizingProcessor
+from processors.winddirspeedtouv import WindDirSpeedToUV
+
+INSTALLED_PROCESSORS = {
+ "CallNcpdq": CallNcpdq,
+ "CallNcra": CallNcra,
+ "ComputeSpeedDirFromUV": ComputeSpeedDirFromUV,
+ "EmptyTileFilter": EmptyTileFilter,
+ "KelvinToCelsius": KelvinToCelsius,
+ "NormalizeTimeBeginningOfMonth": NormalizeTimeBeginningOfMonth,
+ "Regrid1x1": Regrid1x1,
+ "Subtract180Longitude": Subtract180Longitude,
+ "GridReadingProcessor": GridReadingProcessor,
+ "SwathReadingProcessor": SwathReadingProcessor,
+ "TimeSeriesReadingProcessor": TimeSeriesReadingProcessor,
+ "TileSummarizingProcessor": TileSummarizingProcessor,
+ "WindDirSpeedToUV": WindDirSpeedToUV
+}
\ No newline at end of file
diff --git a/ningesterpy/processors/callncpdq.py b/ningesterpy/processors/callncpdq.py
new file mode 100644
index 0000000..3fee959
--- /dev/null
+++ b/ningesterpy/processors/callncpdq.py
@@ -0,0 +1,48 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+
+import logging
+import os
+from subprocess import call
+
+from processors import Processor
+
+
+class CallNcpdq(Processor):
+
+ def __init__(self, dimension_order, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.dimension_order = dimension_order
+ self.output_prefix = self.environ.get("OUTPUT_PREFIX", 'permuted_')
+ self.permute_variable = self.environ["PERMUTE_VARIABLE"]
+
+ def process(self, input_data):
+ """
+ input_data: Path to input netCDF file
+
+ If environment variable `PERMUTE_VARIABLE` is not set:
+ Calls ``ncpdq -a ${DIMENSION_ORDER} in_path ${OUTPUT_PREFIX}in_path``
+ Otherwise:
+ Calls ``ncpdq -v ${PERMUTE_VARIABLE} -a ${DIMENSION_ORDER} in_path ${OUTPUT_PREFIX}in_path``
+ """
+
+ output_filename = self.output_prefix + os.path.basename(input_data)
+ output_path = os.path.join(os.path.dirname(input_data), output_filename)
+
+ command = ['ncpdq', '-a', ','.join(self.dimension_order)]
+
+ if self.permute_variable:
+ command.append('-v')
+ command.append(self.permute_variable)
+
+ command.append(input_data)
+ command.append(output_path)
+
+ logging.debug('Calling command %s' % ' '.join(command))
+ retcode = call(command)
+ logging.debug('Command returned exit code %d' % retcode)
+
+ yield output_path
diff --git a/ningesterpy/processors/callncra.py b/ningesterpy/processors/callncra.py
new file mode 100644
index 0000000..d00e75b
--- /dev/null
+++ b/ningesterpy/processors/callncra.py
@@ -0,0 +1,48 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+
+import glob
+import os
+from subprocess import call
+
+from netCDF4 import Dataset, num2date
+
+from processors import Processor
+
+
+class CallNcra(Processor):
+ def __init__(self, output_filename_pattern, time_var_name, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.output_filename_pattern = output_filename_pattern
+ self.time_var_name = time_var_name
+
+ self.glob_pattern = self.environ.get("FILEMATCH_PATTERN", '*.nc')
+
+ def process(self, in_path):
+ target_datetime = self.get_datetime_from_dataset(in_path)
+ target_yearmonth = target_datetime.strftime('%Y%m')
+
+ output_filename = target_datetime.strftime(self.output_filename_pattern)
+ output_path = os.path.join(os.path.dirname(in_path), output_filename)
+
+ datasets = glob.glob(os.path.join(os.path.dirname(in_path), self.glob_pattern))
+
+ datasets_to_average = [dataset_path for dataset_path in datasets if
+ self.get_datetime_from_dataset(dataset_path).strftime('%Y%m') == target_yearmonth]
+
+ command = ['ncra', '-O']
+ command.extend(datasets_to_average)
+ command.append(output_path)
+ call(command)
+
+ yield output_path
+
+ def get_datetime_from_dataset(self, dataset_path):
+ with Dataset(dataset_path) as dataset_in:
+ time_units = getattr(dataset_in[self.time_var_name], 'units', None)
+ calendar = getattr(dataset_in[self.time_var_name], 'calendar', 'standard')
+ thedatetime = num2date(dataset_in[self.time_var_name][:].item(), units=time_units, calendar=calendar)
+ return thedatetime
diff --git a/ningesterpy/processors/computespeeddirfromuv.py b/ningesterpy/processors/computespeeddirfromuv.py
new file mode 100644
index 0000000..284a668
--- /dev/null
+++ b/ningesterpy/processors/computespeeddirfromuv.py
@@ -0,0 +1,68 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+
+import numpy
+from nexusproto.serialization import from_shaped_array, to_shaped_array
+
+from processors import NexusTileProcessor
+
+
+def calculate_speed_direction(wind_u, wind_v):
+ speed = numpy.sqrt(numpy.add(numpy.multiply(wind_u, wind_u), numpy.multiply(wind_v, wind_v)))
+ direction = numpy.degrees(numpy.arctan2(-wind_u, -wind_v)) % 360
+ return speed, direction
+
+
+class ComputeSpeedDirFromUV(NexusTileProcessor):
+
+ def __init__(self, wind_u_var_name, wind_v_var_name, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.wind_u_var_name = wind_u_var_name
+ self.wind_v_var_name = wind_v_var_name
+
+ def process_nexus_tile(self, nexus_tile):
+ the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
+
+ the_tile_data = getattr(nexus_tile.tile, the_tile_type)
+
+ # Either wind_u or wind_v are in meta. Whichever is not in meta is in variable_data
+ try:
+ wind_v = next(meta for meta in the_tile_data.meta_data if meta.name == self.wind_v_var_name).meta_data
+ wind_u = the_tile_data.variable_data
+ except StopIteration:
+ try:
+ wind_u = next(meta for meta in the_tile_data.meta_data if meta.name == self.wind_u_var_name).meta_data
+ wind_v = the_tile_data.variable_data
+ except StopIteration:
+ if hasattr(nexus_tile, "summary"):
+ raise RuntimeError(
+ "Neither wind_u nor wind_v were found in the meta data for granule %s slice %s."
+ " Cannot compute wind speed or direction." % (
+ getattr(nexus_tile.summary, "granule", "unknown"),
+ getattr(nexus_tile.summary, "section_spec", "unknown")))
+ else:
+ raise RuntimeError(
+ "Neither wind_u nor wind_v were found in the meta data. Cannot compute wind speed or direction.")
+
+ wind_u = from_shaped_array(wind_u)
+ wind_v = from_shaped_array(wind_v)
+
+ assert wind_u.shape == wind_v.shape
+
+ # Do calculation
+ wind_speed_data, wind_dir_data = calculate_speed_direction(wind_u, wind_v)
+
+ # Add wind_speed to meta data
+ wind_speed_meta = the_tile_data.meta_data.add()
+ wind_speed_meta.name = 'wind_speed'
+ wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed_data))
+
+ # Add wind_dir to meta data
+ wind_dir_meta = the_tile_data.meta_data.add()
+ wind_dir_meta.name = 'wind_dir'
+ wind_dir_meta.meta_data.CopyFrom(to_shaped_array(wind_dir_data))
+
+ yield nexus_tile
diff --git a/ningesterpy/processors/emptytilefilter.py b/ningesterpy/processors/emptytilefilter.py
new file mode 100644
index 0000000..ec30d2e
--- /dev/null
+++ b/ningesterpy/processors/emptytilefilter.py
@@ -0,0 +1,31 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import nexusproto.NexusContent_pb2 as nexusproto
+import numpy
+import logging
+from nexusproto.serialization import from_shaped_array
+
+from processors import NexusTileProcessor
+
+logger = logging.getLogger('emptytilefilter')
+
+def parse_input(nexus_tile_data):
+ return nexusproto.NexusTile.FromString(nexus_tile_data)
+
+
+class EmptyTileFilter(NexusTileProcessor):
+ def process_nexus_tile(self, nexus_tile):
+ the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
+
+ the_tile_data = getattr(nexus_tile.tile, the_tile_type)
+
+ data = from_shaped_array(the_tile_data.variable_data)
+
+ # Only supply data if there is actual values in the tile
+ if data.size - numpy.count_nonzero(numpy.isnan(data)) > 0:
+ yield nexus_tile
+ elif nexus_tile.HasField("summary"):
+ logger.warning("Discarding data %s from %s because it is empty" % (
+ nexus_tile.summary.section_spec, nexus_tile.summary.granule))
diff --git a/ningesterpy/processors/kelvintocelsius.py b/ningesterpy/processors/kelvintocelsius.py
new file mode 100644
index 0000000..361a50d
--- /dev/null
+++ b/ningesterpy/processors/kelvintocelsius.py
@@ -0,0 +1,21 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+
+from nexusproto.serialization import from_shaped_array, to_shaped_array
+
+from processors import NexusTileProcessor
+
+
+class KelvinToCelsius(NexusTileProcessor):
+ def process_nexus_tile(self, nexus_tile):
+ the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
+
+ the_tile_data = getattr(nexus_tile.tile, the_tile_type)
+
+ var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
+
+ the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
+
+ yield nexus_tile
diff --git a/ningesterpy/processors/normalizetimebeginningofmonth.py b/ningesterpy/processors/normalizetimebeginningofmonth.py
new file mode 100644
index 0000000..2be593c
--- /dev/null
+++ b/ningesterpy/processors/normalizetimebeginningofmonth.py
@@ -0,0 +1,30 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import datetime
+
+from pytz import timezone
+
+from processors import NexusTileProcessor
+
+EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1))
+
+
+class NormalizeTimeBeginningOfMonth(NexusTileProcessor):
+ def process_nexus_tile(self, nexus_tile):
+ the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
+
+ the_tile_data = getattr(nexus_tile.tile, the_tile_type)
+
+ time = the_tile_data.time
+
+ timeObj = datetime.datetime.utcfromtimestamp(time)
+
+ timeObj = timeObj.replace(day=1)
+
+ timeObj = timezone('UTC').localize(timeObj)
+
+ the_tile_data.time = int((timeObj - EPOCH).total_seconds())
+
+ yield nexus_tile
diff --git a/ningesterpy/processors/processorchain.py b/ningesterpy/processors/processorchain.py
new file mode 100644
index 0000000..aea605e
--- /dev/null
+++ b/ningesterpy/processors/processorchain.py
@@ -0,0 +1,72 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import inspect
+
+import processors
+
+
+class BadChainException(Exception):
+ pass
+
+
+class ProcessorNotFound(Exception):
+ def __init__(self, missing_processor, *args):
+ message = "Processor %s is not defined in INSTALLED_PROCESSORS. See processors/__init__.py" % missing_processor
+
+ self.missing_processor = missing_processor
+ super().__init__(message, *args)
+
+
+class MissingProcessorArguments(Exception):
+ def __init__(self, processor, missing_processor_args, *args):
+ message = "%s is missing required arguments: %s" % (processor, missing_processor_args)
+
+ self.processor = processor
+ self.missing_processor_args = missing_processor_args
+ super().__init__(message, *args)
+
+
+class ProcessorChain(processors.Processor):
+ def __init__(self, processor_list, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.processors = []
+ # Attempt to construct the needed processors
+ for processor in processor_list:
+ try:
+ processor_constructor = processors.INSTALLED_PROCESSORS[processor['name']]
+ except KeyError as e:
+ raise ProcessorNotFound(processor['name']) from e
+
+ missing_args = []
+ for arg in inspect.signature(processor_constructor).parameters.keys():
+ if arg in ['args', 'kwargs']:
+ continue
+ if arg not in processor['config']:
+ missing_args.append(arg)
+
+ if missing_args:
+ raise MissingProcessorArguments(processor['name'], missing_args)
+
+ if 'config' in processor.keys():
+ processor_instance = processor_constructor(**processor['config'])
+ else:
+ processor_instance = processor_constructor()
+
+ self.processors.append(processor_instance)
+
+ def process(self, input_data):
+
+ def recursive_processing_chain(gen_index, message):
+
+ next_gen = self.processors[gen_index + 1].process(message)
+ for next_message in next_gen:
+ if gen_index + 1 == len(self.processors) - 1:
+ yield next_message
+ else:
+ for result in recursive_processing_chain(gen_index + 1, next_message):
+ yield result
+
+ return recursive_processing_chain(-1, input_data)
diff --git a/ningesterpy/processors/regrid1x1.py b/ningesterpy/processors/regrid1x1.py
new file mode 100644
index 0000000..0248f1e
--- /dev/null
+++ b/ningesterpy/processors/regrid1x1.py
@@ -0,0 +1,136 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+
+import os
+from datetime import datetime
+
+import numpy as np
+from netCDF4 import Dataset
+from pytz import timezone
+from scipy import interpolate
+
+from processors import Processor
+
+UTC = timezone('UTC')
+ISO_8601 = '%Y-%m-%dT%H:%M:%S%z'
+
+
+class Regrid1x1(Processor):
+
+ def __init__(self, variables_to_regrid, latitude_var_name, longitude_var_name, time_var_name, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.variables_to_regrid = variables_to_regrid
+ self.latitude_var_name = latitude_var_name
+ self.longitude_var_name = longitude_var_name
+ self.time_var_name = time_var_name
+
+ self.filename_prefix = self.environ.get("FILENAME_PREFIX", '1x1regrid-')
+
+ vvr = self.environ['VARIABLE_VALID_RANGE']
+ if vvr:
+ vvr_iter = iter(vvr.split(':'))
+ self.variable_valid_range = {varrange[0]: [varrange[1], varrange[2]] for varrange in
+ zip(vvr_iter, vvr_iter, vvr_iter)}
+ else:
+ self.variable_valid_range = {}
+
+ def process(self, in_filepath):
+ in_path = os.path.join('/', *in_filepath.split(os.sep)[0:-1])
+ out_filepath = os.path.join(in_path, self.filename_prefix + in_filepath.split(os.sep)[-1])
+
+ with Dataset(in_filepath) as inputds:
+ in_lon = inputds[self.longitude_var_name]
+ in_lat = inputds[self.latitude_var_name]
+ in_time = inputds[self.time_var_name]
+
+ lon1deg = np.arange(np.floor(np.min(in_lon)), np.ceil(np.max(in_lon)), 1)
+ lat1deg = np.arange(np.floor(np.min(in_lat)), np.ceil(np.max(in_lat)), 1)
+ out_time = np.array(in_time)
+
+ with Dataset(out_filepath, mode='w') as outputds:
+ outputds.createDimension(self.longitude_var_name, len(lon1deg))
+ outputds.createVariable(self.longitude_var_name, in_lon.dtype, dimensions=(self.longitude_var_name,))
+ outputds[self.longitude_var_name][:] = lon1deg
+ outputds[self.longitude_var_name].setncatts(
+ {attrname: inputds[self.longitude_var_name].getncattr(attrname) for attrname in
+ inputds[self.longitude_var_name].ncattrs() if
+ str(attrname) not in ['bounds', 'valid_min', 'valid_max']})
+
+ outputds.createDimension(self.latitude_var_name, len(lat1deg))
+ outputds.createVariable(self.latitude_var_name, in_lat.dtype, dimensions=(self.latitude_var_name,))
+ outputds[self.latitude_var_name][:] = lat1deg
+ outputds[self.latitude_var_name].setncatts(
+ {attrname: inputds[self.latitude_var_name].getncattr(attrname) for attrname in
+ inputds[self.latitude_var_name].ncattrs() if
+ str(attrname) not in ['bounds', 'valid_min', 'valid_max']})
+
+ outputds.createDimension(self.time_var_name)
+ outputds.createVariable(self.time_var_name, inputds[self.time_var_name].dtype,
+ dimensions=(self.time_var_name,))
+ outputds[self.time_var_name][:] = out_time
+ outputds[self.time_var_name].setncatts(
+ {attrname: inputds[self.time_var_name].getncattr(attrname) for attrname in
+ inputds[self.time_var_name].ncattrs()
+ if
+ str(attrname) != 'bounds'})
+
+ for variable_name in self.variables_to_regrid.split(','):
+
+ # If longitude is the first dimension, we need to transpose the dimensions
+ transpose_dimensions = inputds[variable_name].dimensions == (
+ self.time_var_name, self.longitude_var_name, self.latitude_var_name)
+
+ outputds.createVariable(variable_name, inputds[variable_name].dtype,
+ dimensions=inputds[variable_name].dimensions)
+ outputds[variable_name].setncatts(
+ {attrname: inputds[variable_name].getncattr(attrname) for attrname in
+ inputds[variable_name].ncattrs()})
+ if variable_name in self.variable_valid_range.keys():
+ outputds[variable_name].valid_range = [
+ np.array([self.variable_valid_range[variable_name][0]],
+ dtype=inputds[variable_name].dtype).item(),
+ np.array([self.variable_valid_range[variable_name][1]],
+ dtype=inputds[variable_name].dtype).item()]
+
+ for ti in range(0, len(out_time)):
+ in_data = inputds[variable_name][ti, :, :]
+ if transpose_dimensions:
+ in_data = in_data.T
+
+ # Produces erroneous values on the edges of data
+ # interp_func = interpolate.interp2d(in_lon[:], in_lat[:], in_data[:], fill_value=float('NaN'))
+
+ x_mesh, y_mesh = np.meshgrid(in_lon[:], in_lat[:], copy=False)
+
+ # Does not work for large datasets (n > 5000)
+ # interp_func = interpolate.Rbf(x_mesh, y_mesh, in_data[:], function='linear', smooth=0)
+
+ x1_mesh, y1_mesh = np.meshgrid(lon1deg, lat1deg, copy=False)
+ out_data = interpolate.griddata(np.array([x_mesh.ravel(), y_mesh.ravel()]).T, in_data.ravel(),
+ (x1_mesh, y1_mesh), method='nearest')
+
+ if transpose_dimensions:
+ out_data = out_data.T
+
+ outputds[variable_name][ti, :] = out_data[np.newaxis, :]
+
+ global_atts = {
+ 'geospatial_lon_min': np.float(np.min(lon1deg)),
+ 'geospatial_lon_max': np.float(np.max(lon1deg)),
+ 'geospatial_lat_min': np.float(np.min(lat1deg)),
+ 'geospatial_lat_max': np.float(np.max(lat1deg)),
+ 'Conventions': 'CF-1.6',
+ 'date_created': datetime.utcnow().replace(tzinfo=UTC).strftime(ISO_8601),
+ 'title': getattr(inputds, 'title', ''),
+ 'time_coverage_start': getattr(inputds, 'time_coverage_start', ''),
+ 'time_coverage_end': getattr(inputds, 'time_coverage_end', ''),
+ 'Institution': getattr(inputds, 'Institution', ''),
+ 'summary': getattr(inputds, 'summary', ''),
+ }
+
+ outputds.setncatts(global_atts)
+
+ yield out_filepath
diff --git a/ningesterpy/processors/subtract180longitude.py b/ningesterpy/processors/subtract180longitude.py
new file mode 100644
index 0000000..b6ed693
--- /dev/null
+++ b/ningesterpy/processors/subtract180longitude.py
@@ -0,0 +1,32 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+from nexusproto.serialization import from_shaped_array, to_shaped_array
+
+from processors import NexusTileProcessor
+
+
+class Subtract180Longitude(NexusTileProcessor):
+ def process_nexus_tile(self, nexus_tile):
+ """
+ This method will transform longitude values in degrees_east from 0 TO 360 to -180 to 180
+
+ :param self:
+ :param nexus_tile: The nexus_tile
+ :return: Tile data with altered longitude values
+ """
+
+ the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
+
+ the_tile_data = getattr(nexus_tile.tile, the_tile_type)
+
+ longitudes = from_shaped_array(the_tile_data.longitude)
+
+ # Only subtract 360 if the longitude is greater than 180
+ longitudes[longitudes > 180] -= 360
+
+ the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes))
+
+ yield nexus_tile
+
diff --git a/ningesterpy/processors/tilereadingprocessor.py b/ningesterpy/processors/tilereadingprocessor.py
new file mode 100644
index 0000000..de63634
--- /dev/null
+++ b/ningesterpy/processors/tilereadingprocessor.py
@@ -0,0 +1,266 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import datetime
+from collections import OrderedDict
+from contextlib import contextmanager
+from os import sep, path, remove
+from urllib.request import urlopen
+
+import nexusproto.NexusContent_pb2 as nexusproto
+import numpy
+from netCDF4 import Dataset, num2date
+from nexusproto.serialization import to_shaped_array, to_metadata
+from pytz import timezone
+
+from processors import Processor
+
+EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1))
+
+
+@contextmanager
+def closing(thing):
+ try:
+ yield thing
+ finally:
+ thing.close()
+
+
+def parse_input(the_input, temp_dir):
+ # Split string on ';'
+ specs_and_path = [str(part).strip() for part in str(the_input).split(';')]
+
+ # Tile specifications are all but the last element
+ specs = specs_and_path[:-1]
+ # Generate a list of tuples, where each tuple is a (string, map) that represents a
+ # tile spec in the form (str(section_spec), { dimension_name : slice, dimension2_name : slice })
+ tile_specifications = [slices_from_spec(section_spec) for section_spec in specs]
+
+ # The path is the last element of the input split by ';'
+ file_path = specs_and_path[-1]
+ file_name = file_path.split(sep)[-1]
+ # If given a temporary directory location, copy the file to the temporary directory and return that path
+ if temp_dir is not None:
+ temp_file_path = path.join(temp_dir, file_name)
+ with closing(urlopen(file_path)) as original_granule:
+ with open(temp_file_path, 'wb') as temp_granule:
+ for chunk in iter((lambda: original_granule.read(512000)), ''):
+ temp_granule.write(chunk)
+
+ file_path = temp_file_path
+
+ # Remove file:// if it's there because netcdf lib doesn't like it
+ file_path = file_path[len('file://'):] if file_path.startswith('file://') else file_path
+
+ return tile_specifications, file_path
+
+
+def slices_from_spec(spec):
+ dimtoslice = {}
+ for dimension in spec.split(','):
+ name, start, stop = dimension.split(':')
+ dimtoslice[name] = slice(int(start), int(stop))
+
+ return spec, dimtoslice
+
+
+def to_seconds_from_epoch(date, timeunits=None, start_day=None, timeoffset=None):
+ try:
+ date = num2date(date, units=timeunits)
+ except ValueError:
+ assert isinstance(start_day, datetime.date), "start_day is not a datetime.date object"
+ the_datetime = datetime.datetime.combine(start_day, datetime.datetime.min.time())
+ date = the_datetime + datetime.timedelta(seconds=date)
+
+ if isinstance(date, datetime.datetime):
+ date = timezone('UTC').localize(date)
+ else:
+ date = timezone('UTC').localize(datetime.datetime.strptime(str(date), '%Y-%m-%d %H:%M:%S'))
+
+ if timeoffset is not None:
+ return int((date - EPOCH).total_seconds()) + timeoffset
+ else:
+ return int((date - EPOCH).total_seconds())
+
+
+def get_ordered_slices(ds, variable, dimension_to_slice):
+ dimensions_for_variable = [str(dimension) for dimension in ds[variable].dimensions]
+ ordered_slices = OrderedDict()
+ for dimension in dimensions_for_variable:
+ ordered_slices[dimension] = dimension_to_slice[dimension]
+ return ordered_slices
+
+
+def new_nexus_tile(file_path, section_spec):
+ nexus_tile = nexusproto.NexusTile()
+ tile_summary = nexusproto.TileSummary()
+ tile_summary.granule = file_path.split(sep)[-1]
+ tile_summary.section_spec = section_spec
+ nexus_tile.summary.CopyFrom(tile_summary)
+ return nexus_tile
+
+
+class TileReadingProcessor(Processor):
+ def __init__(self, variable_to_read, latitude, longitude, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # Required properties for all reader types
+ self.variable_to_read = variable_to_read
+ self.latitude = latitude
+ self.longitude = longitude
+
+ # Common optional properties
+ self.temp_dir = self.environ['TEMP_DIR']
+ self.metadata = self.environ['META']
+ self.start_of_day = self.environ['GLBLATTR_DAY']
+ self.start_of_day_pattern = self.environ['GLBLATTR_DAY_FORMAT']
+ self.time_offset = int(self.environ['TIME_OFFSET']) if self.environ['TIME_OFFSET'] is not None else None
+
+ def process(self, input_data):
+ tile_specifications, file_path = parse_input(input_data, self.temp_dir)
+
+ for data in self.read_data(tile_specifications, file_path):
+ yield data
+
+ # If temp dir is defined, delete the temporary file
+ if self.temp_dir is not None:
+ remove(file_path)
+
+ def read_data(self, tile_specifications, file_path):
+ raise NotImplementedError
+
+
+class GridReadingProcessor(TileReadingProcessor):
+ def read_data(self, tile_specifications, file_path):
+ # Time is optional for Grid data
+ time = self.environ['TIME']
+
+ with Dataset(file_path) as ds:
+ for section_spec, dimtoslice in tile_specifications:
+ tile = nexusproto.GridTile()
+
+ tile.latitude.CopyFrom(
+ to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[self.latitude]], numpy.NaN)))
+
+ tile.longitude.CopyFrom(
+ to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[self.longitude]], numpy.NaN)))
+
+ # Before we read the data we need to make sure the dimensions are in the proper order so we don't have any
+ # indexing issues
+ ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice)
+ # Read data using the ordered slices, replacing masked values with NaN
+ data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN)
+
+ tile.variable_data.CopyFrom(to_shaped_array(data_array))
+
+ if self.metadata is not None:
+ tile.meta_data.add().CopyFrom(
+ to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())]))
+
+ if time is not None:
+ timevar = ds[time]
+ # Note assumption is that index of time is start value in dimtoslice
+ tile.time = to_seconds_from_epoch(timevar[dimtoslice[time].start],
+ timeunits=timevar.getncattr('units'),
+ timeoffset=self.time_offset)
+
+ nexus_tile = new_nexus_tile(file_path, section_spec)
+ nexus_tile.tile.grid_tile.CopyFrom(tile)
+
+ yield nexus_tile
+
+
+class SwathReadingProcessor(TileReadingProcessor):
+ def __init__(self, variable_to_read, latitude, longitude, time, **kwargs):
+ super().__init__(variable_to_read, latitude, longitude, **kwargs)
+
+ # Time is required for swath data
+ self.time = time
+
+ def read_data(self, tile_specifications, file_path):
+ with Dataset(file_path) as ds:
+ for section_spec, dimtoslice in tile_specifications:
+ tile = nexusproto.SwathTile()
+ # Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable
+ ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice)
+ tile.latitude.CopyFrom(
+ to_shaped_array(numpy.ma.filled(ds[self.latitude][tuple(ordered_slices.values())], numpy.NaN)))
+
+ tile.longitude.CopyFrom(
+ to_shaped_array(numpy.ma.filled(ds[self.longitude][tuple(ordered_slices.values())], numpy.NaN)))
+
+ timetile = ds[self.time][
+ tuple([ordered_slices[time_dim] for time_dim in ds[self.time].dimensions])].astype(
+ 'float64',
+ casting='same_kind',
+ copy=False)
+ timeunits = ds[self.time].getncattr('units')
+ try:
+ start_of_day_date = datetime.datetime.strptime(ds.getncattr(self.start_of_day),
+ self.start_of_day_pattern)
+ except Exception:
+ start_of_day_date = None
+
+ for index in numpy.ndindex(timetile.shape):
+ timetile[index] = to_seconds_from_epoch(timetile[index].item(), timeunits=timeunits,
+ start_day=start_of_day_date, timeoffset=self.time_offset)
+
+ tile.time.CopyFrom(to_shaped_array(timetile))
+
+ # Read the data converting masked values to NaN
+ data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN)
+ tile.variable_data.CopyFrom(to_shaped_array(data_array))
+
+ if self.metadata is not None:
+ tile.meta_data.add().CopyFrom(
+ to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())]))
+
+ nexus_tile = new_nexus_tile(file_path, section_spec)
+ nexus_tile.tile.swath_tile.CopyFrom(tile)
+
+ yield nexus_tile
+
+
+class TimeSeriesReadingProcessor(TileReadingProcessor):
+ def __init__(self, variable_to_read, latitude, longitude, time, **kwargs):
+ super().__init__(variable_to_read, latitude, longitude, **kwargs)
+
+ # Time is required for swath data
+ self.time = time
+
+ def read_data(self, tile_specifications, file_path):
+ with Dataset(file_path) as ds:
+ for section_spec, dimtoslice in tile_specifications:
+ tile = nexusproto.TimeSeriesTile()
+
+ instance_dimension = next(
+ iter([dim for dim in ds[self.variable_to_read].dimensions if dim != self.time]))
+
+ tile.latitude.CopyFrom(
+ to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[instance_dimension]], numpy.NaN)))
+
+ tile.longitude.CopyFrom(
+ to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[instance_dimension]], numpy.NaN)))
+
+ # Before we read the data we need to make sure the dimensions are in the proper order so we don't
+ # have any indexing issues
+ ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice)
+ # Read data using the ordered slices, replacing masked values with NaN
+ data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN)
+
+ tile.variable_data.CopyFrom(to_shaped_array(data_array))
+
+ if self.metadata is not None:
+ tile.meta_data.add().CopyFrom(
+ to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())]))
+
+ timevar = ds[self.time]
+ # Note assumption is that index of time is start value in dimtoslice
+ tile.time = to_seconds_from_epoch(timevar[dimtoslice[self.time].start],
+ timeunits=timevar.getncattr('units'),
+ timeoffset=self.time_offset)
+
+ nexus_tile = new_nexus_tile(file_path, section_spec)
+ nexus_tile.tile.time_series_tile.CopyFrom(tile)
+
+ yield nexus_tile
diff --git a/ningesterpy/processors/tilesummarizingprocessor.py b/ningesterpy/processors/tilesummarizingprocessor.py
new file mode 100644
index 0000000..970a251
--- /dev/null
+++ b/ningesterpy/processors/tilesummarizingprocessor.py
@@ -0,0 +1,93 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import nexusproto.NexusContent_pb2 as nexusproto
+import numpy
+from nexusproto.serialization import from_shaped_array
+
+from processors import NexusTileProcessor
+
+
+class NoTimeException(Exception):
+ pass
+
+
+def find_time_min_max(tile_data):
+ # Only try to grab min/max time if it exists as a ShapedArray
+ if tile_data.HasField("time") and isinstance(tile_data.time, nexusproto.ShapedArray):
+ time_data = from_shaped_array(tile_data.time)
+ min_time = int(numpy.nanmin(time_data).item())
+ max_time = int(numpy.nanmax(time_data).item())
+
+ return min_time, max_time
+ elif tile_data.HasField("time") and isinstance(tile_data.time, int):
+ return tile_data.time, tile_data.time
+
+ raise NoTimeException
+
+
+class TileSummarizingProcessor(NexusTileProcessor):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.stored_var_name = self.environ['STORED_VAR_NAME']
+
+ def process_nexus_tile(self, nexus_tile):
+ the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
+
+ the_tile_data = getattr(nexus_tile.tile, the_tile_type)
+
+ latitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.latitude))
+ longitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.longitude))
+
+ data = from_shaped_array(the_tile_data.variable_data)
+
+ if nexus_tile.HasField("summary"):
+ tilesummary = nexus_tile.summary
+ else:
+ tilesummary = nexusproto.TileSummary()
+
+ tilesummary.bbox.lat_min = numpy.nanmin(latitudes).item()
+ tilesummary.bbox.lat_max = numpy.nanmax(latitudes).item()
+ tilesummary.bbox.lon_min = numpy.nanmin(longitudes).item()
+ tilesummary.bbox.lon_max = numpy.nanmax(longitudes).item()
+
+ tilesummary.stats.min = numpy.nanmin(data).item()
+ tilesummary.stats.max = numpy.nanmax(data).item()
+
+ # In order to accurately calculate the average we need to weight the data based on the cosine of its latitude
+ # This is handled slightly differently for swath vs. grid data
+ if the_tile_type == 'swath_tile':
+ # For Swath tiles, len(data) == len(latitudes) == len(longitudes). So we can simply weight each element in the
+ # data array
+ tilesummary.stats.mean = numpy.ma.average(numpy.ma.masked_invalid(data),
+ weights=numpy.cos(numpy.radians(latitudes))).item()
+ elif the_tile_type == 'grid_tile':
+ # Grid tiles need to repeat the weight for every longitude
+ # TODO This assumes data axis' are ordered as latitude x longitude
+ tilesummary.stats.mean = numpy.ma.average(numpy.ma.masked_invalid(data).flatten(),
+ weights=numpy.cos(
+ numpy.radians(
+ numpy.repeat(latitudes, len(longitudes))))).item()
+ else:
+ # Default to simple average with no weighting
+ tilesummary.stats.mean = numpy.nanmean(data).item()
+
+ tilesummary.stats.count = data.size - numpy.count_nonzero(numpy.isnan(data))
+
+ try:
+ min_time, max_time = find_time_min_max(the_tile_data)
+ tilesummary.stats.min_time = min_time
+ tilesummary.stats.max_time = max_time
+ except NoTimeException:
+ pass
+
+ try:
+ tilesummary.data_var_name = self.stored_var_name
+ except TypeError:
+ pass
+
+ nexus_tile.summary.CopyFrom(tilesummary)
+ yield nexus_tile
diff --git a/ningesterpy/processors/winddirspeedtouv.py b/ningesterpy/processors/winddirspeedtouv.py
new file mode 100644
index 0000000..9a4445d
--- /dev/null
+++ b/ningesterpy/processors/winddirspeedtouv.py
@@ -0,0 +1,90 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+from math import cos
+from math import radians
+from math import sin
+
+import numpy
+from nexusproto.serialization import from_shaped_array, to_shaped_array
+
+from processors import NexusTileProcessor
+
+
+def enum(**enums):
+ return type('Enum', (), enums)
+
+
+U_OR_V_ENUM = enum(U='u', V='v')
+
+
+def calculate_u_component_value(direction, speed):
+ if direction is numpy.ma.masked or speed is numpy.ma.masked:
+ return numpy.ma.masked
+
+ return speed * sin(direction)
+
+
+def calculate_v_component_value(direction, speed):
+ if direction is numpy.ma.masked or speed is numpy.ma.masked:
+ return numpy.ma.masked
+
+ return speed * cos(direction)
+
+
+class WindDirSpeedToUV(NexusTileProcessor):
+
+ def __init__(self, u_or_v, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.u_or_v = u_or_v.lower()
+
+ def process_nexus_tile(self, nexus_tile):
+ the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
+
+ the_tile_data = getattr(nexus_tile.tile, the_tile_type)
+
+ wind_speed = from_shaped_array(the_tile_data.variable_data)
+
+ wind_dir = from_shaped_array(
+ next(meta for meta in the_tile_data.meta_data if meta.name == 'wind_dir').meta_data)
+
+ assert wind_speed.shape == wind_dir.shape
+
+ wind_u_component = numpy.ma.empty(wind_speed.shape, dtype=float)
+ wind_v_component = numpy.ma.empty(wind_speed.shape, dtype=float)
+ wind_speed_iter = numpy.nditer(wind_speed, flags=['multi_index'])
+ while not wind_speed_iter.finished:
+ speed = wind_speed_iter[0]
+ current_index = wind_speed_iter.multi_index
+ direction = wind_dir[current_index]
+
+ # Convert degrees to radians
+ direction = radians(direction)
+
+ # Calculate component values
+ wind_u_component[current_index] = calculate_u_component_value(direction, speed)
+ wind_v_component[current_index] = calculate_v_component_value(direction, speed)
+
+ wind_speed_iter.iternext()
+
+ # Stick the original data into the meta data
+ wind_speed_meta = the_tile_data.meta_data.add()
+ wind_speed_meta.name = 'wind_speed'
+ wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed))
+
+ # The u_or_v variable specifies which component variable is the 'data variable' for this tile
+ # Replace data with the appropriate component value and put the other component in metadata
+ if self.u_or_v == U_OR_V_ENUM.U:
+ the_tile_data.variable_data.CopyFrom(to_shaped_array(wind_u_component))
+ wind_component_meta = the_tile_data.meta_data.add()
+ wind_component_meta.name = 'wind_v'
+ wind_component_meta.meta_data.CopyFrom(to_shaped_array(wind_v_component))
+ elif self.u_or_v == U_OR_V_ENUM.V:
+ the_tile_data.variable_data.CopyFrom(to_shaped_array(wind_v_component))
+ wind_component_meta = the_tile_data.meta_data.add()
+ wind_component_meta.name = 'wind_u'
+ wind_component_meta.meta_data.CopyFrom(to_shaped_array(wind_u_component))
+
+ yield nexus_tile
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..22d2912
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,12 @@
+werkzeug=0.12.2
+flask=0.12.2
+flask-accept==0.0.4
+nco==4.7.1
+netCDF4==1.3.1
+nexusproto==0.41
+numpy==1.12.1
+protobuf==3.2.0
+pytz==2017.2
+PyYAML==3.12
+scipy==0.18.1
+six==1.10.0
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..f53f9ea
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,29 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+from setuptools import setup, find_packages
+
+__version__ = '0.1'
+
+setup(
+ name="ningesterpy",
+ version=__version__,
+ url="https://github.jpl.nasa.gov/thuang/nexus",
+
+ author="Team Nexus",
+
+ description="Python modules that can be used for NEXUS ingest.",
+ # long_description=open('README.md').read(),
+
+ packages=find_packages(),
+ test_suite="tests",
+ platforms='any',
+
+ classifiers=[
+ 'Development Status :: 1 - Pre-Alpha',
+ 'Intended Audience :: Developers',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python :: 3.5',
+ ]
+)
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..bd9282c
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1,4 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
\ No newline at end of file
diff --git a/tests/callncpdq_test.py b/tests/callncpdq_test.py
new file mode 100644
index 0000000..388a70b
--- /dev/null
+++ b/tests/callncpdq_test.py
@@ -0,0 +1,53 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import subprocess
+import unittest
+from os import path, remove
+
+from netCDF4 import Dataset
+
+from processors import callncpdq
+
+
+class TestMeasuresData(unittest.TestCase):
+
+ @unittest.skipIf(int(subprocess.call(["ncpdq", "-r"])) not in {0, 1}, "requires ncpdq")
+ def test_permute_all_variables(self):
+ dimension_order = ['Time', 'Latitude', 'Longitude']
+
+ the_module = callncpdq.CallNcpdq(dimension_order)
+
+ expected_dimensions = dimension_order
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc')
+
+ output_path = list(the_module.process(test_file))[0]
+
+ with Dataset(output_path) as ds:
+ sla_var = ds['SLA']
+ actual_dimensions = [str(dim) for dim in sla_var.dimensions]
+
+ remove(output_path)
+ self.assertEqual(expected_dimensions, actual_dimensions)
+
+ @unittest.skipIf(int(subprocess.call(["ncpdq", "-r"])) not in {0, 1}, "requires ncpdq")
+ def test_permute_one_variable(self):
+ dimension_order = ['Time', 'Latitude', 'Longitude']
+ permute_var = 'SLA'
+
+ the_module = callncpdq.CallNcpdq(dimension_order, environ={"PERMUTE_VARIABLE": permute_var})
+
+ expected_dimensions = dimension_order
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc')
+
+ output_path = list(the_module.process(test_file))[0]
+
+ with Dataset(output_path) as ds:
+ sla_var = ds[permute_var]
+ actual_dimensions = [str(dim) for dim in sla_var.dimensions]
+
+ remove(output_path)
+ self.assertEqual(expected_dimensions, actual_dimensions)
diff --git a/tests/computespeeddirfromuv_test.py b/tests/computespeeddirfromuv_test.py
new file mode 100644
index 0000000..d880008
--- /dev/null
+++ b/tests/computespeeddirfromuv_test.py
@@ -0,0 +1,114 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import unittest
+from os import path
+
+import numpy as np
+from nexusproto.serialization import from_shaped_array
+
+import processors
+from processors.computespeeddirfromuv import calculate_speed_direction
+
+
+class TestConversion(unittest.TestCase):
+ def test_dir_from_north(self):
+ # Negative v means wind is wind blowing to the South
+ u = 0
+ v = -1
+
+ speed, direction = calculate_speed_direction(u, v)
+
+ # Degrees are where the wind is blowing from (relative to true North)
+ self.assertEqual(1, speed)
+ # Wind from North (0 degrees)
+ self.assertEqual(0, direction)
+
+ def test_dir_from_east(self):
+ # Negative u means wind is blowing to the West
+ u = -1
+ v = 0
+
+ speed, direction = calculate_speed_direction(u, v)
+
+ # Degrees are where the wind is blowing from (relative to true North)
+ self.assertEqual(1, speed)
+ # Wind from East (90 degrees)
+ self.assertEqual(90, direction)
+
+ def test_dir_from_south(self):
+ # Positive v means wind is blowing to the North
+ u = 0
+ v = 1
+
+ speed, direction = calculate_speed_direction(u, v)
+
+ # Degrees are where the wind is blowing from (relative to true North)
+ self.assertEqual(1, speed)
+ # Wind from South (180 degrees)
+ self.assertEqual(180, direction)
+
+ def test_dir_from_west(self):
+ # Positive u means wind is blowing to the East
+ u = 1
+ v = 0
+
+ speed, direction = calculate_speed_direction(u, v)
+
+ # Degrees are where the wind is blowing from (relative to true North)
+ self.assertEqual(1, speed)
+ # Wind from West (270 degrees)
+ self.assertEqual(270, direction)
+
+ def test_speed(self):
+ # Speed is simply sqrt(u^2 + v^2)
+ u = 2
+ v = 2
+
+ speed, direction = calculate_speed_direction(u, v)
+
+ self.assertAlmostEqual(2.8284271, speed)
+ # Wind should be from the southwest
+ self.assertTrue(180 < direction < 270)
+
+
+class TestCcmpData(unittest.TestCase):
+ def setUp(self):
+ self.module = processors.ComputeSpeedDirFromUV('uwnd', 'vwnd')
+
+ def test_speed_dir_computation(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ccmp_nonempty_nexustile.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ results = list(self.module.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile = results[0]
+
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+
+ # Check data
+ tile_data = np.ma.masked_invalid(from_shaped_array(nexus_tile.tile.grid_tile.variable_data))
+ self.assertEqual(3306, np.ma.count(tile_data))
+
+ # Check meta data
+ meta_list = nexus_tile.tile.grid_tile.meta_data
+ self.assertEqual(3, len(meta_list))
+ wind_dir = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_dir')
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape)
+ self.assertIsNotNone(wind_dir)
+ wind_speed = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_speed')
+ self.assertIsNotNone(wind_speed)
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_speed.meta_data)).shape)
+ wind_v = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'vwnd')
+ self.assertIsNotNone(wind_v)
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_v.meta_data)).shape)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/convert_iceshelf.py b/tests/convert_iceshelf.py
new file mode 100644
index 0000000..cf556dd
--- /dev/null
+++ b/tests/convert_iceshelf.py
@@ -0,0 +1,78 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+
+import datetime
+
+import netCDF4
+import numpy as np
+
+time_units = 'days since 1981-01-01 00:00:00'
+time_units_attr_name = 'units'
+time_calendar = 'standard'
+time_calendar_attr_name = 'calendar'
+time_var_name = 'time'
+lat_var_name = 'lat'
+lon_var_name = 'lon'
+
+phony_dimension_map = {
+ 'phony_dim_0': time_var_name,
+ 'phony_dim_1': lat_var_name,
+ 'phony_dim_2': lon_var_name
+}
+
+
+def float_to_datetime(time_float):
+ """
+ Convert time_float (a float in the form of 4-digit_year.fractional year eg. 1994.0384) to a datetime object
+ """
+ year = int(time_float)
+ remainder = time_float - year
+ beginning_of_year = datetime.datetime(year, 1, 1)
+ end_of_year = datetime.datetime(year + 1, 1, 1)
+ seconds = remainder * (end_of_year - beginning_of_year).total_seconds()
+ return beginning_of_year + datetime.timedelta(seconds=seconds)
+
+
+with netCDF4.Dataset('/Users/greguska/data/ice_shelf_dh/ice_shelf_dh_v1.h5') as input_ds:
+ latitudes_1d = input_ds[lat_var_name][:, 0]
+ longitudes_1d = input_ds[lon_var_name][0, :]
+
+ times_as_int = np.fromiter(
+ (netCDF4.date2num(float_to_datetime(time), time_units, calendar=time_calendar) for time in
+ input_ds[time_var_name][:]),
+ dtype=str(input_ds[time_var_name].dtype), count=len(input_ds[time_var_name][:]))
+
+ with netCDF4.Dataset('/Users/greguska/data/ice_shelf_dh/ice_shelf_dh_v1.nc', mode='w') as output_ds:
+ output_ds.setncatts({att: input_ds.getncattr(att) for att in input_ds.ncattrs()})
+
+ for in_dimension_name in input_ds.dimensions:
+ out_dimension_name = phony_dimension_map[in_dimension_name]
+ output_ds.createDimension(out_dimension_name, len(input_ds.dimensions[in_dimension_name]))
+
+ for in_variable_name, in_variable in input_ds.variables.iteritems():
+
+ if in_variable_name in [time_var_name, lat_var_name, lon_var_name]:
+ output_ds.createVariable(in_variable_name, in_variable.dtype, (in_variable_name,))
+ else:
+ output_ds.createVariable(in_variable_name, in_variable.dtype,
+ tuple([phony_dimension_map[dim] for dim in in_variable.dimensions]))
+
+ for attr_name in in_variable.ncattrs():
+ attr_value = in_variable.getncattr(attr_name)
+ if isinstance(attr_value, list):
+ output_ds[in_variable_name].setncattr_string(attr_name, attr_value)
+ else:
+ output_ds[in_variable_name].setncattr(attr_name, attr_value)
+
+ if in_variable_name == lat_var_name:
+ output_ds[in_variable_name][:] = latitudes_1d
+ elif in_variable_name == lon_var_name:
+ output_ds[in_variable_name][:] = longitudes_1d
+ elif in_variable_name == time_var_name:
+ output_ds[in_variable_name].setncattr(time_calendar_attr_name, time_calendar)
+ output_ds[in_variable_name].setncattr(time_units_attr_name, time_units)
+ output_ds[in_variable_name][:] = times_as_int
+ else:
+ output_ds[in_variable_name][:] = input_ds[in_variable_name][:]
diff --git a/tests/datafiles/empty_mur.nc4 b/tests/datafiles/empty_mur.nc4
new file mode 100644
index 0000000..f65c808
Binary files /dev/null and b/tests/datafiles/empty_mur.nc4 differ
diff --git a/tests/datafiles/not_empty_ascatb.nc4 b/tests/datafiles/not_empty_ascatb.nc4
new file mode 100644
index 0000000..d8ef90b
Binary files /dev/null and b/tests/datafiles/not_empty_ascatb.nc4 differ
diff --git a/tests/datafiles/not_empty_avhrr.nc4 b/tests/datafiles/not_empty_avhrr.nc4
new file mode 100644
index 0000000..af24071
Binary files /dev/null and b/tests/datafiles/not_empty_avhrr.nc4 differ
diff --git a/tests/datafiles/not_empty_ccmp.nc b/tests/datafiles/not_empty_ccmp.nc
new file mode 100644
index 0000000..b7b491d
Binary files /dev/null and b/tests/datafiles/not_empty_ccmp.nc differ
diff --git a/tests/datafiles/not_empty_measures_alt.nc b/tests/datafiles/not_empty_measures_alt.nc
new file mode 100644
index 0000000..fd03c6d
Binary files /dev/null and b/tests/datafiles/not_empty_measures_alt.nc differ
diff --git a/tests/datafiles/not_empty_mur.nc4 b/tests/datafiles/not_empty_mur.nc4
new file mode 100644
index 0000000..09d31fd
Binary files /dev/null and b/tests/datafiles/not_empty_mur.nc4 differ
diff --git a/tests/datafiles/not_empty_smap.h5 b/tests/datafiles/not_empty_smap.h5
new file mode 100644
index 0000000..956cbc5
Binary files /dev/null and b/tests/datafiles/not_empty_smap.h5 differ
diff --git a/tests/datafiles/not_empty_wswm.nc b/tests/datafiles/not_empty_wswm.nc
new file mode 100644
index 0000000..772bbcb
Binary files /dev/null and b/tests/datafiles/not_empty_wswm.nc differ
diff --git a/tests/datafiles/partial_empty_mur.nc4 b/tests/datafiles/partial_empty_mur.nc4
new file mode 100644
index 0000000..d95a5dc
Binary files /dev/null and b/tests/datafiles/partial_empty_mur.nc4 differ
diff --git a/tests/dumped_nexustiles/ascat_longitude_more_than_180.bin b/tests/dumped_nexustiles/ascat_longitude_more_than_180.bin
new file mode 100644
index 0000000..75e3374
Binary files /dev/null and b/tests/dumped_nexustiles/ascat_longitude_more_than_180.bin differ
diff --git a/tests/dumped_nexustiles/ascatb_nonempty_nexustile.bin b/tests/dumped_nexustiles/ascatb_nonempty_nexustile.bin
new file mode 100644
index 0000000..9ace5ea
Binary files /dev/null and b/tests/dumped_nexustiles/ascatb_nonempty_nexustile.bin differ
diff --git a/tests/dumped_nexustiles/avhrr_nonempty_nexustile.bin b/tests/dumped_nexustiles/avhrr_nonempty_nexustile.bin
new file mode 100644
index 0000000..c21398d
Binary files /dev/null and b/tests/dumped_nexustiles/avhrr_nonempty_nexustile.bin differ
diff --git a/tests/dumped_nexustiles/ccmp_nonempty_nexustile.bin b/tests/dumped_nexustiles/ccmp_nonempty_nexustile.bin
new file mode 100644
index 0000000..1c1fd7f
Binary files /dev/null and b/tests/dumped_nexustiles/ccmp_nonempty_nexustile.bin differ
diff --git a/tests/dumped_nexustiles/smap_nonempty_nexustile.bin b/tests/dumped_nexustiles/smap_nonempty_nexustile.bin
new file mode 100644
index 0000000..d19301a
Binary files /dev/null and b/tests/dumped_nexustiles/smap_nonempty_nexustile.bin differ
diff --git a/tests/hd5splitter.py b/tests/hd5splitter.py
new file mode 100644
index 0000000..c2a6655
--- /dev/null
+++ b/tests/hd5splitter.py
@@ -0,0 +1,123 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+
+
+def hd5_copy(source, dest):
+ for key in source.keys():
+ source.copy('/' + key, dest['/'], name=key)
+
+ print(key)
+
+ if str(key) == 'time':
+ dest[key + '_c'] = dest[key][0:4]
+ elif str(key) == 'longitude':
+ dest[key + '_c'] = dest[key][0:87]
+ elif str(key) == 'latitude':
+ dest[key + '_c'] = dest[key][0:38]
+ else:
+ dest[key + '_c'] = dest[key][0:4, 0:38, 0:87]
+
+ # Useful for swath data:
+ # if dest[key].ndim == 2:
+ # dest[key + '_c'] = dest[key][0:76, 181:183]
+ # elif dest[key].ndim == 3:
+ # dest[key + '_c'] = dest[key][0:76, 181:183, :]
+ # elif dest[key].ndim == 1:
+ # dest[key + '_c'] = dest[key][181:183]
+
+ for att in dest[key + '_c'].attrs:
+ try:
+ dest[key + '_c'].attrs.modify(dest[key].attrs.get(att, default=""))
+ except IOError:
+ print("error " + att)
+ pass
+ dest[key + '_c'].attrs.update(dest[key].attrs)
+ del dest[key]
+ dest[key] = dest[key + '_c']
+ del dest[key + '_c']
+
+ print(dest[key])
+
+ for att in dest.attrs:
+ try:
+ dest.attrs.modify(source.attrs.get(att, default=""))
+ except IOError:
+ print("error " + att)
+ pass
+
+ # dest.attrs.update(source.attrs)
+
+ dest.flush()
+
+
+def netcdf_subset(source, dest):
+ dtime = dest.createDimension(dimname=TIME, size=TIME_SLICE.stop - TIME_SLICE.start)
+ # dlat = dest.createDimension(dimname=LATITUDE, size=LATITUDE_SLICE.stop - LATITUDE_SLICE.start)
+ # dlon = dest.createDimension(dimname=LONGITUDE, size=LONGITUDE_SLICE.stop - LONGITUDE_SLICE.start)
+ drivid = dest.createDimension(dimname='rivid', size=LONGITUDE_SLICE.stop - LONGITUDE_SLICE.start)
+
+ dest.setncatts(source.__dict__)
+
+ for variable in [v for v in source.variables if v in ['Qout', TIME, LONGITUDE, LATITUDE]]:
+ variable = source[variable]
+
+ if variable.name == TIME:
+ dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype, dimensions=(dtime.name,))
+ dest[variable.name].setncatts(variable.__dict__)
+ dvar[:] = variable[TIME_SLICE]
+ elif variable.name == LONGITUDE:
+ dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype, dimensions=(drivid.name,))
+ dest[variable.name].setncatts(variable.__dict__)
+ dvar[:] = variable[LONGITUDE_SLICE]
+ elif variable.name == LATITUDE:
+ dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype, dimensions=(drivid.name,))
+ dest[variable.name].setncatts(variable.__dict__)
+ dvar[:] = variable[LATITUDE_SLICE]
+ else:
+ dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype,
+ dimensions=(dtime.name, drivid.name))
+ dest[variable.name].setncatts(variable.__dict__)
+ dvar[:] = variable[TIME_SLICE, LONGITUDE_SLICE]
+
+ dest.sync()
+ dest.close()
+
+
+from netCDF4 import Dataset
+
+LATITUDE = 'lat'
+LATITUDE_SLICE = slice(0, 1000)
+LONGITUDE = 'lon'
+LONGITUDE_SLICE = slice(0, 1000)
+TIME = 'time'
+TIME_SLICE = slice(0, 1)
+
+hinput = Dataset(
+ '/Users/greguska/data/swot_example/latest/Qout_WSWM_729days_p0_dtR900s_n1_preonly_20160416.nc',
+ 'r')
+houtput = Dataset(
+ '/Users/greguska/data/swot_example/latest/Qout_WSWM_729days_p0_dtR900s_n1_preonly_20160416.split.nc',
+ mode='w')
+
+netcdf_subset(hinput, houtput)
+
+# # from h5py import File, Dataset
+# hinput = File(
+# '/Users/greguska/githubprojects/nexus/nexus-ingest/developer-box/data/ccmp/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.nc',
+# 'r')
+# houput = File(
+# '/Users/greguska/githubprojects/nexus/nexus-ingest/developer-box/data/ccmp/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.split.nc',
+# 'w')
+
+# hd5_copy(hinput, houput)
+
+# print hinput['/']
+# print houtput['/']
+
+# print [attr for attr in hinput.attrs]
+# print [attr for attr in houtput.attrs]
+
+# hinput.close()
+# houtput.close()
diff --git a/tests/kelvintocelsius_test.py b/tests/kelvintocelsius_test.py
new file mode 100644
index 0000000..182abc7
--- /dev/null
+++ b/tests/kelvintocelsius_test.py
@@ -0,0 +1,40 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import unittest
+from os import path
+
+import nexusproto.NexusContent_pb2 as nexusproto
+import numpy as np
+from nexusproto.serialization import from_shaped_array
+
+import processors
+
+
+class TestAvhrrData(unittest.TestCase):
+ def setUp(self):
+ self.module = processors.KelvinToCelsius()
+
+ def test_kelvin_to_celsius(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'avhrr_nonempty_nexustile.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str)
+ sst_before = from_shaped_array(nexus_tile_before.tile.grid_tile.variable_data)
+
+ results = list(self.module.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile_after = results[0]
+ sst_after = from_shaped_array(nexus_tile_after.tile.grid_tile.variable_data)
+
+ # Just spot check a couple of values
+ expected_sst = np.subtract(sst_before[0][0][0], np.float32(273.15))
+ self.assertEqual(expected_sst, sst_after[0][0][0])
+
+ expected_sst = np.subtract(sst_before[0][9][9], np.float32(273.15))
+ self.assertEqual(expected_sst, sst_after[0][9][9])
diff --git a/tests/processorchain_test.py b/tests/processorchain_test.py
new file mode 100644
index 0000000..3563a1b
--- /dev/null
+++ b/tests/processorchain_test.py
@@ -0,0 +1,92 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import unittest
+from os import path
+
+from processors.processorchain import ProcessorChain
+
+
+class TestRunChainMethod(unittest.TestCase):
+ def test_run_chain_read_filter_all(self):
+ processor_list = [
+ {'name': 'GridReadingProcessor',
+ 'config': {'latitude': 'lat',
+ 'longitude': 'lon',
+ 'time': 'time',
+ 'variable_to_read': 'analysed_sst'}},
+ {'name': 'EmptyTileFilter'}
+ ]
+ processorchain = ProcessorChain(processor_list)
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4')
+
+ gen = processorchain.process("time:0:1,lat:0:1,lon:0:1;time:0:1,lat:1:2,lon:0:1;file://%s" % test_file)
+ for message in gen:
+ self.fail("Should not produce any messages. Message: %s" % message)
+
+ def test_run_chain_read_filter_none(self):
+ processor_list = [
+ {'name': 'GridReadingProcessor',
+ 'config': {'latitude': 'lat',
+ 'longitude': 'lon',
+ 'time': 'time',
+ 'variable_to_read': 'analysed_sst'}},
+ {'name': 'EmptyTileFilter'}
+ ]
+ processorchain = ProcessorChain(processor_list)
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_mur.nc4')
+
+ results = list(
+ processorchain.process("time:0:1,lat:0:1,lon:0:1;time:0:1,lat:1:2,lon:0:1;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ def test_run_chain_read_filter_kelvin_summarize(self):
+ processor_list = [
+ {'name': 'GridReadingProcessor',
+ 'config': {'latitude': 'lat',
+ 'longitude': 'lon',
+ 'time': 'time',
+ 'variable_to_read': 'analysed_sst'}},
+ {'name': 'EmptyTileFilter'},
+ {'name': 'KelvinToCelsius'},
+ {'name': 'TileSummarizingProcessor'}
+ ]
+ processorchain = ProcessorChain(processor_list)
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_mur.nc4')
+
+ results = list(
+ processorchain.process("time:0:1,lat:0:1,lon:0:1;time:0:1,lat:1:2,lon:0:1;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ def test_run_chain_partial_empty(self):
+ processor_list = [
+ {'name': 'GridReadingProcessor',
+ 'config': {'latitude': 'lat',
+ 'longitude': 'lon',
+ 'time': 'time',
+ 'variable_to_read': 'analysed_sst'}},
+ {'name': 'EmptyTileFilter'},
+ {'name': 'KelvinToCelsius'},
+ {'name': 'TileSummarizingProcessor'}
+ ]
+ processorchain = ProcessorChain(processor_list)
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'partial_empty_mur.nc4')
+
+ results = list(
+ processorchain.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:489:499,lon:0:10;file://%s" % test_file))
+
+ self.assertEqual(1, len(results))
+ tile = results[0]
+
+ self.assertTrue(tile.summary.HasField('bbox'), "bbox is missing")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/regrid1x1_test.py b/tests/regrid1x1_test.py
new file mode 100644
index 0000000..b080989
--- /dev/null
+++ b/tests/regrid1x1_test.py
@@ -0,0 +1,79 @@
+"""
+Copyright (c) 2017 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import os
+import unittest
+
+import processors
+
+
+def delete_file_if_exists(filename):
+ try:
+ os.remove(filename)
+ except OSError:
+ pass
+
+
+class TestSSHData(unittest.TestCase):
+
+ def setUp(self):
+ self.test_file = os.path.join(os.path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc')
+ self.prefix = 'test-regrid'
+ self.expected_output_path = os.path.join(os.path.dirname(self.test_file),
+ self.prefix + os.path.basename(self.test_file))
+ delete_file_if_exists(self.expected_output_path)
+
+ def tearDown(self):
+ delete_file_if_exists(self.expected_output_path)
+
+ def test_ssh_grid(self):
+ regridder = processors.Regrid1x1('SLA', 'Latitude', 'Longitude', 'Time',
+ variable_valid_range='SLA:-100.0:100.0:SLA_ERR:-5000:5000',
+ filename_prefix=self.prefix)
+
+ results = list(regridder.process(self.test_file))
+
+ self.assertEqual(1, len(results))
+
+
+class TestGRACEData(unittest.TestCase):
+ def setUp(self):
+ self.test_file = '' # os.path.join(os.path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc')
+ self.prefix = 'test-regrid'
+ self.expected_output_path = os.path.join(os.path.dirname(self.test_file),
+ self.prefix + os.path.basename(self.test_file))
+ delete_file_if_exists(self.expected_output_path)
+
+ def tearDown(self):
+ delete_file_if_exists(self.expected_output_path)
+
+ @unittest.skip
+ def test_lwe_grid(self):
+ regridder = processors.Regrid1x1('lwe_thickness', 'lat', 'lon', 'tim',
+ filename_prefix=self.prefix)
+
+ results = list(regridder.process(self.test_file))
+
+ self.assertEqual(1, len(results))
+
+
+class TestIceShelfData(unittest.TestCase):
+ def setUp(self):
+ self.test_file = '' # os.path.join(os.path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc')
+ self.prefix = 'test-regrid'
+ self.expected_output_path = os.path.join(os.path.dirname(self.test_file),
+ self.prefix + os.path.basename(self.test_file))
+ delete_file_if_exists(self.expected_output_path)
+
+ def tearDown(self):
+ delete_file_if_exists(self.expected_output_path)
+
+ @unittest.skip
+ def test_height_raw(self):
+ regridder = processors.Regrid1x1('height_raw,height_filt,height_err', 'lat', 'lon', 'tim',
+ filename_prefix=self.prefix)
+
+ results = list(regridder.process(self.test_file))
+
+ self.assertEqual(1, len(results))
diff --git a/tests/subtract180longitude_test.py b/tests/subtract180longitude_test.py
new file mode 100644
index 0000000..5ad55c2
--- /dev/null
+++ b/tests/subtract180longitude_test.py
@@ -0,0 +1,57 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import unittest
+from os import path
+
+import nexusproto.NexusContent_pb2 as nexusproto
+import numpy as np
+from nexusproto.serialization import from_shaped_array
+
+import processors
+
+
+class TestAscatbUData(unittest.TestCase):
+
+ def test_subtraction_longitudes_less_than_180(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascatb_nonempty_nexustile.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str)
+ longitudes_before = from_shaped_array(nexus_tile_before.tile.swath_tile.longitude)
+
+ subtract = processors.Subtract180Longitude()
+
+ results = list(subtract.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile_after = results[0]
+ longitudes_after = from_shaped_array(nexus_tile_after.tile.swath_tile.longitude)
+
+ self.assertTrue(np.all(np.equal(longitudes_before, longitudes_after)))
+
+ def test_subtraction_longitudes_greater_than_180(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascat_longitude_more_than_180.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str)
+ longitudes_before = from_shaped_array(nexus_tile_before.tile.swath_tile.longitude)
+
+ subtract = processors.Subtract180Longitude()
+
+ results = list(subtract.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile_after = results[0]
+ longitudes_after = from_shaped_array(nexus_tile_after.tile.swath_tile.longitude)
+
+ self.assertTrue(np.all(np.not_equal(longitudes_before, longitudes_after)))
+ self.assertTrue(np.all(longitudes_after[longitudes_after < 0]))
+ self.assertAlmostEqual(-96.61, longitudes_after[0][26], places=2)
diff --git a/tests/tilereadingprocessor_test.py b/tests/tilereadingprocessor_test.py
new file mode 100644
index 0000000..2d69bbe
--- /dev/null
+++ b/tests/tilereadingprocessor_test.py
@@ -0,0 +1,330 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import unittest
+from os import path
+
+import numpy as np
+from nexusproto.serialization import from_shaped_array
+
+import processors
+
+
+class TestSummaryData(unittest.TestCase):
+ def setUp(self):
+
+ self.module = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time')
+
+ def test_summary_exists(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4')
+
+ results = list(
+ self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('summary'))
+
+ def test_section_spec_set(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4')
+
+ results = list(
+ self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ # Tests for the first tile
+ self.assertEqual('time:0:1,lat:0:10,lon:0:10', results[0].summary.section_spec)
+
+ # Tests for the second tile
+ self.assertEqual('time:0:1,lat:10:20,lon:0:10', results[1].summary.section_spec)
+
+ def test_granule_set(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4')
+
+ results = list(
+ self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ for nexus_tile in results:
+ self.assertEqual('empty_mur.nc4', nexus_tile.summary.granule)
+
+
+class TestReadMurData(unittest.TestCase):
+ def setUp(self):
+ self.module = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time')
+
+ def test_read_empty_mur(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4')
+
+ results = list(
+ self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+
+ tile = nexus_tile.tile.grid_tile
+ self.assertEqual(10, len(from_shaped_array(tile.latitude)))
+ self.assertEqual(10, len(from_shaped_array(tile.longitude)))
+
+ the_data = np.ma.masked_invalid(from_shaped_array(tile.variable_data))
+ self.assertEqual((1, 10, 10), the_data.shape)
+ self.assertEqual(0, np.ma.count(the_data))
+ self.assertTrue(tile.HasField('time'))
+
+ def test_read_not_empty_mur(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_mur.nc4')
+
+ results = list(
+ self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))
+ self.assertEqual((1, 10, 10), tile1_data.shape)
+ self.assertEqual(100, np.ma.count(tile1_data))
+
+ tile2_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.grid_tile.variable_data))
+ self.assertEqual((1, 10, 10), tile2_data.shape)
+ self.assertEqual(100, np.ma.count(tile2_data))
+
+ self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True), "Both tiles contain identical data")
+
+
+class TestReadAscatbData(unittest.TestCase):
+ def setUp(self):
+ self.module = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time')
+
+ # for data in read_swath_data(None,
+ # "NUMROWS:0:1,NUMCELLS:0:5;NUMROWS:1:2,NUMCELLS:0:5;file:///Users/greguska/data/ascat/ascat_20130314_004801_metopb_02520_eps_o_coa_2101_ovw.l2.nc"):
+ # import sys
+ # from struct import pack
+ # sys.stdout.write(pack("!L", len(data)) + data)
+
+ # VARIABLE=wind_speed,LATITUDE=lat,LONGITUDE=lon,TIME=time,META=wind_dir,READER=SWATHTILE,TEMP_DIR=/tmp
+ def test_read_not_empty_ascatb(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4')
+
+ swath_reader = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time')
+
+ results = list(
+ swath_reader.process("NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
+ self.assertEqual(0, len(nexus_tile.tile.swath_tile.meta_data))
+
+ tile = nexus_tile.tile.swath_tile
+ self.assertEqual(82, from_shaped_array(tile.latitude).size)
+ self.assertEqual(82, from_shaped_array(tile.longitude).size)
+
+ tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.variable_data))
+ self.assertEqual((1, 82), tile1_data.shape)
+ self.assertEqual(82, np.ma.count(tile1_data))
+
+ tile2_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.swath_tile.variable_data))
+ self.assertEqual((1, 82), tile2_data.shape)
+ self.assertEqual(82, np.ma.count(tile2_data))
+
+ self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True), "Both tiles contain identical data")
+
+ def test_read_not_empty_ascatb_meta(self):
+ # with open('./ascat_longitude_more_than_180.bin', 'w') as f:
+ # results = list(self.module.read_swath_data(None,
+ # "NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file:///Users/greguska/Downloads/ascat_longitude_more_than_180.nc4"))
+ # f.write(results[0])
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4')
+
+ swath_reader = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time', meta='wind_dir')
+
+ results = list(
+ swath_reader.process("NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
+ self.assertLess(0, len(nexus_tile.tile.swath_tile.meta_data))
+
+ self.assertEqual(1, len(results[0].tile.swath_tile.meta_data))
+ tile1_meta_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.meta_data[0].meta_data))
+ self.assertEqual((1, 82), tile1_meta_data.shape)
+ self.assertEqual(82, np.ma.count(tile1_meta_data))
+
+ self.assertEqual(1, len(results[1].tile.swath_tile.meta_data))
+ tile2_meta_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.swath_tile.meta_data[0].meta_data))
+ self.assertEqual((1, 82), tile2_meta_data.shape)
+ self.assertEqual(82, np.ma.count(tile2_meta_data))
+
+ self.assertFalse(np.allclose(tile1_meta_data, tile2_meta_data, equal_nan=True),
+ "Both tiles' meta contain identical data")
+
+
+class TestReadSmapData(unittest.TestCase):
+ def test_read_not_empty_smap(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_smap.h5')
+
+ swath_reader = processors.SwathReadingProcessor('smap_sss', 'lat', 'lon',
+ time='row_time',
+ glblattr_day='REV_START_TIME',
+ glblattr_day_format='%Y-%jT%H:%M:%S.%f')
+
+ results = list(swath_reader.process(
+ "phony_dim_0:0:76,phony_dim_1:0:1;phony_dim_0:0:76,phony_dim_1:1:2;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ # with open('./smap_nonempty_nexustile.bin', 'w') as f:
+ # f.write(results[0])
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
+ self.assertEqual(0, len(nexus_tile.tile.swath_tile.meta_data))
+
+ tile = nexus_tile.tile.swath_tile
+ self.assertEqual(76, from_shaped_array(tile.latitude).size)
+ self.assertEqual(76, from_shaped_array(tile.longitude).size)
+
+ tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.variable_data))
+ self.assertEqual((76, 1), tile1_data.shape)
+ self.assertEqual(43, np.ma.count(tile1_data))
+ self.assertAlmostEqual(-50.056,
+ np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.latitude))),
+ places=3)
+ self.assertAlmostEqual(-47.949,
+ np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.latitude))),
+ places=3)
+
+ tile2_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.swath_tile.variable_data))
+ self.assertEqual((76, 1), tile2_data.shape)
+ self.assertEqual(43, np.ma.count(tile2_data))
+
+ self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True), "Both tiles contain identical data")
+
+ self.assertEqual(1427820162, np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.time))[0])
+
+
+class TestReadCcmpData(unittest.TestCase):
+
+ def test_read_not_empty_ccmp(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ccmp.nc')
+
+ ccmp_reader = processors.GridReadingProcessor('uwnd', 'latitude', 'longitude', time='time', meta='vwnd')
+
+ results = list(ccmp_reader.process(
+ "time:0:1,longitude:0:87,latitude:0:38;time:1:2,longitude:0:87,latitude:0:38;time:2:3,longitude:0:87,latitude:0:38;time:3:4,longitude:0:87,latitude:0:38;file://%s" % test_file))
+
+ self.assertEqual(4, len(results))
+
+ # with open('./ccmp_nonempty_nexustile.bin', 'w') as f:
+ # f.write(results[0])
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+ self.assertEqual(1, len(nexus_tile.tile.grid_tile.meta_data))
+
+ tile = nexus_tile.tile.grid_tile
+ self.assertEqual(38, from_shaped_array(tile.latitude).size)
+ self.assertEqual(87, from_shaped_array(tile.longitude).size)
+ self.assertEqual((1, 38, 87), from_shaped_array(tile.variable_data).shape)
+
+ tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))
+ self.assertEqual(3306, np.ma.count(tile1_data))
+ self.assertAlmostEqual(-78.375,
+ np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ places=3)
+ self.assertAlmostEqual(-69.125,
+ np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ places=3)
+
+ self.assertEqual(1451606400, results[0].tile.grid_tile.time)
+
+
+class TestReadAvhrrData(unittest.TestCase):
+ def test_read_not_empty_avhrr(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_avhrr.nc4')
+
+ avhrr_reader = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time')
+
+ results = list(avhrr_reader.process("time:0:1,lat:0:10,lon:0:10;file://%s" % test_file))
+
+ self.assertEqual(1, len(results))
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+
+ tile = nexus_tile.tile.grid_tile
+ self.assertEqual(10, from_shaped_array(tile.latitude).size)
+ self.assertEqual(10, from_shaped_array(tile.longitude).size)
+ self.assertEqual((1, 10, 10), from_shaped_array(tile.variable_data).shape)
+
+ tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))
+ self.assertEqual(100, np.ma.count(tile1_data))
+ self.assertAlmostEqual(-39.875,
+ np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ places=3)
+ self.assertAlmostEqual(-37.625,
+ np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ places=3)
+
+ self.assertEqual(1462060800, results[0].tile.grid_tile.time)
+ self.assertAlmostEqual(289.71,
+ np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))[
+ 0, 0, 0],
+ places=3)
+
+
+class TestReadWSWMData(unittest.TestCase):
+
+ def test_read_not_empty_wswm(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_wswm.nc')
+
+ wswm_reader = processors.TimeSeriesReadingProcessor('Qout', 'lat', 'lon', 'time')
+
+ results = list(wswm_reader.process("time:0:1,rivid:0:500;time:0:1,rivid:500:1000;file://%s" % test_file))
+
+ self.assertEqual(2, len(results))
+
+ for nexus_tile in results:
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('time_series_tile'))
+
+ tile = nexus_tile.tile.time_series_tile
+ self.assertEqual(500, from_shaped_array(tile.latitude).size)
+ self.assertEqual(500, from_shaped_array(tile.longitude).size)
+ self.assertEqual((1, 500), from_shaped_array(tile.variable_data).shape)
+
+ tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.variable_data))
+ self.assertEqual(500, np.ma.count(tile1_data))
+ self.assertAlmostEqual(41.390,
+ np.ma.min(
+ np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.latitude))),
+ places=3)
+ self.assertAlmostEqual(42.071,
+ np.ma.max(
+ np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.latitude))),
+ places=3)
+
+ self.assertEqual(852098400, results[0].tile.time_series_tile.time)
+ self.assertAlmostEqual(0.009,
+ np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.variable_data))[
+ 0, 0],
+ places=3)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/tilesumarizingprocessor_test.py b/tests/tilesumarizingprocessor_test.py
new file mode 100644
index 0000000..b9735d8
--- /dev/null
+++ b/tests/tilesumarizingprocessor_test.py
@@ -0,0 +1,80 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import unittest
+from os import path
+
+import processors
+
+
+class TestSummarizeTile(unittest.TestCase):
+ def test_summarize_swath(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'smap_nonempty_nexustile.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ summarizer = processors.TileSummarizingProcessor()
+
+ results = list(summarizer.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile = results[0]
+
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
+ self.assertTrue(nexus_tile.HasField('summary'))
+
+ # Check summary
+ tile_summary = nexus_tile.summary
+ self.assertAlmostEqual(-50.056, tile_summary.bbox.lat_min, places=3)
+ self.assertAlmostEqual(-47.949, tile_summary.bbox.lat_max, places=3)
+ self.assertAlmostEqual(22.376, tile_summary.bbox.lon_min, places=3)
+ self.assertAlmostEqual(36.013, tile_summary.bbox.lon_max, places=3)
+
+ self.assertAlmostEqual(33.067, tile_summary.stats.min, places=3)
+ self.assertEqual(40, tile_summary.stats.max)
+ self.assertAlmostEqual(36.6348, tile_summary.stats.mean, places=3)
+ self.assertEqual(43, tile_summary.stats.count)
+
+ self.assertEqual(1427820162, tile_summary.stats.min_time)
+ self.assertEqual(1427820162, tile_summary.stats.max_time)
+
+ def test_summarize_grid(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'avhrr_nonempty_nexustile.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ summarizer = processors.TileSummarizingProcessor()
+
+ results = list(summarizer.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile = results[0]
+
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+ self.assertTrue(nexus_tile.HasField('summary'))
+
+ # Check summary
+ tile_summary = nexus_tile.summary
+ self.assertAlmostEqual(-39.875, tile_summary.bbox.lat_min, places=3)
+ self.assertAlmostEqual(-37.625, tile_summary.bbox.lat_max, places=3)
+ self.assertAlmostEqual(-129.875, tile_summary.bbox.lon_min, places=3)
+ self.assertAlmostEqual(-127.625, tile_summary.bbox.lon_max, places=3)
+
+ self.assertAlmostEqual(288.5099, tile_summary.stats.min, places=3)
+ self.assertAlmostEqual(290.4, tile_summary.stats.max, places=3)
+ self.assertAlmostEqual(289.4443, tile_summary.stats.mean, places=3)
+ self.assertEqual(100, tile_summary.stats.count)
+
+ self.assertEqual(1462838400, tile_summary.stats.min_time)
+ self.assertEqual(1462838400, tile_summary.stats.max_time)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/winddirspeedtouv_test.py b/tests/winddirspeedtouv_test.py
new file mode 100644
index 0000000..bb87d81
--- /dev/null
+++ b/tests/winddirspeedtouv_test.py
@@ -0,0 +1,89 @@
+"""
+Copyright (c) 2016 Jet Propulsion Laboratory,
+California Institute of Technology. All rights reserved
+"""
+import unittest
+from os import path
+
+import numpy as np
+from nexusproto.serialization import from_shaped_array
+
+import processors
+
+
+class TestAscatbUData(unittest.TestCase):
+
+ def test_u_conversion(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascatb_nonempty_nexustile.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ converter = processors.WindDirSpeedToUV('U')
+
+ results = list(converter.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile = results[0]
+
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
+
+ # Check data
+ tile_data = np.ma.masked_invalid(from_shaped_array(nexus_tile.tile.swath_tile.variable_data))
+ self.assertEqual(82, np.ma.count(tile_data))
+
+ # Check meta data
+ meta_list = nexus_tile.tile.swath_tile.meta_data
+ self.assertEqual(3, len(meta_list))
+ wind_dir = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_dir')
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape)
+ self.assertIsNotNone(wind_dir)
+ wind_speed = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_speed')
+ self.assertIsNotNone(wind_speed)
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_speed.meta_data)).shape)
+ wind_v = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_v')
+ self.assertIsNotNone(wind_v)
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_v.meta_data)).shape)
+
+
+class TestAscatbVData(unittest.TestCase):
+
+ def test_u_conversion(self):
+ test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascatb_nonempty_nexustile.bin')
+
+ with open(test_file, 'rb') as f:
+ nexustile_str = f.read()
+
+ converter = processors.WindDirSpeedToUV('V')
+
+ results = list(converter.process(nexustile_str))
+
+ self.assertEqual(1, len(results))
+
+ nexus_tile = results[0]
+
+ self.assertTrue(nexus_tile.HasField('tile'))
+ self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
+
+ # Check data
+ tile_data = np.ma.masked_invalid(from_shaped_array(nexus_tile.tile.swath_tile.variable_data))
+ self.assertEqual(82, np.ma.count(tile_data))
+
+ # Check meta data
+ meta_list = nexus_tile.tile.swath_tile.meta_data
+ self.assertEqual(3, len(meta_list))
+ wind_dir = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_dir')
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape)
+ self.assertIsNotNone(wind_dir)
+ wind_speed = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_speed')
+ self.assertIsNotNone(wind_speed)
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_speed.meta_data)).shape)
+ wind_u = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_u')
+ self.assertIsNotNone(wind_u)
+ self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_u.meta_data)).shape)
+
+
+if __name__ == '__main__':
+ unittest.main()