You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/17 22:55:35 UTC
[incubator-sdap-ingester] branch granule-ingester updated:
granule_ingester tests, but not test granules
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch granule-ingester
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/granule-ingester by this push:
new ca35122 granule_ingester tests, but not test granules
ca35122 is described below
commit ca351226f0b0bf48d2d62634a1a2a1a48a733432
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Wed Jun 17 15:55:18 2020 -0700
granule_ingester tests, but not test granules
---
granule_ingester/tests/__init__.py | 0
.../tests/config_files/analysed_sst.yml | 16 ++
.../config_files/ingestion_config_testfile.yaml | 17 ++
granule_ingester/tests/pipeline/__init__.py | 0
granule_ingester/tests/pipeline/test_Pipeline.py | 104 ++++++++
granule_ingester/tests/processors/__init__.py | 0
.../tests/processors/test_GenerateTileId.py | 22 ++
.../tests/reading_processors/__init__.py | 0
.../test_EccoReadingProcessor.py | 64 +++++
.../test_GridReadingProcessor.py | 265 +++++++++++++++++++++
.../test_SwathReadingProcessor.py | 74 ++++++
.../test_TileReadingProcessor.py | 29 +++
.../test_TimeSeriesReadingProcessor.py | 86 +++++++
granule_ingester/tests/slicers/__init__.py | 0
.../tests/slicers/test_SliceFileByDimension.py | 122 ++++++++++
.../tests/slicers/test_SliceFileByStepSize.py | 105 ++++++++
.../tests/slicers/test_SliceFileByTilesDesired.py | 88 +++++++
granule_ingester/tests/slicers/test_TileSlicer.py | 68 ++++++
granule_ingester/tests/writers/__init__.py | 0
granule_ingester/tests/writers/test_SolrStore.py | 54 +++++
20 files changed, 1114 insertions(+)
diff --git a/granule_ingester/tests/__init__.py b/granule_ingester/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/granule_ingester/tests/config_files/analysed_sst.yml b/granule_ingester/tests/config_files/analysed_sst.yml
new file mode 100644
index 0000000..9148f98
--- /dev/null
+++ b/granule_ingester/tests/config_files/analysed_sst.yml
@@ -0,0 +1,16 @@
+slicer:
+ name: sliceFileByStepSize
+ dimension_step_sizes:
+ time: 1
+ lon: 10
+ lat: 10
+processors:
+ - name: GridReadingProcessor
+ latitude: lat
+ longitude: lon
+ time: time
+ variable_to_read: analysed_sst
+ - name: emptyTileFilter
+ - name: tileSummary
+ dataset_name: AVHRR_sst
+ - name: generateTileId
diff --git a/granule_ingester/tests/config_files/ingestion_config_testfile.yaml b/granule_ingester/tests/config_files/ingestion_config_testfile.yaml
new file mode 100644
index 0000000..9af889d
--- /dev/null
+++ b/granule_ingester/tests/config_files/ingestion_config_testfile.yaml
@@ -0,0 +1,17 @@
+granule:
+ resource: ../foo/bar.nc
+slicer:
+ name: sliceFileByStepSize
+ dimension_step_sizes:
+ time: 1
+ lat: 33
+ lon: 26
+processors:
+ - name: EccoReadingProcessor
+ latitude: YC
+ longitude: XC
+ time: time
+ depth: Z
+ tile: tile
+ variable_to_read: THETA
+ - name: generateTileId
diff --git a/granule_ingester/tests/pipeline/__init__.py b/granule_ingester/tests/pipeline/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/granule_ingester/tests/pipeline/test_Pipeline.py b/granule_ingester/tests/pipeline/test_Pipeline.py
new file mode 100644
index 0000000..6db14e3
--- /dev/null
+++ b/granule_ingester/tests/pipeline/test_Pipeline.py
@@ -0,0 +1,104 @@
+import os
+import unittest
+
+from nexusproto import DataTile_pb2 as nexusproto
+
+from sdap.pipeline.Pipeline import Pipeline
+from sdap.processors import GenerateTileId
+from sdap.processors.reading_processors import EccoReadingProcessor
+from sdap.slicers.SliceFileByStepSize import *
+from sdap.writers import DataStore, MetadataStore
+
+
+class TestPipeline(unittest.TestCase):
+ class MockProcessorNoParams:
+ def __init__(self):
+ pass
+
+ class MockProcessorWithParams:
+ def __init__(self, test_param):
+ self.test_param = test_param
+
+ def test_parse_config(self):
+ class MockDataStore(DataStore):
+ def save_data(self, nexus_tile: nexusproto.NexusTile) -> None:
+ pass
+
+ class MockMetadataStore(MetadataStore):
+ def save_metadata(self, nexus_tile: nexusproto.NexusTile) -> None:
+ pass
+
+ relative_path = "../config_files/ingestion_config_testfile.yaml"
+ file_path = os.path.join(os.path.dirname(__file__), relative_path)
+ pipeline = Pipeline.from_file(config_path=str(file_path),
+ data_store_factory=MockDataStore,
+ metadata_store_factory=MockMetadataStore)
+
+ self.assertEqual(pipeline._data_store_factory, MockDataStore)
+ self.assertEqual(pipeline._metadata_store_factory, MockMetadataStore)
+ self.assertEqual(type(pipeline._slicer), SliceFileByStepSize)
+ self.assertEqual(type(pipeline._tile_processors[0]), EccoReadingProcessor)
+ self.assertEqual(type(pipeline._tile_processors[1]), GenerateTileId)
+
+ def test_parse_module(self):
+ module_mappings = {
+ "sliceFileByStepSize": SliceFileByStepSize
+ }
+
+ module_config = {
+ "name": "sliceFileByStepSize",
+ "dimension_step_sizes": {
+ "time": 1,
+ "lat": 10,
+ "lon": 10
+ }
+ }
+ module = Pipeline._parse_module(module_config, module_mappings)
+ self.assertEqual(SliceFileByStepSize, type(module))
+ self.assertEqual(module_config['dimension_step_sizes'], module._dimension_step_sizes)
+
+ def test_parse_module_with_no_parameters(self):
+ module_mappings = {"MockModule": TestPipeline.MockProcessorNoParams}
+ module_config = {"name": "MockModule"}
+ module = Pipeline._parse_module(module_config, module_mappings)
+ self.assertEqual(type(module), TestPipeline.MockProcessorNoParams)
+
+ def test_parse_module_with_too_many_parameters(self):
+ module_mappings = {"MockModule": TestPipeline.MockProcessorNoParams}
+ module_config = {
+ "name": "MockModule",
+ "bogus_param": True
+ }
+ self.assertRaises(TypeError, Pipeline._parse_module, module_config, module_mappings)
+
+ def test_parse_module_with_missing_parameters(self):
+ module_mappings = {"MockModule": TestPipeline.MockProcessorWithParams}
+ module_config = {
+ "name": "MockModule"
+ }
+
+ self.assertRaises(TypeError, Pipeline._parse_module, module_config, module_mappings)
+
+ def test_process_tile(self):
+ # class MockIdProcessor:
+ # def process(self, tile, *args, **kwargs):
+ # tile.summary.tile_id = "test_id"
+ # return tile
+ #
+ # class MockReadingProcessor:
+ # def process(self, tile, *args, **kwargs):
+ # dataset = kwargs['dataset']
+ # tile.tile.grid_tile.variable_data.CopyFrom(to_shaped_array(dataset['test_variable']))
+ # return tile
+ #
+ # test_dataset = xr.Dataset({"test_variable": [1, 2, 3]})
+ # input_tile = nexusproto.NexusTile.SerializeToString(NexusTile())
+ # processor_list = [MockIdProcessor(), MockReadingProcessor()]
+ #
+ # output_tile = _process_tile_in_worker(processor_list, test_dataset, input_tile)
+ # output_tile = nexusproto.NexusTile.FromString(output_tile)
+ # tile_data = from_shaped_array(output_tile.tile.grid_tile.variable_data)
+ #
+ # np.testing.assert_equal(tile_data, [1, 2, 3])
+ # self.assertEqual(output_tile.summary.tile_id, "test_id")
+ ...
diff --git a/granule_ingester/tests/processors/__init__.py b/granule_ingester/tests/processors/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/granule_ingester/tests/processors/test_GenerateTileId.py b/granule_ingester/tests/processors/test_GenerateTileId.py
new file mode 100644
index 0000000..92c1964
--- /dev/null
+++ b/granule_ingester/tests/processors/test_GenerateTileId.py
@@ -0,0 +1,22 @@
+import unittest
+
+import uuid
+from nexusproto import DataTile_pb2 as nexusproto
+
+from sdap.processors import GenerateTileId
+
+
+class TestGenerateTileId(unittest.TestCase):
+
+ def test_process(self):
+ processor = GenerateTileId()
+
+ tile = nexusproto.NexusTile()
+ tile.summary.granule = 'test_dir/test_granule.nc'
+ tile.summary.data_var_name = 'test_variable'
+ tile.summary.section_spec = 'i:0:90,j:0:90,k:8:9,nv:0:2,tile:4:5,time:8:9'
+
+ expected_id = uuid.uuid3(uuid.NAMESPACE_DNS,
+ 'test_granule.nc' + 'test_variable' + 'i:0:90,j:0:90,k:8:9,nv:0:2,tile:4:5,time:8:9')
+
+ self.assertEqual(str(expected_id), processor.process(tile).summary.tile_id)
diff --git a/granule_ingester/tests/reading_processors/__init__.py b/granule_ingester/tests/reading_processors/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/granule_ingester/tests/reading_processors/test_EccoReadingProcessor.py b/granule_ingester/tests/reading_processors/test_EccoReadingProcessor.py
new file mode 100644
index 0000000..4fc3cbe
--- /dev/null
+++ b/granule_ingester/tests/reading_processors/test_EccoReadingProcessor.py
@@ -0,0 +1,64 @@
+import unittest
+from os import path
+
+import xarray as xr
+from nexusproto import DataTile_pb2 as nexusproto
+
+from sdap.processors.reading_processors import EccoReadingProcessor
+
+
+class TestEccoReadingProcessor(unittest.TestCase):
+
+ def test_generate_tile(self):
+ reading_processor = EccoReadingProcessor(variable_to_read='OBP',
+ latitude='YC',
+ longitude='XC',
+ time='time',
+ tile='tile')
+
+ granule_path = path.join(path.dirname(__file__), '../granules/OBP_native_grid.nc')
+ tile_summary = nexusproto.TileSummary()
+ tile_summary.granule = granule_path
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.CopyFrom(tile_summary)
+
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'tile': slice(10, 11),
+ 'j': slice(0, 15),
+ 'i': slice(0, 7)
+ }
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(output_tile.summary.granule, granule_path)
+ self.assertEqual(output_tile.tile.ecco_tile.tile, 10)
+ self.assertEqual(output_tile.tile.ecco_tile.time, 695563200)
+ self.assertEqual(output_tile.tile.ecco_tile.variable_data.shape, [15, 7])
+ self.assertEqual(output_tile.tile.ecco_tile.latitude.shape, [15, 7])
+ self.assertEqual(output_tile.tile.ecco_tile.longitude.shape, [15, 7])
+
+ def test_generate_tile_with_dims_out_of_order(self):
+ reading_processor = EccoReadingProcessor(variable_to_read='OBP',
+ latitude='YC',
+ longitude='XC',
+ time='time',
+ tile='tile')
+ granule_path = path.join(path.dirname(__file__), '../granules/OBP_native_grid.nc')
+ input_tile = nexusproto.NexusTile()
+
+ dimensions_to_slices = {
+ 'j': slice(0, 15),
+ 'tile': slice(10, 11),
+ 'i': slice(0, 7),
+ 'time': slice(0, 1)
+ }
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(output_tile.tile.ecco_tile.tile, 10)
+ self.assertEqual(output_tile.tile.ecco_tile.time, 695563200)
+ self.assertEqual(output_tile.tile.ecco_tile.variable_data.shape, [15, 7])
+ self.assertEqual(output_tile.tile.ecco_tile.latitude.shape, [15, 7])
+ self.assertEqual(output_tile.tile.ecco_tile.longitude.shape, [15, 7])
diff --git a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
new file mode 100644
index 0000000..19384b3
--- /dev/null
+++ b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
@@ -0,0 +1,265 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from os import path
+
+import numpy as np
+import xarray as xr
+from nexusproto import DataTile_pb2 as nexusproto
+from nexusproto.serialization import from_shaped_array
+
+from sdap.processors.reading_processors import GridReadingProcessor
+
+
+class TestReadMurData(unittest.TestCase):
+
+ def test_read_empty_mur(self):
+ reading_processor = GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time')
+ granule_path = path.join(path.dirname(__file__), '../granules/empty_mur.nc4')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'lat': slice(0, 10),
+ 'lon': slice(0, 5)
+ }
+ with xr.open_dataset(granule_path) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
+ self.assertEqual(1451638800, output_tile.tile.grid_tile.time)
+ self.assertEqual([10, 5], output_tile.tile.grid_tile.variable_data.shape)
+ self.assertEqual([10], output_tile.tile.grid_tile.latitude.shape)
+ self.assertEqual([5], output_tile.tile.grid_tile.longitude.shape)
+
+ masked_data = np.ma.masked_invalid(from_shaped_array(output_tile.tile.grid_tile.variable_data))
+ self.assertEqual(0, np.ma.count(masked_data))
+
+ def test_read_not_empty_mur(self):
+ reading_processor = GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time')
+ granule_path = path.join(path.dirname(__file__), '../granules/not_empty_mur.nc4')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'lat': slice(0, 10),
+ 'lon': slice(0, 5)
+ }
+ with xr.open_dataset(granule_path) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
+ self.assertEqual(1451638800, output_tile.tile.grid_tile.time)
+ self.assertEqual([10, 5], output_tile.tile.grid_tile.variable_data.shape)
+ self.assertEqual([10], output_tile.tile.grid_tile.latitude.shape)
+ self.assertEqual([5], output_tile.tile.grid_tile.longitude.shape)
+
+ masked_data = np.ma.masked_invalid(from_shaped_array(output_tile.tile.grid_tile.variable_data))
+ self.assertEqual(50, np.ma.count(masked_data))
+
+
+class TestReadCcmpData(unittest.TestCase):
+
+ def test_read_not_empty_ccmp(self):
+ reading_processor = GridReadingProcessor('uwnd', 'latitude', 'longitude', time='time')
+ granule_path = path.join(path.dirname(__file__), '../granules/not_empty_ccmp.nc')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'latitude': slice(0, 38),
+ 'longitude': slice(0, 87)
+ }
+ with xr.open_dataset(granule_path) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
+ self.assertEqual(1451606400, output_tile.tile.grid_tile.time)
+ self.assertEqual([38, 87], output_tile.tile.grid_tile.variable_data.shape)
+ self.assertEqual([38], output_tile.tile.grid_tile.latitude.shape)
+ self.assertEqual([87], output_tile.tile.grid_tile.longitude.shape)
+
+ masked_data = np.ma.masked_invalid(from_shaped_array(output_tile.tile.grid_tile.variable_data))
+ self.assertEqual(3306, np.ma.count(masked_data))
+
+ # test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ccmp.nc')
+ #
+ # ccmp_reader = GridReadingProcessor('uwnd', 'latitude', 'longitude', time='time', meta='vwnd')
+ #
+ # input_tile = nexusproto.NexusTile()
+ # tile_summary = nexusproto.TileSummary()
+ # tile_summary.granule = "file:%s" % test_file
+ # tile_summary.section_spec = "time:0:1,longitude:0:87,latitude:0:38"
+ # input_tile.summary.CopyFrom(tile_summary)
+ #
+ # results = list(ccmp_reader.process(input_tile))
+ #
+ # self.assertEqual(1, len(results))
+ #
+ # # with open('./ccmp_nonempty_nexustile.bin', 'w') as f:
+ # # f.write(results[0])
+ #
+ # for nexus_tile in results:
+ # self.assertTrue(nexus_tile.HasField('tile'))
+ # self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+ # self.assertEqual(1, len(nexus_tile.tile.grid_tile.meta_data))
+ #
+ # tile = nexus_tile.tile.grid_tile
+ # self.assertEqual(38, from_shaped_array(tile.latitude).size)
+ # self.assertEqual(87, from_shaped_array(tile.longitude).size)
+ # self.assertEqual((1, 38, 87), from_shaped_array(tile.variable_data).shape)
+ #
+ # tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))
+ # self.assertEqual(3306, np.ma.count(tile1_data))
+ # self.assertAlmostEqual(-78.375,
+ # np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ # places=3)
+ # self.assertAlmostEqual(-69.125,
+ # np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ # places=3)
+ #
+ # self.assertEqual(1451606400, results[0].tile.grid_tile.time)
+
+
+class TestReadAvhrrData(unittest.TestCase):
+ def test_read_not_empty_avhrr(self):
+ reading_processor = GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time')
+ granule_path = path.join(path.dirname(__file__), '../granules/not_empty_avhrr.nc4')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'lat': slice(0, 5),
+ 'lon': slice(0, 10)
+ }
+ with xr.open_dataset(granule_path) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
+ self.assertEqual(1462060800, output_tile.tile.grid_tile.time)
+ self.assertEqual([5, 10], output_tile.tile.grid_tile.variable_data.shape)
+ self.assertEqual([5], output_tile.tile.grid_tile.latitude.shape)
+ self.assertEqual([10], output_tile.tile.grid_tile.longitude.shape)
+
+ masked_data = np.ma.masked_invalid(from_shaped_array(output_tile.tile.grid_tile.variable_data))
+ self.assertEqual(50, np.ma.count(masked_data))
+ # test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_avhrr.nc4')
+ #
+ # avhrr_reader = GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time')
+ #
+ # input_tile = nexusproto.NexusTile()
+ # tile_summary = nexusproto.TileSummary()
+ # tile_summary.granule = "file:%s" % test_file
+ # tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
+ # input_tile.summary.CopyFrom(tile_summary)
+ #
+ # results = list(avhrr_reader.process(input_tile))
+ #
+ # self.assertEqual(1, len(results))
+ #
+ # for nexus_tile in results:
+ # self.assertTrue(nexus_tile.HasField('tile'))
+ # self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+ #
+ # tile = nexus_tile.tile.grid_tile
+ # self.assertEqual(10, from_shaped_array(tile.latitude).size)
+ # self.assertEqual(10, from_shaped_array(tile.longitude).size)
+ # self.assertEqual((1, 10, 10), from_shaped_array(tile.variable_data).shape)
+ #
+ # tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))
+ # self.assertEqual(100, np.ma.count(tile1_data))
+ # self.assertAlmostEqual(-39.875,
+ # np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ # places=3)
+ # self.assertAlmostEqual(-37.625,
+ # np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))),
+ # places=3)
+ #
+ # self.assertEqual(1462060800, results[0].tile.grid_tile.time)
+ # self.assertAlmostEqual(289.71,
+ # np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))[
+ # 0, 0, 0],
+ # places=3)
+
+
+class TestReadInterpEccoData(unittest.TestCase):
+ def setUp(self):
+ self.module = GridReadingProcessor('OBP', 'latitude', 'longitude', x_dim='i', y_dim='j',
+ time='time')
+
+ def test_read_indexed_ecco(self):
+ reading_processor = GridReadingProcessor(variable_to_read='OBP',
+ latitude='latitude',
+ longitude='longitude',
+ time='time')
+ granule_path = path.join(path.dirname(__file__), '../granules/OBP_2017_01.nc')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'j': slice(0, 5),
+ 'i': slice(0, 10)
+ }
+ with xr.open_dataset(granule_path) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
+ self.assertEqual(1484568000, output_tile.tile.grid_tile.time)
+ self.assertEqual([5, 10], output_tile.tile.grid_tile.variable_data.shape)
+ self.assertEqual([5], output_tile.tile.grid_tile.latitude.shape)
+ self.assertEqual([10], output_tile.tile.grid_tile.longitude.shape)
+
+ masked_data = np.ma.masked_invalid(from_shaped_array(output_tile.tile.grid_tile.variable_data))
+ self.assertEqual(50, np.ma.count(masked_data))
+
+ # test_file = path.join(path.dirname(__file__), 'datafiles', 'OBP_2017_01.nc')
+ #
+ # input_tile = nexusproto.NexusTile()
+ # tile_summary = nexusproto.TileSummary()
+ # tile_summary.granule = "file:%s" % test_file
+ # tile_summary.section_spec = "time:0:1,j:0:10,i:0:10"
+ # input_tile.summary.CopyFrom(tile_summary)
+ #
+ # results = list(self.module.process(input_tile))
+ #
+ # self.assertEqual(1, len(results))
+ #
+ # for nexus_tile in results:
+ # self.assertTrue(nexus_tile.HasField('tile'))
+ # self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
+ #
+ # tile = nexus_tile.tile.grid_tile
+ # self.assertEqual(10, len(from_shaped_array(tile.latitude)))
+ # self.assertEqual(10, len(from_shaped_array(tile.longitude)))
+ #
+ # the_data = np.ma.masked_invalid(from_shaped_array(tile.variable_data))
+ # self.assertEqual((1, 10, 10), the_data.shape)
+ # self.assertEqual(100, np.ma.count(the_data))
+ # self.assertEqual(1484568000, tile.time)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/granule_ingester/tests/reading_processors/test_SwathReadingProcessor.py b/granule_ingester/tests/reading_processors/test_SwathReadingProcessor.py
new file mode 100644
index 0000000..03d20cb
--- /dev/null
+++ b/granule_ingester/tests/reading_processors/test_SwathReadingProcessor.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from os import path
+
+import xarray as xr
+from nexusproto import DataTile_pb2 as nexusproto
+
+from sdap.processors.reading_processors import SwathReadingProcessor
+
+
+class TestReadAscatbData(unittest.TestCase):
+ def test_read_not_empty_ascatb(self):
+ reading_processor = SwathReadingProcessor(variable_to_read='wind_speed',
+ latitude='lat',
+ longitude='lon',
+ time='time')
+ granule_path = path.join(path.dirname(__file__), '../granules/not_empty_ascatb.nc4')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'NUMROWS': slice(0, 1),
+ 'NUMCELLS': slice(0, 82)
+ }
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
+ self.assertEqual([1, 82], output_tile.tile.swath_tile.time.shape)
+ self.assertEqual([1, 82], output_tile.tile.swath_tile.variable_data.shape)
+ self.assertEqual([1, 82], output_tile.tile.swath_tile.latitude.shape)
+ self.assertEqual([1, 82], output_tile.tile.swath_tile.longitude.shape)
+
+
+class TestReadSmapData(unittest.TestCase):
+ def test_read_not_empty_smap(self):
+ reading_processor = SwathReadingProcessor(
+ variable_to_read='smap_sss',
+ latitude='lat',
+ longitude='lon',
+ time='row_time')
+ granule_path = path.join(path.dirname(__file__), '../granules/not_empty_smap.h5')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'phony_dim_0': slice(0, 38),
+ 'phony_dim_1': slice(0, 1)
+ }
+
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule)
+ self.assertEqual([1], output_tile.tile.swath_tile.time.shape)
+ self.assertEqual([38, 1], output_tile.tile.swath_tile.variable_data.shape)
+ self.assertEqual([38, 1], output_tile.tile.swath_tile.latitude.shape)
+ self.assertEqual([38, 1], output_tile.tile.swath_tile.longitude.shape)
diff --git a/granule_ingester/tests/reading_processors/test_TileReadingProcessor.py b/granule_ingester/tests/reading_processors/test_TileReadingProcessor.py
new file mode 100644
index 0000000..cf54a86
--- /dev/null
+++ b/granule_ingester/tests/reading_processors/test_TileReadingProcessor.py
@@ -0,0 +1,29 @@
+import unittest
+from collections import OrderedDict
+from os import path
+
+import xarray as xr
+
+from sdap.processors.reading_processors import TileReadingProcessor
+
+
+class TestEccoReadingProcessor(unittest.TestCase):
+
+ def test_slices_for_variable(self):
+ dimensions_to_slices = {
+ 'j': slice(0, 1),
+ 'tile': slice(0, 1),
+ 'i': slice(0, 1),
+ 'time': slice(0, 1)
+ }
+
+ expected = {
+ 'tile': slice(0, 1, None),
+ 'j': slice(0, 1, None),
+ 'i': slice(0, 1, None)
+ }
+
+ granule_path = path.join(path.dirname(__file__), '../granules/OBP_native_grid.nc')
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ slices = TileReadingProcessor._slices_for_variable(ds['XC'], dimensions_to_slices)
+ self.assertEqual(slices, expected)
diff --git a/granule_ingester/tests/reading_processors/test_TimeSeriesReadingProcessor.py b/granule_ingester/tests/reading_processors/test_TimeSeriesReadingProcessor.py
new file mode 100644
index 0000000..be6a2d5
--- /dev/null
+++ b/granule_ingester/tests/reading_processors/test_TimeSeriesReadingProcessor.py
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from os import path
+
+import xarray as xr
+from nexusproto import DataTile_pb2 as nexusproto
+
+from sdap.processors.reading_processors import TimeSeriesReadingProcessor
+
+
+class TestReadWSWMData(unittest.TestCase):
+
+ def test_read_not_empty_wswm(self):
+ reading_processor = TimeSeriesReadingProcessor('Qout', 'lat', 'lon', time='time')
+ granule_path = path.join(path.dirname(__file__), '../granules/not_empty_wswm.nc')
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.granule = granule_path
+
+ dimensions_to_slices = {
+ 'time': slice(0, 5832),
+ 'rivid': slice(0, 1),
+ }
+ with xr.open_dataset(granule_path) as ds:
+ output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+
+ self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
+ self.assertEqual([5832], output_tile.tile.time_series_tile.time.shape)
+ self.assertEqual([5832, 1], output_tile.tile.time_series_tile.variable_data.shape)
+ self.assertEqual([1], output_tile.tile.time_series_tile.latitude.shape)
+ self.assertEqual([1], output_tile.tile.time_series_tile.longitude.shape)
+
+ # test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_wswm.nc')
+ # wswm_reader = TimeSeriesReadingProcessor('Qout', 'lat', 'lon', 'time')
+ #
+ # input_tile = nexusproto.NexusTile()
+ # tile_summary = nexusproto.TileSummary()
+ # tile_summary.granule = "file:%s" % test_file
+ # tile_summary.section_spec = "time:0:5832,rivid:0:1"
+ # input_tile.summary.CopyFrom(tile_summary)
+ #
+ # results = list(wswm_reader.process(input_tile))
+ #
+ # self.assertEqual(1, len(results))
+ #
+ # for nexus_tile in results:
+ # self.assertTrue(nexus_tile.HasField('tile'))
+ # self.assertTrue(nexus_tile.tile.HasField('time_series_tile'))
+ #
+ # tile = nexus_tile.tile.time_series_tile
+ # self.assertEqual(1, from_shaped_array(tile.latitude).size)
+ # self.assertEqual(1, from_shaped_array(tile.longitude).size)
+ # self.assertEqual((5832, 1), from_shaped_array(tile.variable_data).shape)
+ #
+ # tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.variable_data))
+ # self.assertEqual(5832, np.ma.count(tile1_data))
+ # self.assertAlmostEqual(45.837,
+ # np.ma.min(
+ # np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.latitude))),
+ # places=3)
+ # self.assertAlmostEqual(-122.789,
+ # np.ma.max(
+ # np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.longitude))),
+ # places=3)
+ #
+ # tile1_times = from_shaped_array(results[0].tile.time_series_tile.time)
+ # self.assertEqual(852098400, tile1_times[0])
+ # self.assertEqual(915073200, tile1_times[-1])
+ # self.assertAlmostEqual(1.473,
+ # np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.variable_data))[
+ # 0, 0],
+ # places=3)
diff --git a/granule_ingester/tests/slicers/__init__.py b/granule_ingester/tests/slicers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/granule_ingester/tests/slicers/test_SliceFileByDimension.py b/granule_ingester/tests/slicers/test_SliceFileByDimension.py
new file mode 100644
index 0000000..4cf264a
--- /dev/null
+++ b/granule_ingester/tests/slicers/test_SliceFileByDimension.py
@@ -0,0 +1,122 @@
+# import unittest
+# from collections import Set
+#
+# from netCDF4 import Dataset
+# from sdap.slicers.SliceFileByDimension import SliceFileByDimension
+#
+#
+# class TestSliceFileByTilesDesired(unittest.TestCase):
+#
+# def test_generate_slices(self):
+# netcdf_path = 'tests/granules/THETA_199201.nc'
+# dataset = Dataset(netcdf_path)
+# dimension_specs = {value.name: value.size for key,
+# value in dataset.dimensions.items()}
+#
+# slicer = SliceFileByDimension(slice_dimension='depth',
+# dimension_name_prefix=None)
+# slices = slicer.generate_slices(dimension_specs)
+# expected_slices = ['nv:0:2,time:0:1,longitude:0:720,depth:0:1,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:1:2,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:2:3,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:3:4,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:4:5,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:5:6,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:6:7,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:7:8,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:8:9,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:9:10,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:10:11,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:11:12,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:12:13,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:13:14,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:14:15,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:15:16,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:16:17,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:17:18,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:18:19,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:19:20,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:20:21,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:21:22,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:22:23,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:23:24,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:24:25,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:25:26,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:26:27,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:27:28,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:28:29,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:29:30,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:30:31,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:31:32,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:32:33,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:33:34,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:34:35,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:35:36,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:36:37,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:37:38,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:38:39,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:39:40,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:40:41,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:41:42,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:42:43,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:43:44,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:44:45,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:45:46,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:46:47,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:47:48,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:48:49,latitude:0:360',
+# 'nv:0:2,time:0:1,longitude:0:720,depth:49:50,latitude:0:360']
+#
+# self.assertEqual(slices, expected_slices)
+#
+# def test_generate_slices_indexed(self):
+# netcdf_path = 'tests/granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5'
+# dataset = Dataset(netcdf_path)
+# dimension_specs = {value.name: value.size for key,
+# value in dataset.dimensions.items()}
+#
+# slicer = SliceFileByDimension(slice_dimension='2',
+# dimension_name_prefix='phony_dim_')
+# slices = slicer.generate_slices(dimension_specs)
+# expected_slices = [
+# 'phony_dim_0:0:76,phony_dim_1:0:1624,phony_dim_2:0:1',
+# 'phony_dim_0:0:76,phony_dim_1:0:1624,phony_dim_2:1:2',
+# 'phony_dim_0:0:76,phony_dim_1:0:1624,phony_dim_2:2:3',
+# 'phony_dim_0:0:76,phony_dim_1:0:1624,phony_dim_2:3:4'
+# ]
+#
+# self.assertEqual(slices, expected_slices)
+#
+# def test_indexed_dimension_slicing(self):
+# # for some reason, python automatically prefixes integer-indexed dimensions with "phony_dim_"
+# dimension_specs = {'phony_dim_0': 8, 'phony_dim_1': 8, 'phony_dim_2': 5}
+# slicer = SliceFileByDimension(slice_dimension='2',
+# dimension_name_prefix=None)
+# boundary_slices = slicer._indexed_dimension_slicing(dimension_specs)
+# expected_slices = [
+# 'phony_dim_0:0:8,phony_dim_1:0:8,phony_dim_2:0:1',
+# 'phony_dim_0:0:8,phony_dim_1:0:8,phony_dim_2:1:2',
+# 'phony_dim_0:0:8,phony_dim_1:0:8,phony_dim_2:2:3',
+# 'phony_dim_0:0:8,phony_dim_1:0:8,phony_dim_2:3:4',
+# 'phony_dim_0:0:8,phony_dim_1:0:8,phony_dim_2:4:5'
+# ]
+#
+# self.assertEqual(boundary_slices, expected_slices)
+#
+# def test_generate_tile_boundary_slices(self):
+# dimension_specs = {'lat': 8, 'lon': 8, 'depth': 5}
+# slicer = SliceFileByDimension(slice_dimension='depth',
+# dimension_name_prefix=None)
+# boundary_slices = slicer._generate_tile_boundary_slices(slicer._slice_by_dimension,dimension_specs)
+# expected_slices = [
+# 'lat:0:8,lon:0:8,depth:0:1',
+# 'lat:0:8,lon:0:8,depth:1:2',
+# 'lat:0:8,lon:0:8,depth:2:3',
+# 'lat:0:8,lon:0:8,depth:3:4',
+# 'lat:0:8,lon:0:8,depth:4:5'
+# ]
+#
+# self.assertEqual(boundary_slices, expected_slices)
+#
+# if __name__ == '__main__':
+# unittest.main()
diff --git a/granule_ingester/tests/slicers/test_SliceFileByStepSize.py b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
new file mode 100644
index 0000000..995a67b
--- /dev/null
+++ b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
@@ -0,0 +1,105 @@
+import unittest
+from os import path
+
+import xarray as xr
+
+from sdap.slicers.SliceFileByStepSize import SliceFileByStepSize
+
+
+class TestSliceFileByStepSize(unittest.TestCase):
+
+ def test_generate_slices(self):
+ netcdf_path = path.join(path.dirname(__file__), '../granules/THETA_199201.nc')
+ with xr.open_dataset(netcdf_path, decode_cf=True) as dataset:
+ dimension_steps = {'nv': 2, 'time': 1, 'latitude': 180, 'longitude': 180, 'depth': 2}
+ slicer = SliceFileByStepSize(dimension_step_sizes=dimension_steps)
+ slices = slicer._generate_slices(dimension_specs=dataset.dims)
+ expected_slices = [
+ 'depth:0:2,latitude:0:180,longitude:0:180,nv:0:2,time:0:1',
+ 'depth:0:2,latitude:0:180,longitude:180:360,nv:0:2,time:0:1',
+ 'depth:0:2,latitude:0:180,longitude:360:540,nv:0:2,time:0:1',
+ 'depth:0:2,latitude:0:180,longitude:540:720,nv:0:2,time:0:1',
+ 'depth:0:2,latitude:180:360,longitude:0:180,nv:0:2,time:0:1',
+ 'depth:0:2,latitude:180:360,longitude:180:360,nv:0:2,time:0:1',
+ 'depth:0:2,latitude:180:360,longitude:360:540,nv:0:2,time:0:1',
+ 'depth:0:2,latitude:180:360,longitude:540:720,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:0:180,longitude:0:180,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:0:180,longitude:180:360,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:0:180,longitude:360:540,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:0:180,longitude:540:720,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:180:360,longitude:0:180,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:180:360,longitude:180:360,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:180:360,longitude:360:540,nv:0:2,time:0:1',
+ 'depth:2:4,latitude:180:360,longitude:540:720,nv:0:2,time:0:1'
+ ]
+
+ self.assertEqual(expected_slices, slices)
+
+ def test_generate_slices_indexed(self):
+ netcdf_path = path.join(path.dirname(__file__), '../granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5')
+ with xr.open_dataset(netcdf_path, decode_cf=True) as dataset:
+ dimension_steps = {'phony_dim_0': 76, 'phony_dim_1': 812, 'phony_dim_2': 1}
+ slicer = SliceFileByStepSize(dimension_step_sizes=dimension_steps)
+ slices = slicer._generate_slices(dimension_specs=dataset.dims)
+ expected_slices = [
+ 'phony_dim_0:0:76,phony_dim_1:0:812,phony_dim_2:0:1',
+ 'phony_dim_0:0:76,phony_dim_1:0:812,phony_dim_2:1:2',
+ 'phony_dim_0:0:76,phony_dim_1:0:812,phony_dim_2:2:3',
+ 'phony_dim_0:0:76,phony_dim_1:0:812,phony_dim_2:3:4',
+ 'phony_dim_0:0:76,phony_dim_1:812:1624,phony_dim_2:0:1',
+ 'phony_dim_0:0:76,phony_dim_1:812:1624,phony_dim_2:1:2',
+ 'phony_dim_0:0:76,phony_dim_1:812:1624,phony_dim_2:2:3',
+ 'phony_dim_0:0:76,phony_dim_1:812:1624,phony_dim_2:3:4'
+ ]
+
+ self.assertEqual(slices, expected_slices)
+
+ def test_generate_chunk_boundary_slices(self):
+ dimension_specs = {'time': 5832, 'rivid': 43}
+ dimension_steps = {'time': 2916, 'rivid': 5}
+ slicer = SliceFileByStepSize(dimension_step_sizes=dimension_steps)
+ boundary_slices = slicer._generate_chunk_boundary_slices(dimension_specs)
+ expected_slices = [
+ 'time:0:2916,rivid:0:5',
+ 'time:0:2916,rivid:5:10',
+ 'time:0:2916,rivid:10:15',
+ 'time:0:2916,rivid:15:20',
+ 'time:0:2916,rivid:20:25',
+ 'time:0:2916,rivid:25:30',
+ 'time:0:2916,rivid:30:35',
+ 'time:0:2916,rivid:35:40',
+ 'time:0:2916,rivid:40:43',
+ 'time:2916:5832,rivid:0:5',
+ 'time:2916:5832,rivid:5:10',
+ 'time:2916:5832,rivid:10:15',
+ 'time:2916:5832,rivid:15:20',
+ 'time:2916:5832,rivid:20:25',
+ 'time:2916:5832,rivid:25:30',
+ 'time:2916:5832,rivid:30:35',
+ 'time:2916:5832,rivid:35:40',
+ 'time:2916:5832,rivid:40:43',
+ ]
+
+ self.assertEqual(boundary_slices, expected_slices)
+
+ def test_generate_chunk_boundary_slices_indexed(self):
+ dimension_steps = {'phony_dim_0': 4, 'phony_dim_1': 4, 'phony_dim_2': 3}
+ dimension_specs = {'phony_dim_0': 8, 'phony_dim_1': 8, 'phony_dim_2': 5}
+ slicer = SliceFileByStepSize(dimension_step_sizes=dimension_steps)
+ boundary_slices = slicer._generate_slices(dimension_specs)
+ expected_slices = [
+ 'phony_dim_0:0:4,phony_dim_1:0:4,phony_dim_2:0:3',
+ 'phony_dim_0:0:4,phony_dim_1:0:4,phony_dim_2:3:5',
+ 'phony_dim_0:0:4,phony_dim_1:4:8,phony_dim_2:0:3',
+ 'phony_dim_0:0:4,phony_dim_1:4:8,phony_dim_2:3:5',
+ 'phony_dim_0:4:8,phony_dim_1:0:4,phony_dim_2:0:3',
+ 'phony_dim_0:4:8,phony_dim_1:0:4,phony_dim_2:3:5',
+ 'phony_dim_0:4:8,phony_dim_1:4:8,phony_dim_2:0:3',
+ 'phony_dim_0:4:8,phony_dim_1:4:8,phony_dim_2:3:5',
+ ]
+
+ self.assertEqual(boundary_slices, expected_slices)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/granule_ingester/tests/slicers/test_SliceFileByTilesDesired.py b/granule_ingester/tests/slicers/test_SliceFileByTilesDesired.py
new file mode 100644
index 0000000..2b0b0ae
--- /dev/null
+++ b/granule_ingester/tests/slicers/test_SliceFileByTilesDesired.py
@@ -0,0 +1,88 @@
+# import unittest
+# from collections import Set
+#
+# from netCDF4 import Dataset
+# from sdap.slicers.SliceFileByTilesDesired import SliceFileByTilesDesired
+#
+#
+# class TestSliceFileByTilesDesired(unittest.TestCase):
+#
+# def test_generate_slices(self):
+# netcdf_path = 'tests/granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc'
+# dataset = Dataset(netcdf_path)
+# dimension_specs = {value.name: value.size for key,
+# value in dataset.dimensions.items()}
+#
+# slicer = SliceFileByTilesDesired(tiles_desired=2,
+# desired_spatial_dimensions=['lat', 'lon'])
+# slices = slicer.generate_slices(dimension_specs)
+# expected_slices = ['lat:0:509,lon:0:1018',
+# 'lat:0:509,lon:1018:1440',
+# 'lat:509:720,lon:0:1018',
+# 'lat:509:720,lon:1018:1440']
+# self.assertEqual(slices, expected_slices)
+#
+# def test_generate_slices_with_time(self):
+# netcdf_path = 'tests/granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc'
+# dataset = Dataset(netcdf_path)
+# dimension_specs = {value.name: value.size for key,
+# value in dataset.dimensions.items()}
+#
+# slicer = SliceFileByTilesDesired(tiles_desired=2,
+# desired_spatial_dimensions=[
+# 'lat', 'lon'],
+# time_dimension=('time', 3))
+# slices = slicer.generate_slices(dimension_specs)
+# expected_slices = ['time:0:1,lat:0:509,lon:0:1018',
+# 'time:1:2,lat:0:509,lon:0:1018',
+#
+# 'time:0:1,lat:0:509,lon:1018:1440',
+# 'time:1:2,lat:0:509,lon:1018:1440',
+#
+# 'time:0:1,lat:509:720,lon:0:1018',
+# 'time:1:2,lat:509:720,lon:0:1018',
+#
+# 'time:0:1,lat:509:720,lon:1018:1440',
+# 'time:1:2,lat:509:720,lon:1018:1440']
+# self.assertEqual(slices, expected_slices)
+#
+# def test_calculate_step_size_perfect_split_2dim(self):
+# step_size = SliceFileByTilesDesired._calculate_step_size(1000, 100, 2)
+# self.assertAlmostEqual(step_size, 100.0)
+#
+# def test_calculate_step_size_perfect_split_3dim(self):
+# step_size = SliceFileByTilesDesired._calculate_step_size(1000, 100, 3)
+# self.assertAlmostEqual(step_size, 215.0)
+#
+# def test_generate_spatial_slices(self):
+# dimension_specs = {'lat': 8, 'lon': 8}
+# slicer = SliceFileByTilesDesired(tiles_desired=2,
+# desired_spatial_dimensions=dimension_specs)
+# boundary_slices = slicer._generate_spatial_slices(tiles_desired=4,
+# dimension_specs=dimension_specs)
+# expected_slices = [
+# 'lat:0:4,lon:0:4',
+# 'lat:0:4,lon:4:8',
+# 'lat:4:8,lon:0:4',
+# 'lat:4:8,lon:4:8'
+# ]
+# self.assertEqual(boundary_slices, expected_slices)
+#
+# def test_generate_temporal_slices(self):
+# slicer = SliceFileByTilesDesired(tiles_desired=2,
+# desired_spatial_dimensions=None)
+# time_slices = slicer._generate_temporal_slices(('time', 10))
+# expected_time_slices = ['time:0:1',
+# 'time:1:2',
+# 'time:2:3',
+# 'time:3:4',
+# 'time:4:5',
+# 'time:5:6',
+# 'time:6:7',
+# 'time:7:8',
+# 'time:8:9']
+# self.assertEqual(time_slices, expected_time_slices)
+#
+#
+# if __name__ == '__main__':
+# unittest.main()
diff --git a/granule_ingester/tests/slicers/test_TileSlicer.py b/granule_ingester/tests/slicers/test_TileSlicer.py
new file mode 100644
index 0000000..41c9b77
--- /dev/null
+++ b/granule_ingester/tests/slicers/test_TileSlicer.py
@@ -0,0 +1,68 @@
+import asyncio
+import os
+import unittest
+from sdap.slicers.TileSlicer import TileSlicer
+import xarray as xr
+
+
+class TestTileSlicer(unittest.TestCase):
+ class ToyTileSlicer(TileSlicer):
+ def _generate_slices(self, dimensions):
+ return [
+ 'time:0:1,lat:0:4,lon:0:4',
+ 'time:1:2,lat:0:4,lon:0:4',
+ 'time:2:3,lat:0:4,lon:0:4',
+
+ 'time:0:1,lat:0:4,lon:4:8',
+ 'time:1:2,lat:0:4,lon:4:8',
+ 'time:2:3,lat:0:4,lon:4:8',
+
+ 'time:0:1,lat:4:8,lon:0:4',
+ 'time:1:2,lat:4:8,lon:0:4',
+ 'time:2:3,lat:4:8,lon:0:4',
+
+ 'time:0:1,lat:4:8,lon:4:8',
+ 'time:1:2,lat:4:8,lon:4:8',
+ 'time:2:3,lat:4:8,lon:4:8'
+ ]
+
+ def test_generate_tiles(self):
+ relative_path = '../granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc'
+ file_path = os.path.join(os.path.dirname(__file__), relative_path)
+ with xr.open_dataset(file_path) as dataset:
+ slicer = TestTileSlicer.ToyTileSlicer().generate_tiles(dataset, file_path)
+
+ expected_slices = slicer._generate_slices(None)
+ self.assertEqual(file_path, slicer._granule_name)
+ self.assertEqual(expected_slices, slicer._tile_spec_list)
+
+ # def test_open_s3(self):
+ # s3_path = 's3://nexus-ingest/avhrr/198109-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc'
+ # slicer = TestTileSlicer.ToyTileSlicer(resource=s3_path)
+ #
+ # expected_slices = slicer._generate_slices(None)
+ # asyncio.run(slicer.open())
+ # self.assertIsNotNone(slicer.dataset)
+ # self.assertEqual(expected_slices, slicer._tile_spec_list)
+
+ def test_next(self):
+ relative_path = '../granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc'
+ file_path = os.path.join(os.path.dirname(__file__), relative_path)
+ with xr.open_dataset(file_path) as dataset:
+ slicer = TestTileSlicer.ToyTileSlicer().generate_tiles(dataset, file_path)
+ generated_tiles = list(slicer)
+
+ expected_slices = slicer._generate_slices(None)
+ self.assertListEqual(expected_slices, [tile.summary.section_spec for tile in generated_tiles])
+ for tile in generated_tiles:
+ self.assertEqual(file_path, tile.summary.granule)
+
+ # def test_download_s3_file(self):
+ # slicer = TestTileSlicer.ToyTileSlicer(resource=None)
+ #
+ # asyncio.run(slicer._download_s3_file(
+ # "s3://nexus-ingest/avhrr/198109-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/granule_ingester/tests/writers/__init__.py b/granule_ingester/tests/writers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/granule_ingester/tests/writers/test_SolrStore.py b/granule_ingester/tests/writers/test_SolrStore.py
new file mode 100644
index 0000000..cfded10
--- /dev/null
+++ b/granule_ingester/tests/writers/test_SolrStore.py
@@ -0,0 +1,54 @@
+import asyncio
+import unittest
+
+from nexusproto import DataTile_pb2 as nexusproto
+
+from sdap.writers import SolrStore
+
+
+class TestSolrStore(unittest.TestCase):
+
+ def test_build_solr_doc(self):
+ tile = nexusproto.NexusTile()
+ tile.summary.tile_id = 'test_id'
+ tile.summary.dataset_name = 'test_dataset'
+ tile.summary.dataset_uuid = 'test_dataset_id'
+ tile.summary.data_var_name = 'test_variable'
+ tile.summary.granule = 'test_granule_path'
+ tile.summary.section_spec = 'time:0:1,j:0:20,i:200:240'
+ tile.summary.bbox.lat_min = -180.1
+ tile.summary.bbox.lat_max = 180.2
+ tile.summary.bbox.lon_min = -90.5
+ tile.summary.bbox.lon_max = 90.0
+ tile.summary.stats.min = -10.0
+ tile.summary.stats.max = 25.5
+ tile.summary.stats.mean = 12.5
+ tile.summary.stats.count = 100
+ tile.summary.stats.min_time = 694224000
+ tile.summary.stats.max_time = 694310400
+
+ tile.tile.ecco_tile.depth = 10.5
+
+ metadata_store = SolrStore()
+ solr_doc = metadata_store._build_solr_doc(tile)
+
+ self.assertEqual('sea_surface_temp', solr_doc['table_s'])
+ self.assertEqual(
+ 'POLYGON((-90.500 -180.100, 90.000 -180.100, 90.000 180.200, -90.500 180.200, -90.500 -180.100))',
+ solr_doc['geo'])
+ self.assertEqual('test_id', solr_doc['id'])
+ self.assertEqual('test_dataset!test_id', solr_doc['solr_id_s'])
+ self.assertEqual('time:0:1,j:0:20,i:200:240', solr_doc['sectionSpec_s'])
+ self.assertEqual('test_granule_path', solr_doc['granule_s'])
+ self.assertEqual('test_variable', solr_doc['tile_var_name_s'])
+ self.assertAlmostEqual(-90.5, solr_doc['tile_min_lon'])
+ self.assertAlmostEqual(90.0, solr_doc['tile_max_lon'])
+ self.assertAlmostEqual(-180.1, solr_doc['tile_min_lat'])
+ self.assertAlmostEqual(180.2, solr_doc['tile_max_lat'])
+ self.assertEqual('1992-01-01T00:00:00Z', solr_doc['tile_min_time_dt'])
+ self.assertEqual('1992-01-02T00:00:00Z', solr_doc['tile_max_time_dt'])
+ self.assertAlmostEqual(-10.0, solr_doc['tile_min_val_d'])
+ self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d'])
+ self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d'])
+ self.assertEqual(100, solr_doc['tile_count_i'])
+ self.assertAlmostEqual(10.5, solr_doc['tile_depth'])