You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by jj...@apache.org on 2021/10/07 02:02:31 UTC
[incubator-sdap-ingester] branch dev updated: Julian datetime fix
for GPM data (#44)
This is an automated email from the ASF dual-hosted git repository.
jjacob pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new 9c01b17 Julian datetime fix for GPM data (#44)
9c01b17 is described below
commit 9c01b17a34160d501546c7ec1359b27f32011cf8
Author: Stepheny Perez <sk...@users.noreply.github.com>
AuthorDate: Wed Oct 6 19:01:40 2021 -0700
Julian datetime fix for GPM data (#44)
* Julian datetime fix for GPM data
* Fixed julian datetime bug for multi-var gridded data
---
.../GridMultiVariableReadingProcessor.py | 4 ++++
.../reading_processors/GridReadingProcessor.py | 3 +++
....MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4 | Bin 0 -> 30146530 bytes
.../test_GridMultiBandReadingProcessor.py | 21 +++++++++++++++++++++
.../test_GridReadingProcessor.py | 16 ++++++++++++++++
5 files changed, 44 insertions(+)
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
index 52142de..dc28c19 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
@@ -1,6 +1,7 @@
import logging
from typing import Dict
+import cftime
import numpy as np
import xarray as xr
from granule_ingester.processors.reading_processors.MultiBandUtils import MultiBandUtils
@@ -72,6 +73,9 @@ class GridMultiVariableReadingProcessor(TileReadingProcessor):
raise RuntimeError(
"Time slices must have length 1, but '{dim}' has length {dim_len}.".format(dim=self.time,
dim_len=time_slice_len))
+
+ if isinstance(ds[self.time][time_slice.start].item(), cftime.DatetimeJulian):
+ ds[self.time] = ds.indexes[self.time].to_datetimeindex()
new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9)
new_tile.latitude.CopyFrom(to_shaped_array(lat_subset))
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
index 7aca4bd..725ba35 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
@@ -1,5 +1,6 @@
from typing import Dict
+import cftime
import numpy as np
import xarray as xr
from nexusproto import DataTile_pb2 as nexusproto
@@ -46,6 +47,8 @@ class GridReadingProcessor(TileReadingProcessor):
raise RuntimeError(
"Time slices must have length 1, but '{dim}' has length {dim_len}.".format(dim=self.time,
dim_len=time_slice_len))
+ if isinstance(ds[self.time][time_slice.start].item(), cftime.DatetimeJulian):
+ ds[self.time] = ds.indexes[self.time].to_datetimeindex()
new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9)
new_tile.latitude.CopyFrom(to_shaped_array(lat_subset))
diff --git a/granule_ingester/tests/granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4 b/granule_ingester/tests/granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4
new file mode 100644
index 0000000..de859ed
Binary files /dev/null and b/granule_ingester/tests/granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4 differ
diff --git a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
index 3442c5a..607dd8b 100644
--- a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
@@ -165,6 +165,27 @@ class TestReadHLSData(unittest.TestCase):
self.assertRaises(RuntimeError, GridMultiVariableReadingProcessor, [], 'lat', 'lon', time='time')
return
+class TestReadGPMData(unittest.TestCase):
+ def test_generate_tile(self):
+ granule_path = path.join(path.dirname(__file__), '../granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4')
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'lat': slice(0, 30),
+ 'lon': slice(0, 30)
+ }
+
+ tile = nexusproto.NexusTile()
+
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ reading_processor = GridMultiVariableReadingProcessor(
+ ['HQprecipitation', 'HQprecipitation_cnt'],
+ 'lat',
+ 'lon',
+ time='time'
+ )
+ tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
+ assert tile.tile.grid_multi_variable_tile.time
+
if __name__ == '__main__':
unittest.main()
diff --git a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
index d4ffcfa..6cfa835 100644
--- a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
@@ -347,6 +347,22 @@ class TestReadHLSData(unittest.TestCase):
self.assertRaises(RuntimeError, GridReadingProcessor, [], 'lat', 'lon', time='time')
return
+class TestReadGPMData(unittest.TestCase):
+ def test_generate_tile(self):
+ granule_path = path.join(path.dirname(__file__), '../granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4')
+ dimensions_to_slices = {
+ 'time': slice(0, 1),
+ 'lat': slice(0, 30),
+ 'lon': slice(0, 30)
+ }
+
+ tile = nexusproto.NexusTile()
+
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ reading_processor = GridReadingProcessor(['HQprecipitation'], 'lat', 'lon', time='time')
+ tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
+ assert tile.tile.grid_tile.time
+
if __name__ == '__main__':
unittest.main()