You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2021/04/15 18:54:07 UTC
[incubator-sdap-ingester] branch ascending_latitudes updated:
update collection manager to activate ascending latitude processor
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch ascending_latitudes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/ascending_latitudes by this push:
new 0b07273 update collection manager to activate ascending latitude processor
0b07273 is described below
commit 0b07273859068af82ccceb608f697a754b3dc717
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Thu Apr 15 11:53:46 2021 -0700
update collection manager to activate ascending latitude processor
---
collection_manager/README.md | 6 ++++
.../services/CollectionProcessor.py | 40 ++++++++++++++--------
.../granule_ingester/pipeline/Modules.py | 6 ++--
3 files changed, 36 insertions(+), 16 deletions(-)
diff --git a/collection_manager/README.md b/collection_manager/README.md
index 90e72fa..67f1907 100644
--- a/collection_manager/README.md
+++ b/collection_manager/README.md
@@ -11,8 +11,14 @@ Manager service will publish a message to RabbitMQ to be picked up by the Granul
Python 3.7
+Use a conda environment for example:
+
+ $ conda create -n cmenv python=3.7
+ $ conda activate cmenv
+
## Building the service
From `incubator-sdap-ingester`, run:
+
$ cd common && python setup.py install
$ cd ../collection_manager python setup.py install
diff --git a/collection_manager/collection_manager/services/CollectionProcessor.py b/collection_manager/collection_manager/services/CollectionProcessor.py
index bab56fc..a328243 100644
--- a/collection_manager/collection_manager/services/CollectionProcessor.py
+++ b/collection_manager/collection_manager/services/CollectionProcessor.py
@@ -70,8 +70,33 @@ class CollectionProcessor:
self._history_manager_cache[dataset_id] = self._history_manager_builder.build(dataset_id=dataset_id)
return self._history_manager_cache[dataset_id]
+
+ @staticmethod
+ def _get_default_processors(collection: Collection):
+ processors = [
+ {
+ 'name': collection.projection,
+ **dict(collection.dimension_names),
+ },
+ {'name': 'emptyTileFilter'},
+ {'name': 'subtract180FromLongitude'}
+ ]
+
+ if collection.projection == 'Grid':
+ processors.append({'name': 'forceAscendingLatitude'})
+ processors.append({'name': 'kelvinToCelsius'})
+ processors.append({
+ 'name': 'tileSummary',
+ 'dataset_name': collection.dataset_id
+ })
+ processors.append({'name': 'generateTileId'})
+
+ return processors
+
+
@staticmethod
def _generate_ingestion_message(granule_path: str, collection: Collection) -> str:
+
config_dict = {
'granule': {
'resource': granule_path
@@ -80,20 +105,7 @@ class CollectionProcessor:
'name': 'sliceFileByStepSize',
'dimension_step_sizes': dict(collection.slices)
},
- 'processors': [
- {
- 'name': collection.projection,
- **dict(collection.dimension_names),
- },
- {'name': 'emptyTileFilter'},
- {'name': 'subtract180FromLongitude'},
- {'name': 'kelvinToCelsius'},
- {
- 'name': 'tileSummary',
- 'dataset_name': collection.dataset_id
- },
- {'name': 'generateTileId'}
- ]
+ 'processors': CollectionProcessor._get_default_processors(collection)
}
config_str = yaml.dump(config_dict)
logger.debug(f"Templated dataset config:\n{config_str}")
diff --git a/granule_ingester/granule_ingester/pipeline/Modules.py b/granule_ingester/granule_ingester/pipeline/Modules.py
index 5db706b..d1511a6 100644
--- a/granule_ingester/granule_ingester/pipeline/Modules.py
+++ b/granule_ingester/granule_ingester/pipeline/Modules.py
@@ -2,7 +2,8 @@ from granule_ingester.processors import (GenerateTileId,
TileSummarizingProcessor,
EmptyTileFilter,
KelvinToCelsius,
- Subtract180FromLongitude)
+ Subtract180FromLongitude,
+ ForceAscendingLatitude)
from granule_ingester.processors.reading_processors import (EccoReadingProcessor,
GridReadingProcessor,
SwathReadingProcessor,
@@ -21,5 +22,6 @@ modules = {
"tileSummary": TileSummarizingProcessor,
"emptyTileFilter": EmptyTileFilter,
"kelvinToCelsius": KelvinToCelsius,
- "subtract180FromLongitude": Subtract180FromLongitude
+ "subtract180FromLongitude": Subtract180FromLongitude,
+ "forceAscendingLatitude": ForceAscendingLatitude
}