You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/25 17:06:49 UTC
[incubator-sdap-ingester] branch dockerize_collection_ingester
updated (375c539 -> 1ea4e82)
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a change to branch dockerize_collection_ingester
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git.
discard 375c539 SDAP-237 Dockerize Collection Manager
add 08093ee SDAP-245: Move granule ingester code into this repo (#2)
add 1ea4e82 SDAP-237 Dockerize Collection Manager
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (375c539)
\
N -- N -- N refs/heads/dockerize_collection_ingester (1ea4e82)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.gitattributes | 1 -
granule_ingester/.gitignore | 9 +
granule_ingester/README.md | 34 +++
granule_ingester/conda-requirements.txt | 10 +
granule_ingester/docker/Dockerfile | 21 ++
granule_ingester/docker/entrypoint.sh | 10 +
granule_ingester/docker/install_nexusproto.sh | 20 ++
.../granule_ingester}/__init__.py | 0
.../granule_ingester/consumer/Consumer.py | 88 +++++++
.../granule_ingester/consumer/__init__.py | 1 +
.../granule_loaders/GranuleLoader.py | 71 ++++++
.../granule_ingester/granule_loaders/__init__.py | 1 +
.../granule_ingester/healthcheck/HealthCheck.py | 10 +-
.../granule_ingester/healthcheck/__init__.py | 1 +
granule_ingester/granule_ingester/main.py | 118 +++++++++
.../granule_ingester/pipeline/Modules.py | 15 ++
.../granule_ingester/pipeline/Pipeline.py | 158 ++++++++++++
.../granule_ingester/pipeline/__init__.py | 2 +
.../granule_ingester/processors/EmptyTileFilter.py | 42 ++++
.../granule_ingester/processors/GenerateTileId.py | 20 +-
.../granule_ingester/processors/TileProcessor.py | 11 +-
.../processors/TileSummarizingProcessor.py | 98 ++++++++
.../granule_ingester/processors/__init__.py | 5 +
.../granule_ingester/processors/kelvintocelsius.py | 19 +-
.../reading_processors/EccoReadingProcessor.py | 64 +++++
.../reading_processors/GridReadingProcessor.py | 53 +++++
.../reading_processors/SwathReadingProcessor.py | 47 ++++
.../reading_processors/TileReadingProcessor.py | 81 +++++++
.../TimeSeriesReadingProcessor.py | 83 +++++++
.../processors/reading_processors/__init__.py | 5 +
.../slicers/SliceFileByDimension.py | 55 +++++
.../slicers/SliceFileByStepSize.py | 55 +++++
.../slicers/SliceFileByTilesDesired.py | 68 ++++++
.../granule_ingester/slicers/TileSlicer.py | 56 +++++
.../granule_ingester/slicers/__init__.py | 2 +
.../granule_ingester/writers/CassandraStore.py | 78 ++++++
.../granule_ingester/writers/DataStore.py | 13 +
.../granule_ingester/writers/MetadataStore.py | 11 +
.../granule_ingester/writers/SolrStore.py | 152 ++++++++++++
.../granule_ingester/writers/__init__.py | 4 +
granule_ingester/requirements.txt | 3 +
granule_ingester/setup.py | 34 +++
.../k8s => granule_ingester/tests}/__init__.py | 0
.../tests/config_files/analysed_sst.yml | 12 +-
.../config_files/ingestion_config_testfile.yaml | 17 ++
...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | Bin 0 -> 1057327 bytes
granule_ingester/tests/granules/OBP_2017_01.nc | Bin 0 -> 2110135 bytes
granule_ingester/tests/granules/OBP_native_grid.nc | Bin 0 -> 1285094 bytes
.../SMAP_L2B_SSS_04892_20160101T005507_R13080.h5 | Bin 0 -> 18672352 bytes
granule_ingester/tests/granules/THETA_199201.nc | Bin 0 -> 4255957 bytes
granule_ingester/tests/granules/empty_mur.nc4 | Bin 0 -> 60937 bytes
.../tests/granules/not_empty_ascatb.nc4 | Bin 0 -> 78036 bytes
.../tests/granules/not_empty_avhrr.nc4 | Bin 0 -> 49511 bytes
granule_ingester/tests/granules/not_empty_ccmp.nc | Bin 0 -> 206870 bytes
granule_ingester/tests/granules/not_empty_mur.nc4 | Bin 0 -> 60907 bytes
granule_ingester/tests/granules/not_empty_smap.h5 | Bin 0 -> 3000192 bytes
granule_ingester/tests/granules/not_empty_wswm.nc | Bin 0 -> 1041568 bytes
.../tests/pipeline}/__init__.py | 0
granule_ingester/tests/pipeline/test_Pipeline.py | 104 ++++++++
.../tests/processors}/__init__.py | 0
.../tests/processors/test_GenerateTileId.py | 22 ++
.../tests/reading_processors}/__init__.py | 0
.../test_EccoReadingProcessor.py | 64 +++++
.../test_GridReadingProcessor.py | 265 +++++++++++++++++++++
.../test_SwathReadingProcessor.py | 74 ++++++
.../test_TileReadingProcessor.py | 29 +++
.../test_TimeSeriesReadingProcessor.py | 86 +++++++
.../tests/slicers}/__init__.py | 0
.../tests/slicers/test_SliceFileByDimension.py | 122 ++++++++++
.../tests/slicers/test_SliceFileByStepSize.py | 105 ++++++++
.../tests/slicers/test_SliceFileByTilesDesired.py | 88 +++++++
granule_ingester/tests/slicers/test_TileSlicer.py | 68 ++++++
.../tests/writers}/__init__.py | 0
granule_ingester/tests/writers/test_SolrStore.py | 54 +++++
74 files changed, 2718 insertions(+), 21 deletions(-)
delete mode 100644 .gitattributes
create mode 100644 granule_ingester/.gitignore
create mode 100644 granule_ingester/README.md
create mode 100644 granule_ingester/conda-requirements.txt
create mode 100644 granule_ingester/docker/Dockerfile
create mode 100644 granule_ingester/docker/entrypoint.sh
create mode 100755 granule_ingester/docker/install_nexusproto.sh
copy {config_operator/tests/k8s => granule_ingester/granule_ingester}/__init__.py (100%)
create mode 100644 granule_ingester/granule_ingester/consumer/Consumer.py
create mode 100644 granule_ingester/granule_ingester/consumer/__init__.py
create mode 100644 granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py
create mode 100644 granule_ingester/granule_ingester/granule_loaders/__init__.py
copy collection_manager/collection_manager/services/__init__.py => granule_ingester/granule_ingester/healthcheck/HealthCheck.py (83%)
create mode 100644 granule_ingester/granule_ingester/healthcheck/__init__.py
create mode 100644 granule_ingester/granule_ingester/main.py
create mode 100644 granule_ingester/granule_ingester/pipeline/Modules.py
create mode 100644 granule_ingester/granule_ingester/pipeline/Pipeline.py
create mode 100644 granule_ingester/granule_ingester/pipeline/__init__.py
create mode 100644 granule_ingester/granule_ingester/processors/EmptyTileFilter.py
copy collection_manager/collection_manager/services/__init__.py => granule_ingester/granule_ingester/processors/GenerateTileId.py (58%)
copy collection_manager/collection_manager/services/__init__.py => granule_ingester/granule_ingester/processors/TileProcessor.py (79%)
create mode 100644 granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/__init__.py
copy collection_manager/collection_manager/services/__init__.py => granule_ingester/granule_ingester/processors/kelvintocelsius.py (60%)
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/EccoReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/SwathReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/TileReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/TimeSeriesReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/__init__.py
create mode 100644 granule_ingester/granule_ingester/slicers/SliceFileByDimension.py
create mode 100644 granule_ingester/granule_ingester/slicers/SliceFileByStepSize.py
create mode 100644 granule_ingester/granule_ingester/slicers/SliceFileByTilesDesired.py
create mode 100644 granule_ingester/granule_ingester/slicers/TileSlicer.py
create mode 100644 granule_ingester/granule_ingester/slicers/__init__.py
create mode 100644 granule_ingester/granule_ingester/writers/CassandraStore.py
create mode 100644 granule_ingester/granule_ingester/writers/DataStore.py
create mode 100644 granule_ingester/granule_ingester/writers/MetadataStore.py
create mode 100644 granule_ingester/granule_ingester/writers/SolrStore.py
create mode 100644 granule_ingester/granule_ingester/writers/__init__.py
create mode 100644 granule_ingester/requirements.txt
create mode 100644 granule_ingester/setup.py
copy {config_operator/tests/k8s => granule_ingester/tests}/__init__.py (100%)
copy collection_manager/collection_manager/resources/dataset_config_template.yml => granule_ingester/tests/config_files/analysed_sst.yml (61%)
create mode 100644 granule_ingester/tests/config_files/ingestion_config_testfile.yaml
create mode 100644 granule_ingester/tests/granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc
create mode 100644 granule_ingester/tests/granules/OBP_2017_01.nc
create mode 100755 granule_ingester/tests/granules/OBP_native_grid.nc
create mode 100644 granule_ingester/tests/granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5
create mode 100644 granule_ingester/tests/granules/THETA_199201.nc
create mode 100644 granule_ingester/tests/granules/empty_mur.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_ascatb.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_avhrr.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_ccmp.nc
create mode 100644 granule_ingester/tests/granules/not_empty_mur.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_smap.h5
create mode 100644 granule_ingester/tests/granules/not_empty_wswm.nc
copy {config_operator/tests/k8s => granule_ingester/tests/pipeline}/__init__.py (100%)
create mode 100644 granule_ingester/tests/pipeline/test_Pipeline.py
copy {config_operator/tests/k8s => granule_ingester/tests/processors}/__init__.py (100%)
create mode 100644 granule_ingester/tests/processors/test_GenerateTileId.py
copy {config_operator/tests/k8s => granule_ingester/tests/reading_processors}/__init__.py (100%)
create mode 100644 granule_ingester/tests/reading_processors/test_EccoReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_SwathReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_TileReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_TimeSeriesReadingProcessor.py
copy {config_operator/tests/k8s => granule_ingester/tests/slicers}/__init__.py (100%)
create mode 100644 granule_ingester/tests/slicers/test_SliceFileByDimension.py
create mode 100644 granule_ingester/tests/slicers/test_SliceFileByStepSize.py
create mode 100644 granule_ingester/tests/slicers/test_SliceFileByTilesDesired.py
create mode 100644 granule_ingester/tests/slicers/test_TileSlicer.py
copy {config_operator/tests/k8s => granule_ingester/tests/writers}/__init__.py (100%)
create mode 100644 granule_ingester/tests/writers/test_SolrStore.py