You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2021/05/14 20:58:38 UTC
[incubator-sdap-ingester] branch master updated (1fedc94 -> 58eccd0)
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git.
from 1fedc94 correct typo in README
add 276e0e8 integrate data directory to kubernetes pods
add 164f462 have dev suffix for the version
add a333355 more log when getting colletion config file
add a1b4102 manage data volume in granule ingester pod
add 51e155b complete management of data volume for granule ingester
add 97811ad temporarilly for test, recreate data volume in granule ingester jobs
add 4315361 add log (was print before, did not show when run in kubernete)
add 5135e1f end of day, correct bad log syntax
add 224a8a5 put back the print which sends the k8s job template
add ce670c9 correct the granule job template
add 6eb572c define data volume as shared k8s volume used by both collection and granule ingester
add 4847cfa usage shared volume claim for data storage
add 51d7944 integrate solr history manager, latest ingested data management remains to be done
add 5fd2eaf replace history configuration with solr
add f51ecb9 correct bugs after test on solr history
add 3b78099 add forward-processing capabilities on solr history
add ec5240f make solr schema update work
add 1df86b0 made solr history instanciation more robust by catching excepions
add 1660d64 made solr history instanciation more robust by catching excepions
add e0c0b70 change default conf to solr pods access
add 5f18529 correction on buffering history on files, started package for order/config service
add d707d7d started implementation of order service
add f4fc5b1 start git synchornization function for configuration
add 8e2a167 synchronize ingestion orders with git repository
add b2a409c simplified obsolete unit test
add 8894a59 remove .idea from repository
add cd15dd4 Code restructuring, add rabbitmq support
add 423ad00 moved files
add f7f0ce7 a lot of cleanup and restructuring
add 01f1773 fix paths
add 57e6260 manage ingestion order list configured on a git repository
add ceae47c Publish messages to RabbitMQ, plus lots of refactoring
add 090576e clean more unused file, some information on how to launch in readme, to be reviewed
add 84218f0 update the README with rabbitmq prerequisite command
add c07eb3d update the docker command to launch rabbitmq for development prerequisite
add 1635e56 Code restructuring, add rabbitmq support
add 62b5c1a Add scheduler
add e4413a5 cleanup merge
add 06b43f7 cleanup
add 4c6aa4e removed flask
add f02ae2f remove ingestion order references
add ce0d737 Merge branch 'master' of https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager into dev
add 70b1455 initiate a configMap object with unit test (with bug)
add 11cccba add config_operator working on local dir or remote git repo
add 5dcb8af make the config-operator robust to syntactically wrong files, by default synchornization is deactivated
add e936306 start to implement config-operator containers
add 825c93f move files in specific subdirectory, start k8s deployment, does not work
add a2843b2 integrates with kubernetes
add 10ba61c run config operator as a k8s operator
add 17bd28c apply change requests by pull request #3 review
add a61fc11 some more cleaining in unit test and README
add 969adf6 allow multiple collections to use the same directory
add 43cced9 Fix bad merge
add 8b6f5e5 SDAP-234: Add more test coverage. (#1)
add 5850e16 SDAP-254, SDAP-255, SDAP-256: Fix bug where ingestion history is not saved, fix bug where messages published to RabbitMQ are incorrect, fix bug where bad collection config file crashes app (#3)
add 08093ee SDAP-245: Move granule ingester code into this repo (#2)
add 12d9101 SDAP-237 Dockerize Collection Manager (#4)
add 09f53d5 SDAP-259: The Collection Manager now reloads the Collections Config file on an interval instead of watching for filesystem events (#5)
add ecd5ecc SDAP-247: config-operator unit tests and support for git username/token (#6)
add 6759457 SDAP-269: Switch to using aio-pika in collection-manager to maintain an asynchronous connection to RabbitMQ (#7)
add 6f48d8a SDAP-270: Fix bad solr endpoints in solr history manager (#8)
add 426a293 Fix syntax error (#9)
add e309727 add a on.update kopf event for crd updates
add c599caf SDAP-271 Cassandra authentication support (#11)
add 68110d0 SDAP-273: Configure max threads in Granule Ingester (#13)
add d94c89f SDAP-266: add README note on synchronization of configmap with local path (#14)
add fe6a1c5 SDAP-272: Support connecting to Solr through Zk in Granule Ingester (#12)
add 4d816f6 SDAP-277: Improved error handling in Granule Ingester (#15)
add 1e492cf SDAP-279: Collection Manager should poll filesystem for file updates (#17)
add bc75c25 SDAP-280: Collection Manager to talk to Ingestion History asynchronously (#16)
add 6097580 SDAP-282, SDAP-284: Support configuring dimension names and projection during ingestion, support hierarchical directory structures (#19)
add 2c4c8bb SDAP-291: Fix netcdf parsing error by freezing pandas at 1.0.4 (#20)
add 7ad004c SDAP-291: Fix netcdf parsing error by freezing pandas at 1.0.4 (#21)
add 72e0f32 SDAP-286: Add processor module to Granule Ingester to properly handle longitudes between 180-360deg (#23)
add 999b432 SDAP-288: S3 ingestion support (#24)
add f66c7e1 SDAP-293: Automatic Kelvin-to-Celsius conversion (#22)
add f49a7ec SDAP-295: Include dataset name hash in tile ID generation (#25)
add 575f25f SDAP-297: Update Collections Config docs to match latest schema (#26)
add 7e04251 SDAP-300: Fix bug that prevented collection manager from seeing files in a directory when the path had no glob-style wildcard character. Also add back support for scanning dirs recursively. (#27)
add 9f84b14 KelvinToCelsius converter can now parse 'units' or 'Units' field and exits gracefully if none exists (#28)
add dc9fb67 SDAP-302: Fix bug where the Collection Manager would crash if the Collections Config was updated while Collection Manager was actively scanning S3 directories (#29)
add 40cc47a feat: Ascending latitudes from dev (#31)
add aeb1481 add default case when collection path is a regex (#32)
add 472ab15 SDAP-310: Added standard_name to nexus tile and solr doc (#33)
new 58eccd0 Merge pull request #34 from apache/dev
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitattributes | 1 -
.github/workflows/github-dev-release.yml | 6 +
.gitignore | 9 +-
.idea/.gitignore | 2 -
.idea/inspectionProfiles/profiles_settings.xml | 6 -
.idea/misc.xml | 7 -
.idea/modules.xml | 8 -
.idea/sdap_ingest_manager.iml | 10 -
.idea/vcs.xml | 6 -
README.md | 199 --------
bin/run_collections | 62 ---
bin/run_granules | 5 -
bin/run_single_collection | 96 ----
collection_manager/README.md | 121 +++++
collection_manager/collection_manager/__init__.py | 1 +
.../collection_manager/entities/Collection.py | 74 +++
.../collection_manager/entities/__init__.py | 2 +
.../entities/exceptions/Exceptions.py | 30 ++
.../entities/exceptions/__init__.py | 6 +
collection_manager/collection_manager/main.py | 100 ++++
.../services/CollectionProcessor.py | 112 ++++
.../services/CollectionWatcher.py | 222 ++++++++
.../services/MessagePublisher.py | 40 ++
.../collection_manager/services/S3Observer.py | 147 ++++++
.../collection_manager/services}/__init__.py | 6 +
.../history_manager/FileIngestionHistory.py | 95 ++--
.../services/history_manager/IngestionHistory.py | 139 +++++
.../history_manager/SolrIngestionHistory.py | 153 ++++++
.../services/history_manager/__init__.py | 4 +
collection_manager/docker/Dockerfile | 20 +
collection_manager/docker/entrypoint.sh | 12 +
.../migration}/__init__.py | 0
.../migration/history_migration.py.bkp | 33 ++
collection_manager/requirements.txt | 9 +
collection_manager/setup.py | 35 ++
.../test => collection_manager/tests}/__init__.py | 0
.../tests/entities}/__init__.py | 0
.../tests/entities/test_Collection.py | 177 +++++++
collection_manager/tests/resources/collections.yml | 47 ++
.../tests/resources/collections_alternate.yml | 45 ++
.../tests/resources/collections_bad_schema.yml | 17 +
.../tests/resources/collections_bad_syntax.yml | 17 +
.../resources}/data/avhrr-oi-analysed-sst.csv | 0
...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | 0
...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | 0
.../tests/services}/__init__.py | 0
.../tests/services/history_manager}/__init__.py | 0
.../history_manager/test_FileIngestionHistory.py | 64 +++
.../history_manager/test_SolrIngestionHistory.py | 17 +-
.../tests/services/test_CollectionProcessor.py | 201 ++++++++
.../tests/services/test_CollectionWatcher.py | 255 ++++++++++
.../tests/services/test_S3Observer.py | 8 +
.../test => common/common}/__init__.py | 0
common/common/async_test_utils/AsyncTestUtils.py | 28 +
common/common/async_test_utils/__init__.py | 1 +
common/common/async_utils/AsyncUtils.py | 11 +
common/common/async_utils/__init__.py | 1 +
common/setup.py | 21 +
config_operator/README.md | 68 +++
config_operator/config_operator/__init__.py | 1 +
.../config_source/LocalDirConfig.py | 80 +++
.../config_source/RemoteGitConfig.py | 82 +++
.../config_operator/config_source/__init__.py | 2 +
.../config_operator/config_source/exceptions.py | 4 +
.../config_operator/k8s/K8sConfigMap.py | 111 ++++
config_operator/config_operator/k8s/__init__.py | 1 +
config_operator/config_operator/main.py | 69 +++
config_operator/containers/docker/Dockerfile | 11 +
.../containers/k8s/config-operator-crd.yml | 79 +++
config_operator/containers/k8s/git-repo-test.yml | 9 +
config_operator/requirements.txt | 5 +
config_operator/setup.py | 38 ++
.../test => config_operator/tests}/__init__.py | 0
.../tests/config_source}/__init__.py | 0
.../tests/config_source/test_LocalDirConfig.py | 72 +++
.../tests/config_source/test_RemoteGitConfig.py | 49 ++
.../test => config_operator/tests/k8s}/__init__.py | 0
config_operator/tests/k8s/test_K8sConfigMap.py | 74 +++
config_operator/tests/resources/collections.yml | 18 +
.../resources/localDirBadTest/collections.yml | 2 +
.../tests/resources/localDirTest/.hidden_file.txt | 1 +
.../tests/resources/localDirTest/README.md | 1 +
.../tests/resources/localDirTest/collections.yml | 1 +
containers/docker/Dockerfile | 11 -
containers/kubernetes/job.yml | 25 -
containers/kubernetes/sdap_ingester_config.yml | 38 --
granule_ingester/README.md | 35 ++
granule_ingester/conda-requirements.txt | 10 +
granule_ingester/docker/Dockerfile | 25 +
granule_ingester/docker/entrypoint.sh | 14 +
granule_ingester/docker/install_nexusproto.sh | 20 +
granule_ingester/granule_ingester/README.md | 11 +
.../granule_ingester}/__init__.py | 0
.../granule_ingester/consumer/MessageConsumer.py | 113 +++++
.../granule_ingester/consumer/__init__.py | 1 +
.../granule_ingester/exceptions/Exceptions.py | 46 ++
.../granule_ingester/exceptions/__init__.py | 8 +
.../granule_loaders/GranuleLoader.py | 78 +++
.../granule_ingester/granule_loaders/__init__.py | 1 +
.../granule_ingester/healthcheck/HealthCheck.py | 8 +
.../granule_ingester/healthcheck/__init__.py | 1 +
granule_ingester/granule_ingester/main.py | 152 ++++++
.../granule_ingester/pipeline/Modules.py | 27 +
.../granule_ingester/pipeline/Pipeline.py | 209 ++++++++
.../granule_ingester/pipeline/__init__.py | 2 +
.../granule_ingester/processors/EmptyTileFilter.py | 42 ++
.../processors/ForceAscendingLatitude.py | 48 ++
.../granule_ingester/processors/GenerateTileId.py | 27 +-
.../processors/Subtract180FromLongitude.py | 38 ++
.../granule_ingester/processors/TileProcessor.py | 41 ++
.../processors/TileSummarizingProcessor.py | 102 ++++
.../granule_ingester/processors/__init__.py | 7 +
.../granule_ingester/processors/kelvintocelsius.py | 43 ++
.../reading_processors/EccoReadingProcessor.py | 64 +++
.../reading_processors/GridReadingProcessor.py | 53 ++
.../reading_processors/SwathReadingProcessor.py | 47 ++
.../reading_processors/TileReadingProcessor.py | 79 +++
.../TimeSeriesReadingProcessor.py | 83 +++
.../processors/reading_processors/__init__.py | 5 +
.../slicers/SliceFileByDimension.py | 55 ++
.../slicers/SliceFileByStepSize.py | 55 ++
.../slicers/SliceFileByTilesDesired.py | 68 +++
.../granule_ingester/slicers/TileSlicer.py | 56 ++
.../granule_ingester/slicers/__init__.py | 2 +
.../granule_ingester/writers/CassandraStore.py | 97 ++++
.../granule_ingester/writers/DataStore.py | 13 +
.../granule_ingester/writers/MetadataStore.py | 11 +
.../granule_ingester/writers/SolrStore.py | 170 +++++++
.../granule_ingester/writers/__init__.py | 4 +
granule_ingester/requirements.txt | 7 +
granule_ingester/setup.py | 34 ++
.../test => granule_ingester/tests}/__init__.py | 0
.../tests/config_files/analysed_sst.yml | 16 +
.../config_files/ingestion_config_testfile.yaml | 17 +
...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | Bin 0 -> 1057327 bytes
.../granules/HLS.S30.T11SPC.2020001.v1.4.hdf.nc | Bin 0 -> 167801 bytes
granule_ingester/tests/granules/OBP_2017_01.nc | Bin 0 -> 2110135 bytes
granule_ingester/tests/granules/OBP_native_grid.nc | Bin 0 -> 1285094 bytes
.../SMAP_L2B_SSS_04892_20160101T005507_R13080.h5 | Bin 0 -> 18672352 bytes
granule_ingester/tests/granules/THETA_199201.nc | Bin 0 -> 4255957 bytes
granule_ingester/tests/granules/empty_mur.nc4 | Bin 0 -> 60937 bytes
.../tests/granules/not_empty_ascatb.nc4 | Bin 0 -> 78036 bytes
.../tests/granules/not_empty_avhrr.nc4 | Bin 0 -> 49511 bytes
granule_ingester/tests/granules/not_empty_ccmp.nc | Bin 0 -> 206870 bytes
granule_ingester/tests/granules/not_empty_mur.nc4 | Bin 0 -> 60907 bytes
granule_ingester/tests/granules/not_empty_smap.h5 | Bin 0 -> 3000192 bytes
granule_ingester/tests/granules/not_empty_wswm.nc | Bin 0 -> 1041568 bytes
.../tests/pipeline}/__init__.py | 0
granule_ingester/tests/pipeline/test_Pipeline.py | 106 ++++
.../tests/processors}/__init__.py | 0
.../processors/test_ForceAscendingLatitude.py | 58 +++
.../tests/processors/test_GenerateTileId.py | 22 +
.../tests/reading_processors}/__init__.py | 0
.../test_EccoReadingProcessor.py | 64 +++
.../test_GridReadingProcessor.py | 265 ++++++++++
.../test_SwathReadingProcessor.py | 74 +++
.../test_TileReadingProcessor.py | 29 ++
.../test_TileSummarizingProcessor.py | 44 ++
.../test_TimeSeriesReadingProcessor.py | 86 ++++
.../tests/slicers}/__init__.py | 0
.../tests/slicers/test_SliceFileByDimension.py | 122 +++++
.../tests/slicers/test_SliceFileByStepSize.py | 105 ++++
.../tests/slicers/test_SliceFileByTilesDesired.py | 88 ++++
granule_ingester/tests/slicers/test_TileSlicer.py | 68 +++
.../tests/writers}/__init__.py | 0
granule_ingester/tests/writers/test_SolrStore.py | 69 +++
requirements.txt | 6 -
sdap_ingest_manager/__init__.py | 1 -
.../collections_ingester/MANIFEST.in | 4 -
.../collections_ingester/collection_ingestion.py | 220 --------
.../google_spreadsheet_collection_config.py | 69 ---
.../collections_ingester/nfs_mount_parse.py | 55 --
.../resources/config/collections.yml.example | 9 -
.../resources/config/credentials.json.template | 10 -
.../resources/config/sdap_ingest_manager.ini | 15 -
.../config/sdap_ingest_manager.ini.default | 50 --
.../resources/dataset_config_template.yml | 45 --
.../collections_ingester/test/data/collections.yml | 9 -
.../test/data/dataset_config_file_ok.yml | 45 --
.../test/unit/collection_ingestion_test.py | 151 ------
.../test/unit/nfs_mount_parse_test.py | 29 --
.../collections_ingester/test/unit/util_test.py | 17 -
.../test/unit/yaml_file_collection_config_test.py | 29 --
.../test/validation/__init__.py | 13 -
.../test/validation/validation_test.py | 136 -----
sdap_ingest_manager/collections_ingester/util.py | 26 -
.../yaml_file_collection_config.py | 32 --
sdap_ingest_manager/granule_ingester/MANIFEST.in | 1 -
sdap_ingest_manager/granule_ingester/Readme.md | 138 -----
sdap_ingest_manager/granule_ingester/__init__.py | 4 -
.../resources/connection-config.yml | 14 -
.../resources/job-deployment-template.yml | 47 --
sdap_ingest_manager/granule_ingester/runjobs.py | 563 ---------------------
sdap_ingest_manager/history_manager/__init__.py | 3 -
.../history_manager/datasetingestionhistorysolr.py | 114 -----
.../test/datasetingestionhistoryfile_test.py | 64 ---
sdap_ingest_manager/history_manager/util.py | 17 -
setup.cfg | 5 -
setup.py | 76 ---
199 files changed, 6230 insertions(+), 2572 deletions(-)
delete mode 100644 .gitattributes
delete mode 100644 .idea/.gitignore
delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml
delete mode 100644 .idea/misc.xml
delete mode 100644 .idea/modules.xml
delete mode 100644 .idea/sdap_ingest_manager.iml
delete mode 100644 .idea/vcs.xml
delete mode 100644 README.md
delete mode 100644 bin/run_collections
delete mode 100644 bin/run_granules
delete mode 100755 bin/run_single_collection
create mode 100644 collection_manager/README.md
create mode 100644 collection_manager/collection_manager/__init__.py
create mode 100644 collection_manager/collection_manager/entities/Collection.py
create mode 100644 collection_manager/collection_manager/entities/__init__.py
create mode 100644 collection_manager/collection_manager/entities/exceptions/Exceptions.py
create mode 100644 collection_manager/collection_manager/entities/exceptions/__init__.py
create mode 100644 collection_manager/collection_manager/main.py
create mode 100644 collection_manager/collection_manager/services/CollectionProcessor.py
create mode 100644 collection_manager/collection_manager/services/CollectionWatcher.py
create mode 100644 collection_manager/collection_manager/services/MessagePublisher.py
create mode 100644 collection_manager/collection_manager/services/S3Observer.py
rename {sdap_ingest_manager/collections_ingester/test/unit => collection_manager/collection_manager/services}/__init__.py (77%)
rename sdap_ingest_manager/history_manager/datasetingestionhistoryfile.py => collection_manager/collection_manager/services/history_manager/FileIngestionHistory.py (54%)
create mode 100644 collection_manager/collection_manager/services/history_manager/IngestionHistory.py
create mode 100644 collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py
create mode 100644 collection_manager/collection_manager/services/history_manager/__init__.py
create mode 100644 collection_manager/docker/Dockerfile
create mode 100644 collection_manager/docker/entrypoint.sh
copy {sdap_ingest_manager/history_manager/test => collection_manager/migration}/__init__.py (100%)
create mode 100644 collection_manager/migration/history_migration.py.bkp
create mode 100644 collection_manager/requirements.txt
create mode 100644 collection_manager/setup.py
copy {sdap_ingest_manager/history_manager/test => collection_manager/tests}/__init__.py (100%)
copy {sdap_ingest_manager/history_manager/test => collection_manager/tests/entities}/__init__.py (100%)
create mode 100644 collection_manager/tests/entities/test_Collection.py
create mode 100644 collection_manager/tests/resources/collections.yml
create mode 100644 collection_manager/tests/resources/collections_alternate.yml
create mode 100644 collection_manager/tests/resources/collections_bad_schema.yml
create mode 100644 collection_manager/tests/resources/collections_bad_syntax.yml
rename {sdap_ingest_manager/collections_ingester/test => collection_manager/tests/resources}/data/avhrr-oi-analysed-sst.csv (100%)
rename {sdap_ingest_manager/collections_ingester/test => collection_manager/tests/resources}/data/avhrr_oi/20151101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc (100%)
rename {sdap_ingest_manager/collections_ingester/test => collection_manager/tests/resources}/data/avhrr_oi/20151102120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc (100%)
copy {sdap_ingest_manager/history_manager/test => collection_manager/tests/services}/__init__.py (100%)
copy {sdap_ingest_manager/history_manager/test => collection_manager/tests/services/history_manager}/__init__.py (100%)
create mode 100644 collection_manager/tests/services/history_manager/test_FileIngestionHistory.py
rename sdap_ingest_manager/history_manager/test/datasetingestionhistorysolr_test.py => collection_manager/tests/services/history_manager/test_SolrIngestionHistory.py (59%)
create mode 100644 collection_manager/tests/services/test_CollectionProcessor.py
create mode 100644 collection_manager/tests/services/test_CollectionWatcher.py
create mode 100644 collection_manager/tests/services/test_S3Observer.py
copy {sdap_ingest_manager/history_manager/test => common/common}/__init__.py (100%)
create mode 100644 common/common/async_test_utils/AsyncTestUtils.py
create mode 100644 common/common/async_test_utils/__init__.py
create mode 100644 common/common/async_utils/AsyncUtils.py
create mode 100644 common/common/async_utils/__init__.py
create mode 100644 common/setup.py
create mode 100644 config_operator/README.md
create mode 100644 config_operator/config_operator/__init__.py
create mode 100644 config_operator/config_operator/config_source/LocalDirConfig.py
create mode 100644 config_operator/config_operator/config_source/RemoteGitConfig.py
create mode 100644 config_operator/config_operator/config_source/__init__.py
create mode 100644 config_operator/config_operator/config_source/exceptions.py
create mode 100644 config_operator/config_operator/k8s/K8sConfigMap.py
create mode 100644 config_operator/config_operator/k8s/__init__.py
create mode 100644 config_operator/config_operator/main.py
create mode 100644 config_operator/containers/docker/Dockerfile
create mode 100644 config_operator/containers/k8s/config-operator-crd.yml
create mode 100644 config_operator/containers/k8s/git-repo-test.yml
create mode 100644 config_operator/requirements.txt
create mode 100644 config_operator/setup.py
copy {sdap_ingest_manager/history_manager/test => config_operator/tests}/__init__.py (100%)
copy {sdap_ingest_manager/history_manager/test => config_operator/tests/config_source}/__init__.py (100%)
create mode 100644 config_operator/tests/config_source/test_LocalDirConfig.py
create mode 100644 config_operator/tests/config_source/test_RemoteGitConfig.py
copy {sdap_ingest_manager/history_manager/test => config_operator/tests/k8s}/__init__.py (100%)
create mode 100644 config_operator/tests/k8s/test_K8sConfigMap.py
create mode 100644 config_operator/tests/resources/collections.yml
create mode 100644 config_operator/tests/resources/localDirBadTest/collections.yml
create mode 100644 config_operator/tests/resources/localDirTest/.hidden_file.txt
create mode 100644 config_operator/tests/resources/localDirTest/README.md
create mode 100644 config_operator/tests/resources/localDirTest/collections.yml
delete mode 100644 containers/docker/Dockerfile
delete mode 100644 containers/kubernetes/job.yml
delete mode 100644 containers/kubernetes/sdap_ingester_config.yml
create mode 100644 granule_ingester/README.md
create mode 100644 granule_ingester/conda-requirements.txt
create mode 100644 granule_ingester/docker/Dockerfile
create mode 100644 granule_ingester/docker/entrypoint.sh
create mode 100755 granule_ingester/docker/install_nexusproto.sh
create mode 100644 granule_ingester/granule_ingester/README.md
copy {sdap_ingest_manager/history_manager/test => granule_ingester/granule_ingester}/__init__.py (100%)
create mode 100644 granule_ingester/granule_ingester/consumer/MessageConsumer.py
create mode 100644 granule_ingester/granule_ingester/consumer/__init__.py
create mode 100644 granule_ingester/granule_ingester/exceptions/Exceptions.py
create mode 100644 granule_ingester/granule_ingester/exceptions/__init__.py
create mode 100644 granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py
create mode 100644 granule_ingester/granule_ingester/granule_loaders/__init__.py
rename sdap_ingest_manager/collections_ingester/test/__init__.py => granule_ingester/granule_ingester/healthcheck/HealthCheck.py (81%)
create mode 100644 granule_ingester/granule_ingester/healthcheck/__init__.py
create mode 100644 granule_ingester/granule_ingester/main.py
create mode 100644 granule_ingester/granule_ingester/pipeline/Modules.py
create mode 100644 granule_ingester/granule_ingester/pipeline/Pipeline.py
create mode 100644 granule_ingester/granule_ingester/pipeline/__init__.py
create mode 100644 granule_ingester/granule_ingester/processors/EmptyTileFilter.py
create mode 100644 granule_ingester/granule_ingester/processors/ForceAscendingLatitude.py
rename sdap_ingest_manager/collections_ingester/__init__.py => granule_ingester/granule_ingester/processors/GenerateTileId.py (55%)
create mode 100644 granule_ingester/granule_ingester/processors/Subtract180FromLongitude.py
create mode 100644 granule_ingester/granule_ingester/processors/TileProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/__init__.py
create mode 100644 granule_ingester/granule_ingester/processors/kelvintocelsius.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/EccoReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/SwathReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/TileReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/TimeSeriesReadingProcessor.py
create mode 100644 granule_ingester/granule_ingester/processors/reading_processors/__init__.py
create mode 100644 granule_ingester/granule_ingester/slicers/SliceFileByDimension.py
create mode 100644 granule_ingester/granule_ingester/slicers/SliceFileByStepSize.py
create mode 100644 granule_ingester/granule_ingester/slicers/SliceFileByTilesDesired.py
create mode 100644 granule_ingester/granule_ingester/slicers/TileSlicer.py
create mode 100644 granule_ingester/granule_ingester/slicers/__init__.py
create mode 100644 granule_ingester/granule_ingester/writers/CassandraStore.py
create mode 100644 granule_ingester/granule_ingester/writers/DataStore.py
create mode 100644 granule_ingester/granule_ingester/writers/MetadataStore.py
create mode 100644 granule_ingester/granule_ingester/writers/SolrStore.py
create mode 100644 granule_ingester/granule_ingester/writers/__init__.py
create mode 100644 granule_ingester/requirements.txt
create mode 100644 granule_ingester/setup.py
copy {sdap_ingest_manager/history_manager/test => granule_ingester/tests}/__init__.py (100%)
create mode 100644 granule_ingester/tests/config_files/analysed_sst.yml
create mode 100644 granule_ingester/tests/config_files/ingestion_config_testfile.yaml
create mode 100644 granule_ingester/tests/granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc
create mode 100644 granule_ingester/tests/granules/HLS.S30.T11SPC.2020001.v1.4.hdf.nc
create mode 100644 granule_ingester/tests/granules/OBP_2017_01.nc
create mode 100755 granule_ingester/tests/granules/OBP_native_grid.nc
create mode 100644 granule_ingester/tests/granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5
create mode 100644 granule_ingester/tests/granules/THETA_199201.nc
create mode 100644 granule_ingester/tests/granules/empty_mur.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_ascatb.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_avhrr.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_ccmp.nc
create mode 100644 granule_ingester/tests/granules/not_empty_mur.nc4
create mode 100644 granule_ingester/tests/granules/not_empty_smap.h5
create mode 100644 granule_ingester/tests/granules/not_empty_wswm.nc
copy {sdap_ingest_manager/history_manager/test => granule_ingester/tests/pipeline}/__init__.py (100%)
create mode 100644 granule_ingester/tests/pipeline/test_Pipeline.py
copy {sdap_ingest_manager/history_manager/test => granule_ingester/tests/processors}/__init__.py (100%)
create mode 100644 granule_ingester/tests/processors/test_ForceAscendingLatitude.py
create mode 100644 granule_ingester/tests/processors/test_GenerateTileId.py
copy {sdap_ingest_manager/history_manager/test => granule_ingester/tests/reading_processors}/__init__.py (100%)
create mode 100644 granule_ingester/tests/reading_processors/test_EccoReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_SwathReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_TileReadingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
create mode 100644 granule_ingester/tests/reading_processors/test_TimeSeriesReadingProcessor.py
copy {sdap_ingest_manager/history_manager/test => granule_ingester/tests/slicers}/__init__.py (100%)
create mode 100644 granule_ingester/tests/slicers/test_SliceFileByDimension.py
create mode 100644 granule_ingester/tests/slicers/test_SliceFileByStepSize.py
create mode 100644 granule_ingester/tests/slicers/test_SliceFileByTilesDesired.py
create mode 100644 granule_ingester/tests/slicers/test_TileSlicer.py
rename {sdap_ingest_manager/history_manager/test => granule_ingester/tests/writers}/__init__.py (100%)
create mode 100644 granule_ingester/tests/writers/test_SolrStore.py
delete mode 100644 requirements.txt
delete mode 100644 sdap_ingest_manager/__init__.py
delete mode 100644 sdap_ingest_manager/collections_ingester/MANIFEST.in
delete mode 100644 sdap_ingest_manager/collections_ingester/collection_ingestion.py
delete mode 100644 sdap_ingest_manager/collections_ingester/google_spreadsheet_collection_config.py
delete mode 100644 sdap_ingest_manager/collections_ingester/nfs_mount_parse.py
delete mode 100644 sdap_ingest_manager/collections_ingester/resources/config/collections.yml.example
delete mode 100644 sdap_ingest_manager/collections_ingester/resources/config/credentials.json.template
delete mode 100644 sdap_ingest_manager/collections_ingester/resources/config/sdap_ingest_manager.ini
delete mode 100644 sdap_ingest_manager/collections_ingester/resources/config/sdap_ingest_manager.ini.default
delete mode 100644 sdap_ingest_manager/collections_ingester/resources/dataset_config_template.yml
delete mode 100644 sdap_ingest_manager/collections_ingester/test/data/collections.yml
delete mode 100644 sdap_ingest_manager/collections_ingester/test/data/dataset_config_file_ok.yml
delete mode 100644 sdap_ingest_manager/collections_ingester/test/unit/collection_ingestion_test.py
delete mode 100644 sdap_ingest_manager/collections_ingester/test/unit/nfs_mount_parse_test.py
delete mode 100644 sdap_ingest_manager/collections_ingester/test/unit/util_test.py
delete mode 100644 sdap_ingest_manager/collections_ingester/test/unit/yaml_file_collection_config_test.py
delete mode 100644 sdap_ingest_manager/collections_ingester/test/validation/__init__.py
delete mode 100644 sdap_ingest_manager/collections_ingester/test/validation/validation_test.py
delete mode 100644 sdap_ingest_manager/collections_ingester/util.py
delete mode 100644 sdap_ingest_manager/collections_ingester/yaml_file_collection_config.py
delete mode 100644 sdap_ingest_manager/granule_ingester/MANIFEST.in
delete mode 100644 sdap_ingest_manager/granule_ingester/Readme.md
delete mode 100644 sdap_ingest_manager/granule_ingester/__init__.py
delete mode 100644 sdap_ingest_manager/granule_ingester/resources/connection-config.yml
delete mode 100644 sdap_ingest_manager/granule_ingester/resources/job-deployment-template.yml
delete mode 100755 sdap_ingest_manager/granule_ingester/runjobs.py
delete mode 100644 sdap_ingest_manager/history_manager/__init__.py
delete mode 100644 sdap_ingest_manager/history_manager/datasetingestionhistorysolr.py
delete mode 100644 sdap_ingest_manager/history_manager/test/datasetingestionhistoryfile_test.py
delete mode 100644 sdap_ingest_manager/history_manager/util.py
delete mode 100644 setup.cfg
delete mode 100644 setup.py
[incubator-sdap-ingester] 01/01: Merge pull request #34 from
apache/dev
Posted by tl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 58eccd06d9622c9a6eab9458c2de7abbd66f4a0e
Merge: 1fedc94 472ab15
Author: thomas loubrieu <60...@users.noreply.github.com>
AuthorDate: Fri May 14 13:58:27 2021 -0700
Merge pull request #34 from apache/dev
merge dev in master (after 13 months without merge)
.gitattributes | 1 -
.github/workflows/github-dev-release.yml | 6 +
.gitignore | 9 +-
.idea/.gitignore | 2 -
.idea/inspectionProfiles/profiles_settings.xml | 6 -
.idea/misc.xml | 7 -
.idea/modules.xml | 8 -
.idea/sdap_ingest_manager.iml | 10 -
.idea/vcs.xml | 6 -
README.md | 199 --------
bin/run_collections | 62 ---
bin/run_granules | 5 -
bin/run_single_collection | 96 ----
collection_manager/README.md | 121 +++++
collection_manager/collection_manager/__init__.py | 1 +
.../collection_manager/entities/Collection.py | 74 +++
.../collection_manager/entities/__init__.py | 2 +
.../entities/exceptions/Exceptions.py | 30 ++
.../entities/exceptions/__init__.py | 6 +
collection_manager/collection_manager/main.py | 100 ++++
.../services/CollectionProcessor.py | 112 ++++
.../services/CollectionWatcher.py | 222 ++++++++
.../services/MessagePublisher.py | 40 ++
.../collection_manager/services/S3Observer.py | 147 ++++++
.../collection_manager/services}/__init__.py | 6 +
.../history_manager/FileIngestionHistory.py | 95 ++--
.../services/history_manager/IngestionHistory.py | 139 +++++
.../history_manager/SolrIngestionHistory.py | 153 ++++++
.../services/history_manager/__init__.py | 4 +
collection_manager/docker/Dockerfile | 20 +
collection_manager/docker/entrypoint.sh | 12 +
.../migration}/__init__.py | 0
.../migration/history_migration.py.bkp | 33 ++
collection_manager/requirements.txt | 9 +
collection_manager/setup.py | 35 ++
.../test => collection_manager/tests}/__init__.py | 0
.../tests/entities}/__init__.py | 0
.../tests/entities/test_Collection.py | 177 +++++++
collection_manager/tests/resources/collections.yml | 47 ++
.../tests/resources/collections_alternate.yml | 45 ++
.../tests/resources/collections_bad_schema.yml | 17 +
.../tests/resources/collections_bad_syntax.yml | 17 +
.../resources}/data/avhrr-oi-analysed-sst.csv | 0
...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | 0
...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | 0
.../tests/services}/__init__.py | 0
.../tests/services/history_manager}/__init__.py | 0
.../history_manager/test_FileIngestionHistory.py | 64 +++
.../history_manager/test_SolrIngestionHistory.py | 17 +-
.../tests/services/test_CollectionProcessor.py | 201 ++++++++
.../tests/services/test_CollectionWatcher.py | 255 ++++++++++
.../tests/services/test_S3Observer.py | 8 +
.../test => common/common}/__init__.py | 0
common/common/async_test_utils/AsyncTestUtils.py | 28 +
common/common/async_test_utils/__init__.py | 1 +
common/common/async_utils/AsyncUtils.py | 11 +
common/common/async_utils/__init__.py | 1 +
common/setup.py | 21 +
config_operator/README.md | 68 +++
config_operator/config_operator/__init__.py | 1 +
.../config_source/LocalDirConfig.py | 80 +++
.../config_source/RemoteGitConfig.py | 82 +++
.../config_operator/config_source/__init__.py | 2 +
.../config_operator/config_source/exceptions.py | 4 +
.../config_operator/k8s/K8sConfigMap.py | 111 ++++
config_operator/config_operator/k8s/__init__.py | 1 +
config_operator/config_operator/main.py | 69 +++
config_operator/containers/docker/Dockerfile | 11 +
.../containers/k8s/config-operator-crd.yml | 79 +++
config_operator/containers/k8s/git-repo-test.yml | 9 +
config_operator/requirements.txt | 5 +
config_operator/setup.py | 38 ++
.../test => config_operator/tests}/__init__.py | 0
.../tests/config_source}/__init__.py | 0
.../tests/config_source/test_LocalDirConfig.py | 72 +++
.../tests/config_source/test_RemoteGitConfig.py | 49 ++
.../test => config_operator/tests/k8s}/__init__.py | 0
config_operator/tests/k8s/test_K8sConfigMap.py | 74 +++
config_operator/tests/resources/collections.yml | 18 +
.../resources/localDirBadTest/collections.yml | 2 +
.../tests/resources/localDirTest/.hidden_file.txt | 1 +
.../tests/resources/localDirTest/README.md | 1 +
.../tests/resources/localDirTest/collections.yml | 1 +
containers/docker/Dockerfile | 11 -
containers/kubernetes/job.yml | 25 -
containers/kubernetes/sdap_ingester_config.yml | 38 --
granule_ingester/README.md | 35 ++
granule_ingester/conda-requirements.txt | 10 +
granule_ingester/docker/Dockerfile | 25 +
granule_ingester/docker/entrypoint.sh | 14 +
granule_ingester/docker/install_nexusproto.sh | 20 +
granule_ingester/granule_ingester/README.md | 11 +
.../granule_ingester}/__init__.py | 0
.../granule_ingester/consumer/MessageConsumer.py | 113 +++++
.../granule_ingester/consumer/__init__.py | 1 +
.../granule_ingester/exceptions/Exceptions.py | 46 ++
.../granule_ingester/exceptions/__init__.py | 8 +
.../granule_loaders/GranuleLoader.py | 78 +++
.../granule_ingester/granule_loaders/__init__.py | 1 +
.../granule_ingester/healthcheck/HealthCheck.py | 8 +
.../granule_ingester/healthcheck/__init__.py | 1 +
granule_ingester/granule_ingester/main.py | 152 ++++++
.../granule_ingester/pipeline/Modules.py | 27 +
.../granule_ingester/pipeline/Pipeline.py | 209 ++++++++
.../granule_ingester/pipeline/__init__.py | 2 +
.../granule_ingester/processors/EmptyTileFilter.py | 42 ++
.../processors/ForceAscendingLatitude.py | 48 ++
.../granule_ingester/processors/GenerateTileId.py | 27 +-
.../processors/Subtract180FromLongitude.py | 38 ++
.../granule_ingester/processors/TileProcessor.py | 41 ++
.../processors/TileSummarizingProcessor.py | 102 ++++
.../granule_ingester/processors/__init__.py | 7 +
.../granule_ingester/processors/kelvintocelsius.py | 43 ++
.../reading_processors/EccoReadingProcessor.py | 64 +++
.../reading_processors/GridReadingProcessor.py | 53 ++
.../reading_processors/SwathReadingProcessor.py | 47 ++
.../reading_processors/TileReadingProcessor.py | 79 +++
.../TimeSeriesReadingProcessor.py | 83 +++
.../processors/reading_processors/__init__.py | 5 +
.../slicers/SliceFileByDimension.py | 55 ++
.../slicers/SliceFileByStepSize.py | 55 ++
.../slicers/SliceFileByTilesDesired.py | 68 +++
.../granule_ingester/slicers/TileSlicer.py | 56 ++
.../granule_ingester/slicers/__init__.py | 2 +
.../granule_ingester/writers/CassandraStore.py | 97 ++++
.../granule_ingester/writers/DataStore.py | 13 +
.../granule_ingester/writers/MetadataStore.py | 11 +
.../granule_ingester/writers/SolrStore.py | 170 +++++++
.../granule_ingester/writers/__init__.py | 4 +
granule_ingester/requirements.txt | 7 +
granule_ingester/setup.py | 34 ++
.../test => granule_ingester/tests}/__init__.py | 0
.../tests/config_files/analysed_sst.yml | 16 +
.../config_files/ingestion_config_testfile.yaml | 17 +
...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | Bin 0 -> 1057327 bytes
.../granules/HLS.S30.T11SPC.2020001.v1.4.hdf.nc | Bin 0 -> 167801 bytes
granule_ingester/tests/granules/OBP_2017_01.nc | Bin 0 -> 2110135 bytes
granule_ingester/tests/granules/OBP_native_grid.nc | Bin 0 -> 1285094 bytes
.../SMAP_L2B_SSS_04892_20160101T005507_R13080.h5 | Bin 0 -> 18672352 bytes
granule_ingester/tests/granules/THETA_199201.nc | Bin 0 -> 4255957 bytes
granule_ingester/tests/granules/empty_mur.nc4 | Bin 0 -> 60937 bytes
.../tests/granules/not_empty_ascatb.nc4 | Bin 0 -> 78036 bytes
.../tests/granules/not_empty_avhrr.nc4 | Bin 0 -> 49511 bytes
granule_ingester/tests/granules/not_empty_ccmp.nc | Bin 0 -> 206870 bytes
granule_ingester/tests/granules/not_empty_mur.nc4 | Bin 0 -> 60907 bytes
granule_ingester/tests/granules/not_empty_smap.h5 | Bin 0 -> 3000192 bytes
granule_ingester/tests/granules/not_empty_wswm.nc | Bin 0 -> 1041568 bytes
.../tests/pipeline}/__init__.py | 0
granule_ingester/tests/pipeline/test_Pipeline.py | 106 ++++
.../tests/processors}/__init__.py | 0
.../processors/test_ForceAscendingLatitude.py | 58 +++
.../tests/processors/test_GenerateTileId.py | 22 +
.../tests/reading_processors}/__init__.py | 0
.../test_EccoReadingProcessor.py | 64 +++
.../test_GridReadingProcessor.py | 265 ++++++++++
.../test_SwathReadingProcessor.py | 74 +++
.../test_TileReadingProcessor.py | 29 ++
.../test_TileSummarizingProcessor.py | 44 ++
.../test_TimeSeriesReadingProcessor.py | 86 ++++
.../tests/slicers}/__init__.py | 0
.../tests/slicers/test_SliceFileByDimension.py | 122 +++++
.../tests/slicers/test_SliceFileByStepSize.py | 105 ++++
.../tests/slicers/test_SliceFileByTilesDesired.py | 88 ++++
granule_ingester/tests/slicers/test_TileSlicer.py | 68 +++
.../tests/writers}/__init__.py | 0
granule_ingester/tests/writers/test_SolrStore.py | 69 +++
requirements.txt | 6 -
sdap_ingest_manager/__init__.py | 1 -
.../collections_ingester/MANIFEST.in | 4 -
.../collections_ingester/collection_ingestion.py | 220 --------
.../google_spreadsheet_collection_config.py | 69 ---
.../collections_ingester/nfs_mount_parse.py | 55 --
.../resources/config/collections.yml.example | 9 -
.../resources/config/credentials.json.template | 10 -
.../resources/config/sdap_ingest_manager.ini | 15 -
.../config/sdap_ingest_manager.ini.default | 50 --
.../resources/dataset_config_template.yml | 45 --
.../collections_ingester/test/data/collections.yml | 9 -
.../test/data/dataset_config_file_ok.yml | 45 --
.../test/unit/collection_ingestion_test.py | 151 ------
.../test/unit/nfs_mount_parse_test.py | 29 --
.../collections_ingester/test/unit/util_test.py | 17 -
.../test/unit/yaml_file_collection_config_test.py | 29 --
.../test/validation/__init__.py | 13 -
.../test/validation/validation_test.py | 136 -----
sdap_ingest_manager/collections_ingester/util.py | 26 -
.../yaml_file_collection_config.py | 32 --
sdap_ingest_manager/granule_ingester/MANIFEST.in | 1 -
sdap_ingest_manager/granule_ingester/Readme.md | 138 -----
sdap_ingest_manager/granule_ingester/__init__.py | 4 -
.../resources/connection-config.yml | 14 -
.../resources/job-deployment-template.yml | 47 --
sdap_ingest_manager/granule_ingester/runjobs.py | 563 ---------------------
sdap_ingest_manager/history_manager/__init__.py | 3 -
.../history_manager/datasetingestionhistorysolr.py | 114 -----
.../test/datasetingestionhistoryfile_test.py | 64 ---
sdap_ingest_manager/history_manager/util.py | 17 -
setup.cfg | 5 -
setup.py | 76 ---
199 files changed, 6230 insertions(+), 2572 deletions(-)