You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2022/06/08 13:21:32 UTC
[incubator-sdap-ingester] 01/01: ignore path starting with http, adapt requirements for conda build to work
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch issue_388_proxy
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f8ad7c77679c15bee852111a4eb99085dc5eab7e
Author: Thomas Loubrieu <lo...@jpl.nasa.gov>
AuthorDate: Wed Jun 8 15:21:15 2022 +0200
ignore path starting with http, adapt requirements for conda build to work
---
collection_manager/collection_manager/entities/Collection.py | 4 ++++
collection_manager/collection_manager/services/CollectionWatcher.py | 5 +++--
config_operator/containers/k8s/config-operator-crd.yml | 3 ++-
granule_ingester/conda-requirements.txt | 3 ++-
granule_ingester/requirements.txt | 3 ++-
5 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/collection_manager/collection_manager/entities/Collection.py b/collection_manager/collection_manager/entities/Collection.py
index 0875e7a..e04315e 100644
--- a/collection_manager/collection_manager/entities/Collection.py
+++ b/collection_manager/collection_manager/entities/Collection.py
@@ -14,9 +14,11 @@ from collection_manager.entities.exceptions import MissingValueCollectionError
logger = logging.getLogger(__name__)
+
class CollectionStorageType(Enum):
LOCAL = 1
S3 = 2
+ REMOTE = 3
@dataclass(frozen=True)
@@ -78,6 +80,8 @@ class Collection:
def storage_type(self):
if urlparse(self.path).scheme == 's3':
return CollectionStorageType.S3
+ elif urlparse(self.path).scheme in {'http', 'https'}:
+ return CollectionStorageType.REMOTE
else:
return CollectionStorageType.LOCAL
diff --git a/collection_manager/collection_manager/services/CollectionWatcher.py b/collection_manager/collection_manager/services/CollectionWatcher.py
index b713f2d..1226351 100644
--- a/collection_manager/collection_manager/services/CollectionWatcher.py
+++ b/collection_manager/collection_manager/services/CollectionWatcher.py
@@ -84,8 +84,9 @@ class CollectionWatcher:
for collection_dict in collections_yaml['collections']:
try:
collection = Collection.from_dict(collection_dict)
- self._validate_collection(collection)
- self._collections_by_dir[collection.directory()].add(collection)
+ if collection.storage_type() != CollectionStorageType.REMOTE:
+ self._validate_collection(collection)
+ self._collections_by_dir[collection.directory()].add(collection)
except MissingValueCollectionError as e:
logger.error(f"A collection is missing '{e.missing_value}'. Ignoring this collection for now.")
except RelativePathCollectionError as e:
diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml
index 2159c00..c02460b 100644
--- a/config_operator/containers/k8s/config-operator-crd.yml
+++ b/config_operator/containers/k8s/config-operator-crd.yml
@@ -1,5 +1,6 @@
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
+
metadata:
name: gitbasedconfigs.sdap.apache.org
spec:
@@ -76,4 +77,4 @@ spec:
containers:
- image: tloubrieu/config-operator:latest
name: git-repo-config-operator
- imagePullPolicy: IfNotPresent
\ No newline at end of file
+ imagePullPolicy: IfNotPresent
diff --git a/granule_ingester/conda-requirements.txt b/granule_ingester/conda-requirements.txt
index 810e278..a1e4206 100644
--- a/granule_ingester/conda-requirements.txt
+++ b/granule_ingester/conda-requirements.txt
@@ -5,6 +5,7 @@ pandas==1.0.4
pytz==2019.3
xarray
pyyaml==5.3.1
-requests==2.23.0
aiohttp==3.6.2
tenacity
+requests==2.27.1
+
diff --git a/granule_ingester/requirements.txt b/granule_ingester/requirements.txt
index 92f31f3..daf53e9 100644
--- a/granule_ingester/requirements.txt
+++ b/granule_ingester/requirements.txt
@@ -1,8 +1,9 @@
cassandra-driver==3.23.0
aiomultiprocess==0.7.0
-aioboto3==8.0.5
+aioboto3==8.3.0
tblib==1.6.0
pysolr==3.9.0
kazoo==2.8.0
aio-pika==6.7.1
elasticsearch[async]
+urllib3==1.26.2