You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2022/06/08 13:21:31 UTC
[incubator-sdap-ingester] branch issue_388_proxy created (now f8ad7c7)
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a change to branch issue_388_proxy
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
at f8ad7c7 ignore path starting with http, adapt requirements for conda build to work
This branch includes the following new commits:
new f8ad7c7 ignore path starting with http, adapt requirements for conda build to work
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-ingester] 01/01: ignore path starting with http, adapt requirements for conda build to work
Posted by tl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch issue_388_proxy
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f8ad7c77679c15bee852111a4eb99085dc5eab7e
Author: Thomas Loubrieu <lo...@jpl.nasa.gov>
AuthorDate: Wed Jun 8 15:21:15 2022 +0200
ignore path starting with http, adapt requirements for conda build to work
---
collection_manager/collection_manager/entities/Collection.py | 4 ++++
collection_manager/collection_manager/services/CollectionWatcher.py | 5 +++--
config_operator/containers/k8s/config-operator-crd.yml | 3 ++-
granule_ingester/conda-requirements.txt | 3 ++-
granule_ingester/requirements.txt | 3 ++-
5 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/collection_manager/collection_manager/entities/Collection.py b/collection_manager/collection_manager/entities/Collection.py
index 0875e7a..e04315e 100644
--- a/collection_manager/collection_manager/entities/Collection.py
+++ b/collection_manager/collection_manager/entities/Collection.py
@@ -14,9 +14,11 @@ from collection_manager.entities.exceptions import MissingValueCollectionError
logger = logging.getLogger(__name__)
+
class CollectionStorageType(Enum):
LOCAL = 1
S3 = 2
+ REMOTE = 3
@dataclass(frozen=True)
@@ -78,6 +80,8 @@ class Collection:
def storage_type(self):
if urlparse(self.path).scheme == 's3':
return CollectionStorageType.S3
+ elif urlparse(self.path).scheme in {'http', 'https'}:
+ return CollectionStorageType.REMOTE
else:
return CollectionStorageType.LOCAL
diff --git a/collection_manager/collection_manager/services/CollectionWatcher.py b/collection_manager/collection_manager/services/CollectionWatcher.py
index b713f2d..1226351 100644
--- a/collection_manager/collection_manager/services/CollectionWatcher.py
+++ b/collection_manager/collection_manager/services/CollectionWatcher.py
@@ -84,8 +84,9 @@ class CollectionWatcher:
for collection_dict in collections_yaml['collections']:
try:
collection = Collection.from_dict(collection_dict)
- self._validate_collection(collection)
- self._collections_by_dir[collection.directory()].add(collection)
+ if collection.storage_type() != CollectionStorageType.REMOTE:
+ self._validate_collection(collection)
+ self._collections_by_dir[collection.directory()].add(collection)
except MissingValueCollectionError as e:
logger.error(f"A collection is missing '{e.missing_value}'. Ignoring this collection for now.")
except RelativePathCollectionError as e:
diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml
index 2159c00..c02460b 100644
--- a/config_operator/containers/k8s/config-operator-crd.yml
+++ b/config_operator/containers/k8s/config-operator-crd.yml
@@ -1,5 +1,6 @@
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
+
metadata:
name: gitbasedconfigs.sdap.apache.org
spec:
@@ -76,4 +77,4 @@ spec:
containers:
- image: tloubrieu/config-operator:latest
name: git-repo-config-operator
- imagePullPolicy: IfNotPresent
\ No newline at end of file
+ imagePullPolicy: IfNotPresent
diff --git a/granule_ingester/conda-requirements.txt b/granule_ingester/conda-requirements.txt
index 810e278..a1e4206 100644
--- a/granule_ingester/conda-requirements.txt
+++ b/granule_ingester/conda-requirements.txt
@@ -5,6 +5,7 @@ pandas==1.0.4
pytz==2019.3
xarray
pyyaml==5.3.1
-requests==2.23.0
aiohttp==3.6.2
tenacity
+requests==2.27.1
+
diff --git a/granule_ingester/requirements.txt b/granule_ingester/requirements.txt
index 92f31f3..daf53e9 100644
--- a/granule_ingester/requirements.txt
+++ b/granule_ingester/requirements.txt
@@ -1,8 +1,9 @@
cassandra-driver==3.23.0
aiomultiprocess==0.7.0
-aioboto3==8.0.5
+aioboto3==8.3.0
tblib==1.6.0
pysolr==3.9.0
kazoo==2.8.0
aio-pika==6.7.1
elasticsearch[async]
+urllib3==1.26.2