You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2022/06/21 09:56:47 UTC
[incubator-sdap-ingester] branch dev updated: Issue 388 proxy (#52)
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new b8f31b6 Issue 388 proxy (#52)
b8f31b6 is described below
commit b8f31b6cb0136e6fc8d845fe82895dc6133b412f
Author: thomas loubrieu <60...@users.noreply.github.com>
AuthorDate: Tue Jun 21 11:56:43 2022 +0200
Issue 388 proxy (#52)
* ignore path starting with http, adapt requirements for conda build to work
* add exact python version
Co-authored-by: Thomas Loubrieu <lo...@jpl.nasa.gov>
---
collection_manager/collection_manager/entities/Collection.py | 4 ++++
collection_manager/collection_manager/services/CollectionWatcher.py | 5 +++--
collection_manager/docker/Dockerfile | 2 +-
config_operator/containers/k8s/config-operator-crd.yml | 3 ++-
granule_ingester/conda-requirements.txt | 3 ++-
granule_ingester/requirements.txt | 3 ++-
6 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/collection_manager/collection_manager/entities/Collection.py b/collection_manager/collection_manager/entities/Collection.py
index 0875e7a..e04315e 100644
--- a/collection_manager/collection_manager/entities/Collection.py
+++ b/collection_manager/collection_manager/entities/Collection.py
@@ -14,9 +14,11 @@ from collection_manager.entities.exceptions import MissingValueCollectionError
logger = logging.getLogger(__name__)
+
class CollectionStorageType(Enum):
LOCAL = 1
S3 = 2
+ REMOTE = 3
@dataclass(frozen=True)
@@ -78,6 +80,8 @@ class Collection:
def storage_type(self):
if urlparse(self.path).scheme == 's3':
return CollectionStorageType.S3
+ elif urlparse(self.path).scheme in {'http', 'https'}:
+ return CollectionStorageType.REMOTE
else:
return CollectionStorageType.LOCAL
diff --git a/collection_manager/collection_manager/services/CollectionWatcher.py b/collection_manager/collection_manager/services/CollectionWatcher.py
index b713f2d..1226351 100644
--- a/collection_manager/collection_manager/services/CollectionWatcher.py
+++ b/collection_manager/collection_manager/services/CollectionWatcher.py
@@ -84,8 +84,9 @@ class CollectionWatcher:
for collection_dict in collections_yaml['collections']:
try:
collection = Collection.from_dict(collection_dict)
- self._validate_collection(collection)
- self._collections_by_dir[collection.directory()].add(collection)
+ if collection.storage_type() != CollectionStorageType.REMOTE:
+ self._validate_collection(collection)
+ self._collections_by_dir[collection.directory()].add(collection)
except MissingValueCollectionError as e:
logger.error(f"A collection is missing '{e.missing_value}'. Ignoring this collection for now.")
except RelativePathCollectionError as e:
diff --git a/collection_manager/docker/Dockerfile b/collection_manager/docker/Dockerfile
index 83e94ad..8bfb8ab 100644
--- a/collection_manager/docker/Dockerfile
+++ b/collection_manager/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3
+FROM python:3.8.13
RUN apt-get update && apt-get install -y apt-transport-https gnupg2
RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml
index 2159c00..c02460b 100644
--- a/config_operator/containers/k8s/config-operator-crd.yml
+++ b/config_operator/containers/k8s/config-operator-crd.yml
@@ -1,5 +1,6 @@
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
+
metadata:
name: gitbasedconfigs.sdap.apache.org
spec:
@@ -76,4 +77,4 @@ spec:
containers:
- image: tloubrieu/config-operator:latest
name: git-repo-config-operator
- imagePullPolicy: IfNotPresent
\ No newline at end of file
+ imagePullPolicy: IfNotPresent
diff --git a/granule_ingester/conda-requirements.txt b/granule_ingester/conda-requirements.txt
index 810e278..a1e4206 100644
--- a/granule_ingester/conda-requirements.txt
+++ b/granule_ingester/conda-requirements.txt
@@ -5,6 +5,7 @@ pandas==1.0.4
pytz==2019.3
xarray
pyyaml==5.3.1
-requests==2.23.0
aiohttp==3.6.2
tenacity
+requests==2.27.1
+
diff --git a/granule_ingester/requirements.txt b/granule_ingester/requirements.txt
index 92f31f3..daf53e9 100644
--- a/granule_ingester/requirements.txt
+++ b/granule_ingester/requirements.txt
@@ -1,8 +1,9 @@
cassandra-driver==3.23.0
aiomultiprocess==0.7.0
-aioboto3==8.0.5
+aioboto3==8.3.0
tblib==1.6.0
pysolr==3.9.0
kazoo==2.8.0
aio-pika==6.7.1
elasticsearch[async]
+urllib3==1.26.2