[incubator-devlake] branch main updated: Fix substream.fetch (#5310)

[he...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

hez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 5481f4fd8 Fix substream.fetch (#5310)
5481f4fd8 is described below

commit 5481f4fd88208b28805dee0307c108023d75ee0f
Author: Camille Teruel <ca...@gmail.com>
AuthorDate: Wed May 31 01:33:36 2023 +0200

    Fix substream.fetch (#5310)
    
    * style: Rename ill-cased attr
    
    * fix: Fix row filtering in substream.fetch
    
    ---------
    
    Co-authored-by: Camille Teruel <ca...@meri.co>
---
 backend/python/plugins/azuredevops/azuredevops/main.py   | 2 +-
 backend/python/plugins/azuredevops/azuredevops/models.py | 2 +-
 backend/python/pydevlake/pydevlake/subtasks.py           | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py b/backend/python/plugins/azuredevops/azuredevops/main.py
index 42b98f186..fabc53a63 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -108,7 +108,7 @@ class AzureDevOpsPlugin(Plugin):
 
     def extra_tasks(self, scope: GitRepository, tx_rule: AzureDevOpsTransformationRule, entity_types: list[DomainType], connection: AzureDevOpsConnection):
         if DomainType.CODE in entity_types:
-            url = urlparse(scope.remoteUrl)
+            url = urlparse(scope.remote_url)
             url = url._replace(netloc=f'{url.username}:{connection.token}@{url.hostname}')
             yield gitextractor(url.geturl(), scope.domain_id(), connection.proxy)
 
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py
index 87c2c423e..5c656afd1 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -38,7 +38,7 @@ class AzureDevOpsTransformationRule(TransformationRule):
 
 class GitRepository(ToolScope, table=True):
     url: str
-    remoteUrl: str
+    remote_url: str
     default_branch: Optional[str]
     project_id: str
     org_id: str
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py b/backend/python/pydevlake/pydevlake/subtasks.py
index 3193bbad5..8842825f2 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -157,7 +157,9 @@ class Collector(Subtask):
 
 class SubstreamCollector(Collector):
     def fetch(self, state: Dict, session, ctx: Context):
-        for parent in session.exec(sql.select(self.stream.parent_stream.tool_model)).scalars():
+        parent_model = self.stream.parent_stream.tool_model
+        query = session.query(parent_model).where(parent_model.raw_data_params == self._params(ctx))
+        for parent in query.all():
             yield from self.stream.collect(state, ctx, parent)