[incubator-devlake] branch main updated: 5383 build not found (#5386)

[ka...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ka94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new fced9715c 5383 build not found (#5386)
fced9715c is described below

commit fced9715c0b17e273c4a0ee6e97209081e9b881a
Author: Camille Teruel <ca...@gmail.com>
AuthorDate: Wed Jun 7 05:39:01 2023 +0200

    5383 build not found (#5386)
    
    * fix: Delete rows from tool layer when not incremental
    
    * fix: Exclude "deleted" builds from collection
    
    * fix: Continue collecting jobs on 404
    
    ---------
    
    Co-authored-by: Camille Teruel <ca...@meri.co>
---
 backend/python/plugins/azuredevops/azuredevops/api.py        |  2 +-
 .../python/plugins/azuredevops/azuredevops/streams/jobs.py   | 12 +++++++++++-
 backend/python/pydevlake/pydevlake/subtasks.py               |  9 +++++++--
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/backend/python/plugins/azuredevops/azuredevops/api.py b/backend/python/plugins/azuredevops/azuredevops/api.py
index 73a35de5a..f187f4125 100644
--- a/backend/python/plugins/azuredevops/azuredevops/api.py
+++ b/backend/python/plugins/azuredevops/azuredevops/api.py
@@ -70,7 +70,7 @@ class AzureDevOpsAPI(API):
         return self.get(org, project, '_apis/git/repositories', repo_id, 'commits')
 
     def builds(self, org: str, project: str, repository_id: str, provider: str):
-        return self.get(org, project, '_apis/build/builds', repositoryId=repository_id, repositoryType=provider)
+        return self.get(org, project, '_apis/build/builds', repositoryId=repository_id, repositoryType=provider, deletedFilter='excludeDeleted')
 
     def jobs(self, org: str, project: str, build_id: int):
         return self.get(org, project, '_apis/build/builds', build_id, 'timeline')
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
index d85d971be..76a80f981 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
@@ -21,6 +21,7 @@ from azuredevops.api import AzureDevOpsAPI
 from azuredevops.models import Job, Build, GitRepository
 from azuredevops.streams.builds import Builds
 from pydevlake import Context, Substream, DomainType
+from pydevlake.api import APIException
 import pydevlake.domain_layer.devops as devops
 
 
@@ -32,7 +33,16 @@ class Jobs(Substream):
     def collect(self, state, context, parent: Build) -> Iterable[tuple[object, dict]]:
         repo: GitRepository = context.scope
         api = AzureDevOpsAPI(context.connection)
-        response = api.jobs(repo.org_id, repo.project_id, parent.id)
+        try:
+            response = api.jobs(repo.org_id, repo.project_id, parent.id)
+        except APIException as e:
+            # Asking for the timeline of a deleted build returns a 204.
+            # But a "deleted" build may be "cleaned" (i.e. deleted for real)
+            # after some time. In this case, the timeline endpoint returns a
+            # 404 instead.
+            if e.response.status == HTTPStatus.NOT_FOUND:
+                return
+            raise
         # If a build has failed before any jobs have started, e.g. due to a
         # bad YAML file, then the jobs endpoint will return a 204 NO CONTENT.
         if response.status == HTTPStatus.NO_CONTENT:
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py b/backend/python/pydevlake/pydevlake/subtasks.py
index 8842825f2..159316db1 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -132,6 +132,10 @@ class Subtask:
             "scope_id": ctx.scope.id
         }, separators=(',', ':'))
 
+    @abstractmethod
+    def delete(self, session, ctx):
+        pass
+
 
 class Collector(Subtask):
     @property
@@ -181,7 +185,8 @@ class Extractor(Subtask):
         session.merge(tool_model)
 
     def delete(self, session, ctx):
-        pass
+        model = self.stream.tool_model
+        session.execute(sql.delete(model).where(model.raw_data_params == self._params(ctx)))
 
 class Convertor(Subtask):
     @property
@@ -209,4 +214,4 @@ class Convertor(Subtask):
         session.merge(domain_model)
 
     def delete(self, session, ctx):
-        pass
+        pass
\ No newline at end of file