You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ly...@apache.org on 2024/02/28 03:35:48 UTC
(incubator-devlake) branch main updated: fix `updated_at` field in cicd_tasks and cicd_pipelines (#7032)
This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 57efdc027 fix `updated_at` field in cicd_tasks and cicd_pipelines (#7032)
57efdc027 is described below
commit 57efdc02733e271718b88f543577849fdebb8053
Author: Lynwee <li...@merico.dev>
AuthorDate: Wed Feb 28 11:35:39 2024 +0800
fix `updated_at` field in cicd_tasks and cicd_pipelines (#7032)
* fix(gitextractor): cover errors that may leak sensitive information
* fix(azuredevops): fix updated_at field in cicd_tasks and cicd_pipelines
* fix(azuredevops): fix test errors
* fix(azuredevops): remove domain layer tables when collecting data fully
* fix(azuredevops): remove debug codes
* fix(azuredevops): fix testing errors
* fix(azuredevops): fix test errors
* fix(azuredevops): fix comments(most of them are about format)
---
backend/python/plugins/azuredevops/azuredevops/streams/builds.py | 1 +
backend/python/plugins/azuredevops/azuredevops/streams/jobs.py | 1 +
.../azuredevops/azuredevops/streams/pull_request_commits.py | 1 +
.../plugins/azuredevops/azuredevops/streams/pull_requests.py | 1 +
backend/python/pydevlake/pydevlake/model.py | 9 ++++++++-
backend/python/pydevlake/pydevlake/stream.py | 7 ++++++-
backend/python/pydevlake/pydevlake/subtasks.py | 7 ++++++-
backend/python/pydevlake/pydevlake/testing/testing.py | 6 +++++-
8 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
index da44bc03c..aaf7a7be0 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
@@ -25,6 +25,7 @@ from pydevlake import Context, DomainType, Stream
class Builds(Stream):
tool_model = Build
domain_types = [DomainType.CICD]
+ domain_models = [devops.CiCDPipelineCommit, devops.CICDPipeline]
def collect(self, state, context) -> Iterable[tuple[object, dict]]:
repo: GitRepository = context.scope
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
index dae50ad47..39c8883cc 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
@@ -28,6 +28,7 @@ class Jobs(Substream):
tool_model = Job
domain_types = [DomainType.CICD]
parent_stream = Builds
+ domain_models = [devops.CICDTask]
def collect(self, state, context, parent: Build) -> Iterable[tuple[object, dict]]:
repo: GitRepository = context.scope
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py b/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py
index c06e77309..bd5ee0be8 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_request_commits.py
@@ -26,6 +26,7 @@ class GitPullRequestCommits(Substream):
tool_model = GitPullRequestCommit
domain_types = [DomainType.CODE]
parent_stream = GitPullRequests
+ domain_models = [code.PullRequestCommit]
def should_run_on(self, scope: GitRepository) -> bool:
return not scope.is_external()
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
index 77cc118b0..3ae9d97c1 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
@@ -24,6 +24,7 @@ import pydevlake.domain_layer.code as code
class GitPullRequests(Stream):
tool_model = GitPullRequest
domain_types = [DomainType.CODE]
+ domain_models = [code.PullRequest]
def should_run_on(self, scope: GitRepository) -> bool:
return not scope.is_external()
diff --git a/backend/python/pydevlake/pydevlake/model.py b/backend/python/pydevlake/pydevlake/model.py
index 803df9ffe..944750965 100644
--- a/backend/python/pydevlake/pydevlake/model.py
+++ b/backend/python/pydevlake/pydevlake/model.py
@@ -29,7 +29,6 @@ from sqlalchemy.inspection import inspect
from sqlmodel import SQLModel
from pydevlake import Field
-
inflect_engine = inflect.engine()
@@ -42,6 +41,10 @@ class Model(SQLModel):
sa_column=Column(DateTime(), default=datetime.utcnow, onupdate=datetime.utcnow)
)
+ def set_updated_at(self):
+ if self.updated_at is None:
+ self.updated_at = datetime.utcnow()
+
class ToolTable(SQLModel):
@declared_attr
@@ -132,6 +135,10 @@ class NoPKModel(RawDataOrigin):
sa_column=Column(DateTime(), default=datetime.utcnow, onupdate=datetime.utcnow)
)
+ def set_updated_at(self):
+ if self.updated_at is None:
+ self.updated_at = datetime.utcnow()
+
class ToolModel(ToolTable, NoPKModel):
connection_id: Optional[int] = Field(primary_key=True, auto_increment=False)
diff --git a/backend/python/pydevlake/pydevlake/stream.py b/backend/python/pydevlake/pydevlake/stream.py
index 035d2c9fe..6534aadb8 100644
--- a/backend/python/pydevlake/pydevlake/stream.py
+++ b/backend/python/pydevlake/pydevlake/stream.py
@@ -14,7 +14,7 @@
# limitations under the License.
-from typing import Iterable, Type
+from typing import Iterable, Type, Optional
from abc import abstractmethod
from pydevlake.context import Context
@@ -52,6 +52,11 @@ class Stream:
def domain_types(self) -> list[DomainType]:
pass
+ @property
+ @abstractmethod
+ def domain_models(self) -> Optional[list[Type[DomainModel]]]:
+ pass
+
def raw_model(self, session) -> Type[RawModel]:
if self._raw_model is not None:
return self._raw_model
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py b/backend/python/pydevlake/pydevlake/subtasks.py
index 588e478fe..52cadb89b 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -179,6 +179,7 @@ class Extractor(Subtask):
tool_model = self.stream.extract(json.loads(raw.data))
tool_model.set_raw_origin(raw)
tool_model.connection_id = ctx.connection.id
+ tool_model.set_updated_at()
session.merge(tool_model)
def delete(self, session, ctx):
@@ -206,9 +207,13 @@ class Convertor(Subtask):
def _save(self, tool_model: ToolModel, domain_model: DomainModel, session: Session, connection_id: int):
domain_model.set_tool_origin(tool_model)
+ domain_model.set_updated_at()
if isinstance(domain_model, DomainModel):
domain_model.id = tool_model.domain_id()
session.merge(domain_model)
def delete(self, session, ctx):
- pass
\ No newline at end of file
+ domain_models = self.stream.domain_models
+ if domain_models is not None:
+ for domain_model in domain_models:
+ session.execute(sql.delete(domain_model).where(domain_model.raw_data_params == self._params(ctx)))
diff --git a/backend/python/pydevlake/pydevlake/testing/testing.py b/backend/python/pydevlake/pydevlake/testing/testing.py
index 225f47b0e..3b1ec9617 100644
--- a/backend/python/pydevlake/pydevlake/testing/testing.py
+++ b/backend/python/pydevlake/pydevlake/testing/testing.py
@@ -78,7 +78,11 @@ def assert_stream_convert(plugin: Union[Plugin, Type[Plugin]], stream_name: str,
if not isinstance(domain_models, (Iterable, Generator)):
domain_models = [domain_models]
for res, exp in zip(domain_models, expected):
- assert res == exp
+ if (hasattr(res, "__getitem__") and res[0] == "updated_at") or (
+ hasattr(exp, "__getitem__") and exp[0] == "updated_at"):
+ pass
+ else:
+ assert res == exp
def assert_stream_run(stream: Stream, connection: Connection, scope: ToolScope, scope_config: ScopeConfig):