You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ly...@apache.org on 2024/02/23 10:55:52 UTC
(incubator-devlake) 01/01: fix(azuredevops): fix some fields' value
This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch lw-debug-azure-devops
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 304830f0a5cae4600b96cc408ab09983f9a8f2e7
Author: Lynwee Hou <li...@merico.dev>
AuthorDate: Fri Feb 23 18:55:40 2024 +0800
fix(azuredevops): fix some fields' value
---
backend/core/models/domainlayer/code/repo.go | 10 +++++-----
backend/core/plugin/hub.go | 2 +-
backend/python/plugins/azuredevops/azuredevops/main.py | 8 ++++----
backend/python/plugins/azuredevops/azuredevops/migrations.py | 5 +++++
backend/python/plugins/azuredevops/azuredevops/models.py | 3 ++-
.../plugins/azuredevops/azuredevops/streams/pull_requests.py | 4 ++--
backend/python/pydevlake/pydevlake/plugin.py | 2 +-
7 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/backend/core/models/domainlayer/code/repo.go b/backend/core/models/domainlayer/code/repo.go
index 90469c1cb..f6ad9faf9 100644
--- a/backend/core/models/domainlayer/code/repo.go
+++ b/backend/core/models/domainlayer/code/repo.go
@@ -19,7 +19,7 @@ package code
import (
"github.com/apache/incubator-devlake/core/models/domainlayer"
- plugin "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/plugin"
"time"
)
@@ -30,11 +30,11 @@ type Repo struct {
Name string `json:"name"`
Url string `json:"url"`
Description string `json:"description"`
- OwnerId string `json:"ownerId" gorm:"type:varchar(255)"`
+ OwnerId string `json:"owner_id" gorm:"type:varchar(255)"`
Language string `json:"language" gorm:"type:varchar(255)"`
- ForkedFrom string `json:"forkedFrom"`
- CreatedDate *time.Time `json:"createdDate"`
- UpdatedDate *time.Time `json:"updatedDate"`
+ ForkedFrom string `json:"forked_from"`
+ CreatedDate *time.Time `json:"created_date"`
+ UpdatedDate *time.Time `json:"updated_date"`
Deleted bool `json:"deleted"`
}
diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go
index e3c42a8d7..6788c9057 100644
--- a/backend/core/plugin/hub.go
+++ b/backend/core/plugin/hub.go
@@ -80,7 +80,7 @@ func InitPlugins(basicRes context.BasicRes) {
panic(fmt.Errorf("failed to initialize plugin %v due to %w", pluginName, err))
}
} else {
- basicRes.GetLogger().Info("plugin: %s doesn't implement 'PluginInit', it will be skipped.")
+ basicRes.GetLogger().Info("plugin: %s doesn't implement 'PluginInit', it will be skipped.", pluginName)
}
}
}
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py b/backend/python/plugins/azuredevops/azuredevops/main.py
index a58d53591..d8b7a4acb 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -4,7 +4,6 @@
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
-
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
@@ -27,6 +26,7 @@ from pydevlake.domain_layer.code import Repo
from pydevlake.domain_layer.devops import CicdScope
from pydevlake.pipeline_tasks import gitextractor, refdiff
from pydevlake.api import APIException
+from pydevlake.extractor import autoextract
_SUPPORTED_EXTERNAL_SOURCE_PROVIDERS = ['github', 'githubenterprise', 'bitbucket', 'git']
@@ -50,9 +50,9 @@ class AzureDevOpsPlugin(Plugin):
yield Repo(
name=git_repo.name,
url=git_repo.url,
- forked_from=git_repo.parent_repository_url
+ forked_from=git_repo.parent_repository_url,
+ updated_date=git_repo.updated_date,
)
-
yield CicdScope(
name=git_repo.name,
description=git_repo.name,
@@ -88,7 +88,7 @@ class AzureDevOpsPlugin(Plugin):
url = urlparse(raw_repo['remoteUrl'])
url = url._replace(netloc=url.hostname)
raw_repo['url'] = url.geturl()
- repo = GitRepository(**raw_repo)
+ repo = autoextract(raw_repo, GitRepository)
if not repo.default_branch:
continue
if "parentRepository" in raw_repo:
diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py b/backend/python/plugins/azuredevops/azuredevops/migrations.py
index fd3b0d58e..ab33e8a98 100644
--- a/backend/python/plugins/azuredevops/azuredevops/migrations.py
+++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py
@@ -192,3 +192,8 @@ def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder):
table = '_tool_azuredevops_builds'
b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time timestamptz', Dialect.POSTGRESQL)
b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time datetime', Dialect.MYSQL)
+
+
+@migration(20240223170000, name="add updated_date field in _tool_azuredevops_gitrepositories")
+def add_updated_date_field_in_tool_azuredevops_gitrepositories(b: MigrationScriptBuilder):
+ b.add_column('_tool_azuredevops_gitrepositories', 'updated_date', 'datetime(3) DEFAULT NULL')
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py
index 712a582af..8863f1b9f 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -45,8 +45,9 @@ class GitRepository(ToolScope, table=True):
default_branch: Optional[str]
project_id: str
org_id: str
- parent_repository_url: Optional[str] = Field(source='parentRepository/url')
+ parent_repository_url: Optional[str] = Field(source='/parentRepository/url')
provider: Optional[str]
+ updated_date: datetime.datetime = Field(source='/project/lastUpdateTime')
def is_external(self):
return bool(self.provider)
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
index 6922d1b7e..bebb7bed8 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
@@ -37,7 +37,7 @@ class GitPullRequests(Stream):
def convert(self, pr: GitPullRequest, ctx):
repo_id = ctx.scope.domain_id()
- # If the PR is from a fork, we forge a new repo ID for the base repo but it doesn't correspond to a real repo
+ # If the PR is from a fork, we forge a new repo ID for the base repo, but it doesn't correspond to a real repo
base_repo_id = domain_id(GitRepository, ctx.connection.id, pr.fork_repo_id) if pr.fork_repo_id is not None else repo_id
# Use the same status values as GitHub plugin
@@ -56,7 +56,7 @@ class GitPullRequests(Stream):
original_status=pr.status.value,
title=pr.title,
description=pr.description,
- url=pr.url,
+ url=f"{ctx.scope.url}/pullrequest/{pr.pull_request_id}",
author_name=pr.created_by_name,
author_id=pr.created_by_id,
pull_request_key=pr.pull_request_id,
diff --git a/backend/python/pydevlake/pydevlake/plugin.py b/backend/python/pydevlake/pydevlake/plugin.py
index fc368261a..e3303300e 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -71,7 +71,7 @@ class Plugin(ABC):
@abstractmethod
def test_connection(self, connection: Connection) -> msg.TestConnectionResult:
"""
- Test if the the connection with the datasource can be established with the given connection.
+ Test if the connection with the datasource can be established with the given connection.
Must raise an exception if the connection can't be established.
"""
pass