You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ly...@apache.org on 2024/02/23 10:55:51 UTC
(incubator-devlake) branch lw-debug-azure-devops created (now 304830f0a)
This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a change to branch lw-debug-azure-devops
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
at 304830f0a fix(azuredevops): fix some fields' value
This branch includes the following new commits:
new 304830f0a fix(azuredevops): fix some fields' value
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-devlake) 01/01: fix(azuredevops): fix some fields' value
Posted by ly...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch lw-debug-azure-devops
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 304830f0a5cae4600b96cc408ab09983f9a8f2e7
Author: Lynwee Hou <li...@merico.dev>
AuthorDate: Fri Feb 23 18:55:40 2024 +0800
fix(azuredevops): fix some fields' value
---
backend/core/models/domainlayer/code/repo.go | 10 +++++-----
backend/core/plugin/hub.go | 2 +-
backend/python/plugins/azuredevops/azuredevops/main.py | 8 ++++----
backend/python/plugins/azuredevops/azuredevops/migrations.py | 5 +++++
backend/python/plugins/azuredevops/azuredevops/models.py | 3 ++-
.../plugins/azuredevops/azuredevops/streams/pull_requests.py | 4 ++--
backend/python/pydevlake/pydevlake/plugin.py | 2 +-
7 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/backend/core/models/domainlayer/code/repo.go b/backend/core/models/domainlayer/code/repo.go
index 90469c1cb..f6ad9faf9 100644
--- a/backend/core/models/domainlayer/code/repo.go
+++ b/backend/core/models/domainlayer/code/repo.go
@@ -19,7 +19,7 @@ package code
import (
"github.com/apache/incubator-devlake/core/models/domainlayer"
- plugin "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/plugin"
"time"
)
@@ -30,11 +30,11 @@ type Repo struct {
Name string `json:"name"`
Url string `json:"url"`
Description string `json:"description"`
- OwnerId string `json:"ownerId" gorm:"type:varchar(255)"`
+ OwnerId string `json:"owner_id" gorm:"type:varchar(255)"`
Language string `json:"language" gorm:"type:varchar(255)"`
- ForkedFrom string `json:"forkedFrom"`
- CreatedDate *time.Time `json:"createdDate"`
- UpdatedDate *time.Time `json:"updatedDate"`
+ ForkedFrom string `json:"forked_from"`
+ CreatedDate *time.Time `json:"created_date"`
+ UpdatedDate *time.Time `json:"updated_date"`
Deleted bool `json:"deleted"`
}
diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go
index e3c42a8d7..6788c9057 100644
--- a/backend/core/plugin/hub.go
+++ b/backend/core/plugin/hub.go
@@ -80,7 +80,7 @@ func InitPlugins(basicRes context.BasicRes) {
panic(fmt.Errorf("failed to initialize plugin %v due to %w", pluginName, err))
}
} else {
- basicRes.GetLogger().Info("plugin: %s doesn't implement 'PluginInit', it will be skipped.")
+ basicRes.GetLogger().Info("plugin: %s doesn't implement 'PluginInit', it will be skipped.", pluginName)
}
}
}
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py b/backend/python/plugins/azuredevops/azuredevops/main.py
index a58d53591..d8b7a4acb 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -4,7 +4,6 @@
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
-
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
@@ -27,6 +26,7 @@ from pydevlake.domain_layer.code import Repo
from pydevlake.domain_layer.devops import CicdScope
from pydevlake.pipeline_tasks import gitextractor, refdiff
from pydevlake.api import APIException
+from pydevlake.extractor import autoextract
_SUPPORTED_EXTERNAL_SOURCE_PROVIDERS = ['github', 'githubenterprise', 'bitbucket', 'git']
@@ -50,9 +50,9 @@ class AzureDevOpsPlugin(Plugin):
yield Repo(
name=git_repo.name,
url=git_repo.url,
- forked_from=git_repo.parent_repository_url
+ forked_from=git_repo.parent_repository_url,
+ updated_date=git_repo.updated_date,
)
-
yield CicdScope(
name=git_repo.name,
description=git_repo.name,
@@ -88,7 +88,7 @@ class AzureDevOpsPlugin(Plugin):
url = urlparse(raw_repo['remoteUrl'])
url = url._replace(netloc=url.hostname)
raw_repo['url'] = url.geturl()
- repo = GitRepository(**raw_repo)
+ repo = autoextract(raw_repo, GitRepository)
if not repo.default_branch:
continue
if "parentRepository" in raw_repo:
diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py b/backend/python/plugins/azuredevops/azuredevops/migrations.py
index fd3b0d58e..ab33e8a98 100644
--- a/backend/python/plugins/azuredevops/azuredevops/migrations.py
+++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py
@@ -192,3 +192,8 @@ def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder):
table = '_tool_azuredevops_builds'
b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time timestamptz', Dialect.POSTGRESQL)
b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time datetime', Dialect.MYSQL)
+
+
+@migration(20240223170000, name="add updated_date field in _tool_azuredevops_gitrepositories")
+def add_updated_date_field_in_tool_azuredevops_gitrepositories(b: MigrationScriptBuilder):
+ b.add_column('_tool_azuredevops_gitrepositories', 'updated_date', 'datetime(3) DEFAULT NULL')
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py
index 712a582af..8863f1b9f 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -45,8 +45,9 @@ class GitRepository(ToolScope, table=True):
default_branch: Optional[str]
project_id: str
org_id: str
- parent_repository_url: Optional[str] = Field(source='parentRepository/url')
+ parent_repository_url: Optional[str] = Field(source='/parentRepository/url')
provider: Optional[str]
+ updated_date: datetime.datetime = Field(source='/project/lastUpdateTime')
def is_external(self):
return bool(self.provider)
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
index 6922d1b7e..bebb7bed8 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
@@ -37,7 +37,7 @@ class GitPullRequests(Stream):
def convert(self, pr: GitPullRequest, ctx):
repo_id = ctx.scope.domain_id()
- # If the PR is from a fork, we forge a new repo ID for the base repo but it doesn't correspond to a real repo
+ # If the PR is from a fork, we forge a new repo ID for the base repo, but it doesn't correspond to a real repo
base_repo_id = domain_id(GitRepository, ctx.connection.id, pr.fork_repo_id) if pr.fork_repo_id is not None else repo_id
# Use the same status values as GitHub plugin
@@ -56,7 +56,7 @@ class GitPullRequests(Stream):
original_status=pr.status.value,
title=pr.title,
description=pr.description,
- url=pr.url,
+ url=f"{ctx.scope.url}/pullrequest/{pr.pull_request_id}",
author_name=pr.created_by_name,
author_id=pr.created_by_id,
pull_request_key=pr.pull_request_id,
diff --git a/backend/python/pydevlake/pydevlake/plugin.py b/backend/python/pydevlake/pydevlake/plugin.py
index fc368261a..e3303300e 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -71,7 +71,7 @@ class Plugin(ABC):
@abstractmethod
def test_connection(self, connection: Connection) -> msg.TestConnectionResult:
"""
- Test if the the connection with the datasource can be established with the given connection.
+ Test if the connection with the datasource can be established with the given connection.
Must raise an exception if the connection can't be established.
"""
pass