You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by he...@apache.org on 2023/03/08 17:42:30 UTC
[incubator-devlake] branch main updated: [feat-2604]: Azure devops plugin (#4450)
This is an automated email from the ASF dual-hosted git repository.
hez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new bf4f3c4c9 [feat-2604]: Azure devops plugin (#4450)
bf4f3c4c9 is described below
commit bf4f3c4c9ea4c6fe6b71e3eb8ff96273819829d8
Author: Keon Amini <ke...@merico.dev>
AuthorDate: Wed Mar 8 11:42:25 2023 -0600
[feat-2604]: Azure devops plugin (#4450)
* feat: More crossdomain domain models
* feat: More devops domain models
* feat: Add code domain models
* feat: Azure devops plugin
* style: Remove leftover debug code
* fix: Define domain_types
* feat: Add paginator
* fix: Fix base_url and improve APIBase.get
* feat: Add api-version query parameter
* fix: Pass context to convert methods
Also removes context from extract methods.
* fix: Fix base_url
* fix: Make plugin instantiable
* test: Add stream tests
* feat: Implement remote scope methods
* feat: Pass tool scope to context
* refactor: Remove GitRepositories stream
Remove GitRepositories stream and use context.scope to get org and project ids instead of context.options.
* fix: Fix tests
* fix: Add missing Licences
* fix: Put connectionId in pipeline tasks options
* style: Remove unused ScopeItem struct
* test: Add transformation rule to test cases
* fix: Remove unused redundant org and project from connection
---------
Co-authored-by: Camille Teruel <ca...@meri.co>
---
backend/python/plugins/azure/README.md | 0
.../azure/azure/__init__.py} | 10 -
backend/python/plugins/azure/azure/api.py | 103 +++++++
.../azure/azure/helper/db.py} | 14 +-
backend/python/plugins/azure/azure/main.py | 97 +++++++
backend/python/plugins/azure/azure/models.py | 202 +++++++++++++
.../python/plugins/azure/azure/streams/builds.py | 128 +++++++++
.../python/plugins/azure/azure/streams/commits.py | 80 ++++++
backend/python/plugins/azure/azure/streams/jobs.py | 81 ++++++
.../azure/azure/streams/pull_request_commits.py | 47 +++
.../plugins/azure/azure/streams/pull_requests.py | 92 ++++++
.../{pydevlake => plugins/azure}/poetry.lock | 317 +++++++++++----------
.../{pydevlake => plugins/azure}/pyproject.toml | 18 +-
backend/python/plugins/azure/tests/test_streams.py | 185 ++++++++++++
backend/python/pydevlake/README.md | 4 +-
backend/python/pydevlake/poetry.lock | 30 +-
backend/python/pydevlake/pydevlake/api.py | 11 +-
backend/python/pydevlake/pydevlake/context.py | 4 +-
.../pydevlake/pydevlake/domain_layer/code.py | 169 +++++++++++
.../pydevlake/domain_layer/crossdomain.py | 102 ++++++-
.../pydevlake/pydevlake/domain_layer/devops.py | 89 +++---
backend/python/pydevlake/pydevlake/ipc.py | 8 +-
backend/python/pydevlake/pydevlake/plugin.py | 13 +-
backend/python/pydevlake/pydevlake/stream.py | 3 +-
backend/python/pydevlake/pydevlake/subtasks.py | 6 +-
.../crossdomain.py => testing/__init__.py} | 11 +-
.../crossdomain.py => testing/testing.py} | 21 +-
backend/python/pydevlake/pyproject.toml | 1 +
backend/python/pydevlake/test/remote_test.go | 99 -------
backend/python/pydevlake/test/stream_test.py | 17 +-
backend/server/services/remote/models/models.go | 2 +-
.../services/remote/plugin/plugin_extensions.go | 2 +-
.../server/services/remote/plugin/plugin_impl.go | 29 +-
.../services/remote/plugin/remote_scope_api.go | 7 -
backend/server/services/remote/plugin/scope_api.go | 7 +
backend/test/helper/client.go | 2 +-
backend/test/remote/fakeplugin/fakeplugin/main.py | 14 +-
backend/test/remote/remote_test.go | 59 +++-
38 files changed, 1691 insertions(+), 393 deletions(-)
diff --git a/backend/python/plugins/azure/README.md b/backend/python/plugins/azure/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py b/backend/python/plugins/azure/azure/__init__.py
similarity index 84%
copy from backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
copy to backend/python/plugins/azure/azure/__init__.py
index b001531df..65d64ce95 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
+++ b/backend/python/plugins/azure/azure/__init__.py
@@ -12,13 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-
-from pydevlake.model import DomainModel
-
-
-class User(DomainModel, table=True):
- __table_name__ = 'users'
-
- name: str
- email: str
diff --git a/backend/python/plugins/azure/azure/api.py b/backend/python/plugins/azure/azure/api.py
new file mode 100644
index 000000000..67e8fee39
--- /dev/null
+++ b/backend/python/plugins/azure/azure/api.py
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+import base64
+
+from pydevlake.api import API, request_hook, Paginator, Request
+
+
+class AzurePaginator(Paginator):
+ def get_items(self, response) -> Optional[list[object]]:
+ return response.json['value']
+
+ def get_next_page_id(self, response) -> Optional[int | str]:
+ return response.headers.get('x-ms-continuation')
+
+ def set_next_page_param(self, request, next_page_id):
+ request.query_args['continuationToken'] = next_page_id
+
+
+class AzureDevOpsAPI(API):
+ paginator = AzurePaginator()
+
+ def __init__(self, base_url: str, pat: str):
+ self._base_url = base_url or "https://dev.azure.com/"
+ self.pat = pat
+
+ @property
+ def base_url(self):
+ return self._base_url
+
+ @request_hook
+ def authenticate(self, request: Request):
+ if self.pat:
+ pat_b64 = base64.b64encode((':' + self.pat).encode()).decode()
+ request.headers['Authorization'] = 'Basic ' + pat_b64
+
+ @request_hook
+ def set_api_version(self, request: Request):
+ request.query_args['api-version'] = "7.0"
+
+ def my_profile(self):
+ req = Request('https://app.vssps.visualstudio.com/_apis/profile/profiles/me')
+ return self.send(req)
+
+ def accounts(self, member_id: str):
+ req = Request('https://app.vssps.visualstudio.com/_apis/accounts', query_args={"memberId": member_id})
+ return self.send(req)
+
+ def orgs(self) -> list[str]:
+ response = self.accounts()
+ return [acct["AccountName"] for acct in response.json]
+
+ def projects(self, org: str):
+ return self.get(org, '_apis/projects')
+
+ # Get a project
+ def project(self, org: str, project: str):
+ return self.get(org, '_apis/projects', project)
+
+ # List repos under an org
+ def git_repos(self, org: str, project: str):
+ return self.get(org, project, '_apis/git/repositories')
+
+ def git_repo_pull_requests(self, org: str, project: str, repo_id: str):
+ # see https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-requests/get-pull-requests?view=azure-devops-rest-7.1&tabs=HTTP
+ return self.get(org, project, '_apis/git/repositories', repo_id, 'pullrequests')
+
+ def git_repo_pull_request_commits(self, org: str, project: str, repo_id: str, pull_request_id: int):
+ return self.get(org, project, '_apis/git/repositories', repo_id, 'pullRequests', pull_request_id, 'commits')
+
+ def git_repo_pull_request_comments(self, org: str, project: str, repo_id: str, pull_request_id: int):
+ return self.get(org, project, '_apis/git/repositories', repo_id, 'pullRequests', pull_request_id, 'threads')
+
+ # not needed
+ def commits(self, org: str, project: str, repo_id: str):
+ return self.get(org, project, '_apis/git/repositories', repo_id, 'commits')
+
+ def builds(self, org: str, project: str):
+ return self.get(org, project, '_apis/build/builds')
+
+ def jobs(self, org: str, project: str, build_id: int):
+ return self.get(org, project, '_apis/build/builds', build_id, 'timeline')
+
+ # unused
+ def deployments(self, org: str, project: str):
+ return self.get(org, project, '_apis/release/deployments')
+
+ # unused
+ def releases(self, org: str, project: str):
+ return self.get(org, project, '_apis/release/releases')
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py b/backend/python/plugins/azure/azure/helper/db.py
similarity index 70%
copy from backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
copy to backend/python/plugins/azure/azure/helper/db.py
index b001531df..0c1f81be3 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
+++ b/backend/python/plugins/azure/azure/helper/db.py
@@ -13,12 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from typing import Type
-from pydevlake.model import DomainModel
+from sqlalchemy import sql
+from sqlmodel import Session
+from pydevlake import Context
-class User(DomainModel, table=True):
- __table_name__ = 'users'
- name: str
- email: str
+def get(ctx: Context, model_type: Type, *query) -> any:
+ with Session(ctx.engine) as session:
+ stmt = sql.select(model_type).filter(*query)
+ model = session.exec(stmt).scalar()
+ return model
diff --git a/backend/python/plugins/azure/azure/main.py b/backend/python/plugins/azure/azure/main.py
new file mode 100644
index 000000000..86a2644df
--- /dev/null
+++ b/backend/python/plugins/azure/azure/main.py
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from azure.api import AzureDevOpsAPI
+from azure.models import AzureDevOpsConnection, GitRepository
+from azure.streams.builds import Builds
+from azure.streams.commits import GitCommits
+from azure.streams.jobs import Jobs
+from azure.streams.pull_request_commits import GitPullRequestCommits
+from azure.streams.pull_requests import GitPullRequests
+
+from pydevlake import Plugin, RemoteScopeGroup
+from pydevlake.domain_layer.code import Repo
+from pydevlake.domain_layer.devops import CicdScope
+
+
+class AzureDevOpsPlugin(Plugin):
+
+ @property
+ def connection_type(self):
+ return AzureDevOpsConnection
+
+ @property
+ def tool_scope_type(self):
+ return GitRepository
+
+ def domain_scopes(self, git_repo: GitRepository):
+ yield Repo(
+ name=git_repo.name,
+ url=git_repo.url,
+ forked_from=git_repo.parentRepositoryUrl,
+ deleted=git_repo.isDisabled,
+ )
+
+ yield CicdScope(
+ name=git_repo.name,
+ description=git_repo.name,
+ url=git_repo.url
+ )
+
+ def remote_scope_groups(self, ctx) -> list[RemoteScopeGroup]:
+ api = AzureDevOpsAPI(ctx.connection.base_url, ctx.connection.pat)
+ member_id = api.my_profile.json['id']
+ accounts = api.accounts(member_id).json
+ orgs = [acc['accountId'] for acc in accounts]
+ for org in orgs:
+ for proj in api.projects(org):
+ yield RemoteScopeGroup(
+ id=f'{org}/{proj["name"]}',
+ name=proj['name']
+ )
+
+ def remote_scopes(self, ctx, group_id: str) -> list[GitRepository]:
+ org, proj = group_id.split('/')
+ api = AzureDevOpsAPI(ctx.connection.base_url, ctx.connection.pat)
+ for raw_repo in api.git_repos(org, proj):
+ repo = GitRepository(**raw_repo, project_id=proj, org_id=org)
+ if not repo.defaultBranch:
+ return None
+ if "parentRepository" in raw_repo:
+ repo.parentRepositoryUrl = raw_repo["parentRepository"]["url"]
+ yield repo
+
+ @property
+ def name(self) -> str:
+ return "azure"
+
+ def test_connection(self, connection: AzureDevOpsConnection):
+ resp = AzureDevOpsAPI(connection.base_url, connection.pat).projects(connection.org)
+ if resp.status != 200:
+ raise Exception(f"Invalid connection: {resp.json}")
+
+ @property
+ def streams(self):
+ return [
+ GitPullRequests,
+ GitPullRequestCommits,
+ GitCommits,
+ Builds,
+ Jobs,
+ ]
+
+
+if __name__ == '__main__':
+ AzureDevOpsPlugin.start()
diff --git a/backend/python/plugins/azure/azure/models.py b/backend/python/plugins/azure/azure/models.py
new file mode 100644
index 000000000..8f2575052
--- /dev/null
+++ b/backend/python/plugins/azure/azure/models.py
@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+from enum import Enum
+from typing import Optional
+
+from sqlmodel import Field
+
+from pydevlake import Connection
+from pydevlake.model import ToolModel, ToolScope
+
+default_date = datetime.datetime.fromisoformat("1970-01-01")
+
+
+class AzureDevOpsConnection(Connection):
+ base_url: str
+ pat: str
+
+
+class Project(ToolModel, table=True):
+ id: str = Field(primary_key=True)
+ name: str
+ url: str
+
+
+class GitRepository(ToolScope, table=True):
+ url: str
+ sshUrl: str
+ remoteUrl: str
+ defaultBranch: Optional[str]
+ project_id: str # = Field(foreign_key=Project.id)
+ org_id: str
+ size: int
+ isDisabled: bool
+ isInMaintenance: bool
+ isFork: Optional[bool]
+ parentRepositoryUrl: Optional[str]
+
+
+# https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-requests/get-pull-requests?view=azure-devops-rest-7.1&tabs=HTTP#identityrefwithvote
+class GitPullRequest(ToolModel, table=True):
+ class Status(Enum):
+ Abandoned = "abandoned"
+ Active = "active"
+ All = "all"
+ Completed = "completed"
+ NotSet = "notSet"
+
+ id: int = Field(primary_key=True)
+ project_id: Optional[str]
+ description: Optional[str]
+ code_review_id: int = 0
+ repo_id: Optional[str]
+ status: Status
+ created_by_id: Optional[str]
+ created_by_name: Optional[str]
+ creation_date: datetime.datetime = default_date
+ closed_date: datetime.datetime = default_date
+ source_commit_sha: Optional[str] # lastmergesourcecommit #base
+ target_commit_sha: Optional[str] # lastmergetargetcommit #head
+ merge_commit_sha: Optional[str]
+ url: Optional[str]
+ type: Optional[str]
+ title: Optional[str]
+ target_ref_name: Optional[str]
+ source_ref_name: Optional[str]
+ fork_repo_id: Optional[str]
+
+
+class GitCommit(ToolModel, table=True):
+ commit_sha: str = Field(primary_key=True)
+ project_id: str
+ repo_id: str
+ committer_name: str = ""
+ committer_email: str = ""
+ commit_date: datetime.datetime = default_date
+ author_name: str = ""
+ author_email: str = ""
+ authored_date: datetime.datetime = default_date
+ comment: str = ""
+ url: str = ""
+ additions: int = 0
+ deletions: int = 0
+
+
+class Account(ToolModel, table=True):
+ class Type(Enum):
+ Organization = "organization"
+ Personal = "personal"
+
+ class Status(Enum):
+ Deleted = "deleted"
+ Disabled = "disabled"
+ Enabled = "enabled"
+ Moved = "moved"
+ Non = "none"
+
+ account_id: str = Field(primary_key=True)
+ account_name: str
+ account_owner: str
+ account_type: Type
+ account_status: Status
+ organization_name: str
+ namespace_id: str
+
+
+class Build(ToolModel, table=True):
+ class Status(Enum):
+ All = "all"
+ Cancelling = "cancelling"
+ Completed = "completed"
+ InProgress = "inProgress"
+ Non = "none"
+ NotStarted = "notStarted"
+ Postponed = "postponed"
+
+ class Priority(Enum):
+ AboveNormal = "aboveNormal"
+ BelowNormal = "belowNormal"
+ High = "high"
+ Low = "low"
+ Normal = "normal"
+
+ class Result(Enum):
+ Canceled = "canceled"
+ Failed = "failed"
+ Non = "none"
+ PartiallySucceeded = "partiallySucceeded"
+ Succeeded = "succeeded"
+
+ id: int = Field(primary_key=True)
+ project_id: str
+ repo_id: str
+ repo_type: str
+ build_number: str
+ build_number_revision: Optional[str]
+ controller_id: Optional[str]
+ definition_id: Optional[str]
+ deleted: Optional[bool]
+ start_time: Optional[datetime.datetime]
+ finish_time: Optional[datetime.datetime]
+ status: Status
+ tags: list[str] = []
+ priority: Priority
+ build_result: Result
+ source_branch: str
+ source_version: str
+
+
+class Job(ToolModel, table=True):
+ class Type(Enum):
+ Task = "Task"
+ Job = "Job"
+ Checkpoint = "Checkpoint"
+ Stage = "Stage"
+ Phase = "Phase"
+
+ class State(Enum):
+ Completed = "completed"
+ InProgress = "inProgress"
+ Pending = "pending"
+
+ class Result(Enum):
+ Abandoned = "abandoned"
+ Canceled = "canceled"
+ Failed = "failed"
+ Skipped = "skipped"
+ Succeeded = "succeeded"
+ SucceededWithIssues = "succeededWithIssues"
+
+ id: str = Field(primary_key=True)
+ build_id: int
+ repo_id: str
+ parentId: Optional[str]
+ type: Optional[Type]
+ name: str
+ startTime: datetime.datetime
+ finishTime: datetime.datetime
+ lastModified: datetime.datetime
+ currentOperation: Optional[int]
+ percentComplete: Optional[int]
+ state: State
+ result: Result
+ resultCode: Optional[int]
+ changeId: Optional[int]
+ workerName: Optional[str]
+ order: Optional[int]
+ errorCount: Optional[int]
+ warningCount: Optional[int]
diff --git a/backend/python/plugins/azure/azure/streams/builds.py b/backend/python/plugins/azure/azure/streams/builds.py
new file mode 100644
index 000000000..c47a6597f
--- /dev/null
+++ b/backend/python/plugins/azure/azure/streams/builds.py
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+from typing import Iterable
+
+import iso8601 as iso8601
+
+from azure.api import AzureDevOpsAPI
+from azure.helper import db
+from azure.models import AzureDevOpsConnection, GitRepository
+from azure.models import Build
+from pydevlake import Context, DomainType, Stream, logger
+import pydevlake.domain_layer.devops as devops
+
+
+class Builds(Stream):
+ tool_model = Build
+ domain_types = [DomainType.CICD]
+
+ def collect(self, state, context) -> Iterable[tuple[object, dict]]:
+ connection: AzureDevOpsConnection = context.connection
+ repo: GitRepository = context.scope
+ azure_api = AzureDevOpsAPI(connection.base_url, connection.pat)
+ cached_repos = dict()
+ response = azure_api.builds(repo.org_id, repo.project_id)
+ for raw_build in response:
+ if self.validate_repo(context, raw_build, cached_repos):
+ yield raw_build, state
+
+ def extract(self, raw_data: dict) -> Build:
+ build: Build = self.tool_model(**raw_data)
+ build.id = raw_data["id"]
+ build.project_id = raw_data["project"]["id"]
+ build.repo_id = raw_data["repository"]["id"]
+ build.repo_type = raw_data["repository"]["type"]
+ build.source_branch = raw_data["sourceBranch"]
+ build.source_version = raw_data["sourceVersion"]
+ build.build_number = raw_data["buildNumber"]
+ if "buildNumberRevision" in raw_data:
+ build.build_number_revision = raw_data["buildNumberRevision"]
+ build.start_time = iso8601.parse_date(raw_data["startTime"])
+ build.finish_time = iso8601.parse_date(raw_data["finishTime"])
+ build.status = Build.Status(raw_data["status"])
+ build.tags = ",".join(raw_data["tags"])
+ build.priority = raw_data["priority"]
+ build.build_result = Build.Result(raw_data["result"])
+ trigger_info: dict = raw_data["triggerInfo"]
+ if "ci.sourceSha" in trigger_info: # this key is not guaranteed to be in here per docs
+ assert build.source_version == trigger_info["ci.sourceSha"]
+ return build
+
+ def convert(self, b: Build, ctx: Context):
+ result = None
+ match b.build_result:
+ case Build.Result.Canceled:
+ result = devops.CICDResult.ABORT
+ case Build.Result.Failed:
+ result = devops.CICDResult.FAILURE
+ case Build.Result.PartiallySucceeded:
+ result = devops.CICDResult.SUCCESS
+ case Build.Result.Succeeded:
+ result = devops.CICDResult.SUCCESS
+
+ status = None
+ match b.status:
+ case Build.Status.All:
+ status = devops.CICDStatus.IN_PROGRESS
+ case Build.Status.Cancelling:
+ status = devops.CICDStatus.DONE
+ case Build.Status.Completed:
+ status = devops.CICDStatus.DONE
+ case Build.Status.InProgress:
+ status = devops.CICDStatus.IN_PROGRESS
+ case Build.Status.NotStarted:
+ status = devops.CICDStatus.IN_PROGRESS
+ case Build.Status.Postponed:
+ status = devops.CICDStatus.IN_PROGRESS
+
+ yield devops.CICDPipeline(
+ name=b.id,
+ status=status,
+ created_date=b.start_time,
+ finished_date=b.finish_time,
+ result=result,
+ duration_sec=abs(b.finish_time.second-b.start_time.second),
+ environment=devops.CICDEnvironment.PRODUCTION,
+ type=devops.CICDType.DEPLOYMENT,
+ cicd_scope_id=b.repo_id,
+ )
+
+ repo_url = None
+ if b.repo_type == 'GitHub':
+ repo_url = f'https://github.com/{b.repo_id}'
+
+ yield devops.CiCDPipelineCommit(
+ pipeline_id=b.id,
+ commit_sha=b.source_version,
+ branch=b.source_branch,
+ repo_id=b.repo_id,
+ repo=repo_url,
+ )
+
+ # workaround because azure also returns builds for unmanaged repos (we don't want them)
+ @classmethod
+ def validate_repo(cls, context: Context, raw_build: dict, cached_repos: typing.Dict[str, GitRepository]) -> bool:
+ repo_id = raw_build["repository"]["id"]
+ if repo_id not in cached_repos:
+ repo: GitRepository = db.get(context, GitRepository, GitRepository.id == repo_id)
+ if repo is None:
+ logger.warn(f"no Azure repo associated with {repo_id}")
+ cached_repos[repo_id] = repo
+ if cached_repos[repo_id] is None:
+ return False
+ raw_build["repository"]["url"] = cached_repos[repo_id].url
+ return True
diff --git a/backend/python/plugins/azure/azure/streams/commits.py b/backend/python/plugins/azure/azure/streams/commits.py
new file mode 100644
index 000000000..088344300
--- /dev/null
+++ b/backend/python/plugins/azure/azure/streams/commits.py
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Iterable
+
+import iso8601 as iso8601
+
+from azure.api import AzureDevOpsAPI
+from azure.models import GitRepository, GitCommit
+from pydevlake import Stream, DomainType, Context
+from pydevlake.domain_layer.code import Commit as DomainCommit
+from pydevlake.domain_layer.code import RepoCommit as DomainRepoCommit
+
+
+class GitCommits(Stream):
+ tool_model = GitCommit
+ domain_types = [DomainType.CODE]
+
+ def collect(self, state, context) -> Iterable[tuple[object, dict]]:
+ connection = context.connection
+ repo: GitRepository = context.scope
+ azure_api = AzureDevOpsAPI(connection.base_url, connection.pat)
+ response = azure_api.commits(repo.org_id, repo.project_id, repo.id)
+ for raw_commit in response:
+ raw_commit["repo_id"] = repo.id
+ yield raw_commit, state
+
+ def extract(self, raw_data: dict) -> GitCommit:
+ return extract_raw_commit(raw_data)
+
+ def convert(self, commit: GitCommit, ctx: Context) -> Iterable[DomainCommit]:
+ yield DomainCommit(
+ sha=commit.commit_sha,
+ additions=commit.additions,
+ deletions=commit.deletions,
+ message=commit.comment,
+ author_name=commit.author_name,
+ author_email=commit.author_email,
+ authored_date=commit.authored_date,
+ author_id=commit.author_name,
+ committer_name=commit.committer_name,
+ committer_email=commit.committer_email,
+ committed_date=commit.commit_date,
+ committer_id=commit.committer_name,
+ )
+
+ yield DomainRepoCommit(
+ repo_id=commit.repo_id,
+ commit_sha=commit.commit_sha,
+ )
+
+
+def extract_raw_commit(stream: Stream, raw_data: dict, ctx: Context) -> GitCommit:
+ commit: GitCommit = stream.tool_model(**raw_data)
+ repo: GitRepository = ctx.scope
+ commit.project_id = repo.project_id
+ commit.repo_id = raw_data["repo_id"]
+ commit.commit_sha = raw_data["commitId"]
+ commit.author_name = raw_data["author"]["name"]
+ commit.author_email = raw_data["author"]["email"]
+ commit.authored_date = iso8601.parse_date(raw_data["author"]["date"])
+ commit.committer_name = raw_data["committer"]["name"]
+ commit.committer_email = raw_data["committer"]["email"]
+ commit.commit_date = iso8601.parse_date(raw_data["committer"]["date"])
+ if "changeCounts" in raw_data:
+ commit.additions = raw_data["changeCounts"]["Add"]
+ commit.deletions = raw_data["changeCounts"]["Delete"]
+ return commit
diff --git a/backend/python/plugins/azure/azure/streams/jobs.py b/backend/python/plugins/azure/azure/streams/jobs.py
new file mode 100644
index 000000000..b6f11c410
--- /dev/null
+++ b/backend/python/plugins/azure/azure/streams/jobs.py
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Iterable
+
+from azure.api import AzureDevOpsAPI
+from azure.models import AzureDevOpsConnection, Job, Build, GitRepository
+from azure.streams.builds import Builds
+from pydevlake import Context, Substream, DomainType
+import pydevlake.domain_layer.devops as devops
+
+
+class Jobs(Substream):
+ tool_model = Job
+ domain_types = [DomainType.CICD]
+ parent_stream = Builds
+
+ def collect(self, state, context, parent: Build) -> Iterable[tuple[object, dict]]:
+ connection: AzureDevOpsConnection = context.connection
+ repo: GitRepository = context.scope
+ azure_api = AzureDevOpsAPI(connection.base_url, connection.pat)
+ response = azure_api.jobs(repo.org_id, repo.project_id, parent.id)
+ if response.status != 200:
+ yield None, state
+ else:
+ for raw_job in response.json["records"]:
+ raw_job["build_id"] = parent.id
+ raw_job["repo_id"] = parent.repo_id
+ yield raw_job, state
+
+
+ def convert(self, j: Job, ctx: Context) -> Iterable[devops.CICDPipeline]:
+ result = None
+ match j.result:
+ case Job.Result.Abandoned:
+ result = devops.CICDResult.ABORT
+ case Job.Result.Canceled:
+ result = devops.CICDResult.ABORT
+ case Job.Result.Failed:
+ result = devops.CICDResult.FAILURE
+ case Job.Result.Skipped:
+ result = devops.CICDResult.ABORT
+ case Job.Result.Succeeded:
+ result = devops.CICDResult.SUCCESS
+ case Job.Result.SucceededWithIssues:
+ result = devops.CICDResult.FAILURE
+
+ status = None
+ match j.state:
+ case Job.State.Completed:
+ status = devops.CICDStatus.DONE
+ case Job.State.InProgress:
+ status = devops.CICDStatus.IN_PROGRESS
+ case Job.State.Pending:
+ status = devops.CICDStatus.IN_PROGRESS
+
+ yield devops.CICDTask(
+ id=j.id,
+ name=j.name,
+ pipeline_id=j.build_id,
+ status=status,
+ created_date=j.startTime,
+ finished_date=j.finishTime,
+ result=result,
+ type=devops.CICDType.BUILD,
+ duration_sec=abs(j.finishTime.second-j.startTime.second),
+ environment=devops.CICDEnvironment.PRODUCTION,
+ cicd_scope_id=j.repo_id
+ )
diff --git a/backend/python/plugins/azure/azure/streams/pull_request_commits.py b/backend/python/plugins/azure/azure/streams/pull_request_commits.py
new file mode 100644
index 000000000..661fa7f4c
--- /dev/null
+++ b/backend/python/plugins/azure/azure/streams/pull_request_commits.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Iterable
+
+from azure.api import AzureDevOpsAPI
+from azure.models import GitPullRequest, GitCommit, GitRepository
+from azure.streams.commits import extract_raw_commit
+from azure.streams.pull_requests import GitPullRequests
+from pydevlake import Substream, DomainType
+from pydevlake.domain_layer.code import PullRequestCommit as DomainPullRequestCommit
+
+
+class GitPullRequestCommits(Substream):
+ tool_model = GitCommit
+ domain_types = [DomainType.CODE]
+ parent_stream = GitPullRequests
+
+ def collect(self, state, context, parent: GitPullRequest) -> Iterable[tuple[object, dict]]:
+ connection = context.connection
+ repo: GitRepository = context.scope
+ azure_api = AzureDevOpsAPI(connection.base_url, connection.pat)
+ response = azure_api.git_repo_pull_request_commits(repo.org_id, repo.project_id, parent.repo_id, parent.id)
+ for raw_commit in response:
+ raw_commit["repo_id"] = parent.repo_id
+ yield raw_commit, state
+
+ def extract(self, raw_data: dict) -> GitCommit:
+ return extract_raw_commit(self, raw_data)
+
+ def convert(self, commit: GitCommit, context) -> Iterable[DomainPullRequestCommit]:
+ yield DomainPullRequestCommit(
+ commit_sha=commit.commit_sha,
+ pull_request_id=commit.repo_id,
+ )
diff --git a/backend/python/plugins/azure/azure/streams/pull_requests.py b/backend/python/plugins/azure/azure/streams/pull_requests.py
new file mode 100644
index 000000000..5c2384ee8
--- /dev/null
+++ b/backend/python/plugins/azure/azure/streams/pull_requests.py
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from typing import Iterable
+
+import iso8601 as iso8601
+
+from azure.api import AzureDevOpsAPI
+from azure.helper import db
+from azure.models import GitRepository, GitPullRequest, GitCommit
+from pydevlake import Stream, Context, DomainType
+from pydevlake.domain_layer.code import PullRequest as DomainPullRequest
+
+
+class GitPullRequests(Stream):
+ tool_model = GitPullRequest
+ domain_types = [DomainType.CODE]
+
+ def collect(self, state, context) -> Iterable[tuple[object, dict]]:
+ connection = context.connection
+ azure_api = AzureDevOpsAPI(connection.base_url, connection.pat)
+ repo: GitRepository = context.scope
+ response = azure_api.git_repo_pull_requests(repo.org_id, repo.project_id, repo.id)
+ for raw_pr in response:
+ yield raw_pr, state
+
+ def extract(self, raw_data: dict) -> GitPullRequest:
+ pr = GitPullRequest(**raw_data)
+ pr.id = raw_data["pullRequestId"]
+ pr.created_by_id = raw_data["createdBy"]["id"]
+ pr.created_by_name = raw_data["createdBy"]["displayName"]
+ if "closedDate" in raw_data:
+ pr.closed_date = iso8601.parse_date(raw_data["closedDate"])
+ pr.creation_date = iso8601.parse_date(raw_data["creationDate"])
+ pr.code_review_id = raw_data["codeReviewId"]
+ pr.repo_id = raw_data["repository"]["id"]
+ pr.title = raw_data["title"]
+ pr.description = raw_data["description"]
+ pr.source_commit_sha = raw_data["lastMergeSourceCommit"]["commitId"]
+ pr.target_commit_sha = raw_data["lastMergeTargetCommit"]["commitId"]
+ pr.merge_commit_sha = raw_data["lastMergeCommit"]["commitId"]
+ pr.source_ref_name = raw_data["sourceRefName"]
+ pr.target_ref_name = raw_data["targetRefName"]
+ pr.status = raw_data["status"]
+ pr.url = raw_data["url"]
+ if "labels" in raw_data:
+ # TODO get this off transformation rules regex
+ pr.type = raw_data["labels"][0]["name"]
+ if "forkSource" in raw_data:
+ pr.fork_repo_id = raw_data["forkSource"]["repository"]["id"]
+ return pr
+
+ def convert(self, pr: GitPullRequest, context: Context) -> Iterable[DomainPullRequest]:
+ merged_date: datetime = None
+ if pr.status == GitPullRequest.Status.Completed:
+ # query from commits
+ merge_commit: GitCommit = db.get(context, GitCommit, GitCommit.commit_sha == pr.merge_commit_sha)
+ merged_date = merge_commit.commit_date
+ yield DomainPullRequest(
+ base_repo_id=(pr.fork_repo_id if pr.fork_repo_id is not None else pr.repo_id),
+ head_repo_id=pr.repo_id,
+ status=pr.status.value,
+ title=pr.title,
+ description=pr.description,
+ url=pr.url,
+ author_name=pr.created_by_name,
+ author_id=pr.created_by_id,
+ pull_request_key=pr.id,
+ created_date=pr.creation_date,
+ merged_date=merged_date,
+ closed_date=pr.closed_date,
+ type=pr.type,
+ component="", # not supported
+ merge_commit_sha=pr.merge_commit_sha,
+ head_ref=pr.target_ref_name,
+ base_ref=pr.source_ref_name,
+ base_commit_sha=pr.source_commit_sha,
+ head_commit_sha=pr.target_commit_sha,
+ )
diff --git a/backend/python/pydevlake/poetry.lock b/backend/python/plugins/azure/poetry.lock
similarity index 59%
copy from backend/python/pydevlake/poetry.lock
copy to backend/python/plugins/azure/poetry.lock
index 7c89577ec..dd9b5118d 100644
--- a/backend/python/pydevlake/poetry.lock
+++ b/backend/python/plugins/azure/poetry.lock
@@ -4,7 +4,7 @@
name = "attrs"
version = "22.2.0"
description = "Classes Without Boilerplate"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.6"
files = [
@@ -33,107 +33,94 @@ files = [
[[package]]
name = "charset-normalizer"
-version = "3.0.1"
+version = "3.1.0"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false
-python-versions = "*"
+python-versions = ">=3.7.0"
files = [
- {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"},
- {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"},
- {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"},
- {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"},
- {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"},
- {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"},
- {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"},
- {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"},
+ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"},
+ {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"},
+ {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"},
+ {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"},
+ {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"},
+ {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"},
+ {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"},
]
[[package]]
name = "colorama"
version = "0.4.6"
description = "Cross-platform colored terminal text."
-category = "dev"
+category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
files = [
@@ -145,7 +132,7 @@ files = [
name = "exceptiongroup"
version = "1.1.0"
description = "Backport of PEP 654 (exception groups)"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -280,7 +267,7 @@ testing = ["flake8 (<5)", "pygments", "pytest (>=6)", "pytest-black (>=0.3.7)",
name = "iniconfig"
version = "2.0.0"
description = "brain-dead simple config-ini parsing"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -288,6 +275,18 @@ files = [
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
]
+[[package]]
+name = "iso8601"
+version = "1.1.0"
+description = "Simple module to parse ISO 8601 dates"
+category = "main"
+optional = false
+python-versions = ">=3.6.2,<4.0"
+files = [
+ {file = "iso8601-1.1.0-py3-none-any.whl", hash = "sha256:8400e90141bf792bce2634df533dc57e3bee19ea120a87bebcd3da89a58ad73f"},
+ {file = "iso8601-1.1.0.tar.gz", hash = "sha256:32811e7b81deee2063ea6d2e94f8819a86d1f3811e49d23623a41fa832bef03f"},
+]
+
[[package]]
name = "mysqlclient"
version = "2.1.1"
@@ -309,7 +308,7 @@ files = [
name = "packaging"
version = "23.0"
description = "Core utilities for Python packages"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -321,7 +320,7 @@ files = [
name = "pluggy"
version = "1.0.0"
description = "plugin and hook calling mechanisms for python"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.6"
files = [
@@ -335,48 +334,48 @@ testing = ["pytest", "pytest-benchmark"]
[[package]]
name = "pydantic"
-version = "1.10.4"
+version = "1.10.5"
description = "Data validation and settings management using python type hints"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
- {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"},
- {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"},
- {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"},
- {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"},
- {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"},
- {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"},
- {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"},
- {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"},
- {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"},
- {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"},
- {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"},
- {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"},
- {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"},
- {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"},
- {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"},
- {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"},
- {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"},
- {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"},
- {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"},
- {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"},
- {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"},
- {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"},
- {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"},
- {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"},
- {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"},
- {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"},
- {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"},
- {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"},
- {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"},
- {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"},
- {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"},
- {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"},
- {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"},
- {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"},
- {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"},
- {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"},
+ {file = "pydantic-1.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5920824fe1e21cbb3e38cf0f3dd24857c8959801d1031ce1fac1d50857a03bfb"},
+ {file = "pydantic-1.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3bb99cf9655b377db1a9e47fa4479e3330ea96f4123c6c8200e482704bf1eda2"},
+ {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2185a3b3d98ab4506a3f6707569802d2d92c3a7ba3a9a35683a7709ea6c2aaa2"},
+ {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f582cac9d11c227c652d3ce8ee223d94eb06f4228b52a8adaafa9fa62e73d5c9"},
+ {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c9e5b778b6842f135902e2d82624008c6a79710207e28e86966cd136c621bfee"},
+ {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72ef3783be8cbdef6bca034606a5de3862be6b72415dc5cb1fb8ddbac110049a"},
+ {file = "pydantic-1.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:45edea10b75d3da43cfda12f3792833a3fa70b6eee4db1ed6aed528cef17c74e"},
+ {file = "pydantic-1.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63200cd8af1af2c07964546b7bc8f217e8bda9d0a2ef0ee0c797b36353914984"},
+ {file = "pydantic-1.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:305d0376c516b0dfa1dbefeae8c21042b57b496892d721905a6ec6b79494a66d"},
+ {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd326aff5d6c36f05735c7c9b3d5b0e933b4ca52ad0b6e4b38038d82703d35b"},
+ {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6bb0452d7b8516178c969d305d9630a3c9b8cf16fcf4713261c9ebd465af0d73"},
+ {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9a9d9155e2a9f38b2eb9374c88f02fd4d6851ae17b65ee786a87d032f87008f8"},
+ {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f836444b4c5ece128b23ec36a446c9ab7f9b0f7981d0d27e13a7c366ee163f8a"},
+ {file = "pydantic-1.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:8481dca324e1c7b715ce091a698b181054d22072e848b6fc7895cd86f79b4449"},
+ {file = "pydantic-1.10.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:87f831e81ea0589cd18257f84386bf30154c5f4bed373b7b75e5cb0b5d53ea87"},
+ {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ce1612e98c6326f10888df951a26ec1a577d8df49ddcaea87773bfbe23ba5cc"},
+ {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58e41dd1e977531ac6073b11baac8c013f3cd8706a01d3dc74e86955be8b2c0c"},
+ {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6a4b0aab29061262065bbdede617ef99cc5914d1bf0ddc8bcd8e3d7928d85bd6"},
+ {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:36e44a4de37b8aecffa81c081dbfe42c4d2bf9f6dff34d03dce157ec65eb0f15"},
+ {file = "pydantic-1.10.5-cp37-cp37m-win_amd64.whl", hash = "sha256:261f357f0aecda005934e413dfd7aa4077004a174dafe414a8325e6098a8e419"},
+ {file = "pydantic-1.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b429f7c457aebb7fbe7cd69c418d1cd7c6fdc4d3c8697f45af78b8d5a7955760"},
+ {file = "pydantic-1.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:663d2dd78596c5fa3eb996bc3f34b8c2a592648ad10008f98d1348be7ae212fb"},
+ {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51782fd81f09edcf265823c3bf43ff36d00db246eca39ee765ef58dc8421a642"},
+ {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c428c0f64a86661fb4873495c4fac430ec7a7cef2b8c1c28f3d1a7277f9ea5ab"},
+ {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:76c930ad0746c70f0368c4596020b736ab65b473c1f9b3872310a835d852eb19"},
+ {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3257bd714de9db2102b742570a56bf7978e90441193acac109b1f500290f5718"},
+ {file = "pydantic-1.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:f5bee6c523d13944a1fdc6f0525bc86dbbd94372f17b83fa6331aabacc8fd08e"},
+ {file = "pydantic-1.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:532e97c35719f137ee5405bd3eeddc5c06eb91a032bc755a44e34a712420daf3"},
+ {file = "pydantic-1.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ca9075ab3de9e48b75fa8ccb897c34ccc1519177ad8841d99f7fd74cf43be5bf"},
+ {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd46a0e6296346c477e59a954da57beaf9c538da37b9df482e50f836e4a7d4bb"},
+ {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3353072625ea2a9a6c81ad01b91e5c07fa70deb06368c71307529abf70d23325"},
+ {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3f9d9b2be177c3cb6027cd67fbf323586417868c06c3c85d0d101703136e6b31"},
+ {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b473d00ccd5c2061fd896ac127b7755baad233f8d996ea288af14ae09f8e0d1e"},
+ {file = "pydantic-1.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:5f3bc8f103b56a8c88021d481410874b1f13edf6e838da607dcb57ecff9b4594"},
+ {file = "pydantic-1.10.5-py3-none-any.whl", hash = "sha256:7c5b94d598c90f2f46b3a983ffb46ab806a67099d118ae0da7ef21a2a4033b28"},
+ {file = "pydantic-1.10.5.tar.gz", hash = "sha256:9e337ac83686645a46db0e825acceea8e02fca4062483f40e9ae178e8bd1103a"},
]
[package.dependencies]
@@ -388,25 +387,49 @@ email = ["email-validator (>=1.0.3)"]
[[package]]
name = "pydevd-pycharm"
-version = "231.6471.3"
+version = "231.7864.77"
description = "PyCharm Debugger (used in PyCharm and PyDev)"
category = "main"
optional = false
python-versions = "*"
files = [
- {file = "pydevd-pycharm-231.6471.3.tar.gz", hash = "sha256:3c77340fd1f7e984790081e0a9e89e0284450ce242df7d01aecabbd24e4e46ae"},
+ {file = "pydevd-pycharm-231.7864.77.tar.gz", hash = "sha256:934288028c5fb452f13585a5994c1f6664d899f067d86e74a276e8edba451a4e"},
]
+[[package]]
+name = "pydevlake"
+version = "0.1.0"
+description = "Devlake plugin framework"
+category = "main"
+optional = false
+python-versions = "^3.10"
+files = []
+develop = false
+
+[package.dependencies]
+fire = "^0.4.0"
+inflect = "^6.0.2"
+mysqlclient = "^2.1.1"
+pydantic = "^1.10.2"
+pydevd-pycharm = "^231.6471.3"
+pytest = "^7.2.2"
+requests = "^2.28.1"
+sqlmodel = "^0.0.8"
+
+[package.source]
+type = "directory"
+url = "../../pydevlake"
+
[[package]]
name = "pytest"
-version = "7.2.1"
+version = "7.2.2"
description = "pytest: simple powerful testing with Python"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
- {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"},
- {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"},
+ {file = "pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"},
+ {file = "pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"},
]
[package.dependencies]
@@ -511,7 +534,7 @@ greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platfo
[package.extras]
aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
-aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"]
+aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"]
asyncio = ["greenlet (!=0.4.17)"]
asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"]
mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"]
@@ -521,14 +544,14 @@ mssql-pyodbc = ["pyodbc"]
mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"]
mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"]
mysql-connector = ["mysql-connector-python"]
-oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"]
+oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"]
postgresql = ["psycopg2 (>=2.7)"]
postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"]
postgresql-psycopg2binary = ["psycopg2-binary"]
postgresql-psycopg2cffi = ["psycopg2cffi"]
pymysql = ["pymysql", "pymysql (<1)"]
-sqlcipher = ["sqlcipher3_binary"]
+sqlcipher = ["sqlcipher3-binary"]
[[package]]
name = "sqlalchemy2-stubs"
@@ -581,7 +604,7 @@ tests = ["pytest", "pytest-cov"]
name = "tomli"
version = "2.0.1"
description = "A lil' TOML parser"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -591,14 +614,14 @@ files = [
[[package]]
name = "typing-extensions"
-version = "4.4.0"
+version = "4.5.0"
description = "Backported and Experimental Type Hints for Python 3.7+"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
- {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"},
- {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"},
+ {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"},
+ {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"},
]
[[package]]
@@ -621,4 +644,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
-content-hash = "e4293f87235f9c63a69951ac8ea600dd64719fa7f28c059decad45974adc5222"
+content-hash = "f850c5453c713910c5835d6cd74afc7f036ecdabc4cedc7b9c5934675d5b5cbc"
diff --git a/backend/python/pydevlake/pyproject.toml b/backend/python/plugins/azure/pyproject.toml
similarity index 74%
copy from backend/python/pydevlake/pyproject.toml
copy to backend/python/plugins/azure/pyproject.toml
index e46ee96e5..97fe442ec 100644
--- a/backend/python/pydevlake/pyproject.toml
+++ b/backend/python/plugins/azure/pyproject.toml
@@ -14,26 +14,20 @@
# limitations under the License.
[tool.poetry]
-name = "pydevlake"
+name = "azure"
version = "0.1.0"
-description = "Devlake plugin framework"
-authors = ["Camille Teruel <ca...@meri.co>", "Keon Amini <ke...@merico.dev>"]
-license = "Apache-2.0"
+description = ""
+authors = ["Hezheng Yin <he...@merico.dev>"]
readme = "README.md"
[tool.poetry.dependencies]
python = "^3.10"
-sqlmodel = "^0.0.8"
-mysqlclient = "^2.1.1"
-requests = "^2.28.1"
-inflect = "^6.0.2"
-fire = "^0.4.0"
-pydantic = "^1.10.2"
-pydevd-pycharm = "^231.6471.3"
+pydevlake = { path = "../../pydevlake", develop = false }
+iso8601 = "^1.1.0"
[tool.poetry.group.dev.dependencies]
-pytest = "^7.2.0"
+pytest = "^7.2.2"
[build-system]
requires = ["poetry-core"]
diff --git a/backend/python/plugins/azure/tests/test_streams.py b/backend/python/plugins/azure/tests/test_streams.py
new file mode 100644
index 000000000..467d0e76a
--- /dev/null
+++ b/backend/python/plugins/azure/tests/test_streams.py
@@ -0,0 +1,185 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from pydevlake.testing import assert_convert
+import pydevlake.domain_layer.code as code
+import pydevlake.domain_layer.devops as devops
+
+from azure.main import AzureDevOpsPlugin
+
+
+def test_builds_stream():
+ raw = {
+ 'properties': {},
+ 'tags': [],
+ 'validationResults': [],
+ 'plans': [{'planId': 'c672e778-a9e9-444a-b1e0-92f839c061e0'}],
+ 'triggerInfo': {},
+ 'id': 12,
+ 'buildNumber': 'azure-job',
+ 'status': 'completed',
+ 'result': 'succeeded',
+ 'queueTime': '2023-02-25T06:22:21.2237625Z',
+ 'startTime': '2023-02-25T06:22:32.8097789Z',
+ 'finishTime': '2023-02-25T06:23:04.0061884Z',
+ 'url': 'https://dev.azure.com/testorg/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/build/Builds/12',
+ 'definition': {
+ 'drafts': [],
+ 'id': 5,
+ 'name': 'johndoe.test-repo',
+ 'url': 'https://dev.azure.com/testorg/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/build/Definitions/5?revision=1',
+ 'uri': 'vstfs:///Build/Definition/5',
+ 'path': '\\',
+ 'type': 'build',
+ 'queueStatus': 'enabled',
+ 'revision': 1,
+ 'project': {
+ 'id': '7a3fd40e-2aed-4fac-bac9-511bf1a70206',
+ 'name': 'test-project',
+ 'url': 'https://dev.azure.com/testorg/_apis/projects/7a3fd40e-2aed-4fac-bac9-511bf1a70206',
+ 'state': 'wellFormed',
+ 'revision': 11,
+ 'visibility': 'private',
+ 'lastUpdateTime': '2023-01-26T19:38:04.267Z'
+ }
+ },
+ 'project': {
+ 'id': '7a3fd40e-2aed-4fac-bac9-511bf1a70206',
+ 'name': 'Test project',
+ 'url': 'https://dev.azure.com/testorg/_apis/projects/7a3fd40e-2aed-4fac-bac9-511bf1a70206',
+ },
+ 'uri': 'vstfs:///Build/Build/12',
+ 'sourceBranch': 'refs/heads/main',
+ 'sourceVersion': '40c59264e73fc5e1a6cab192f1622d26b7bd5c2a',
+ 'queue': {
+ 'id': 9,
+ 'name': 'Azure Pipelines',
+ 'pool': {'id': 9, 'name': 'Azure Pipelines', 'isHosted': True}
+ },
+ 'priority': 'normal',
+ 'reason': 'manual',
+ 'requestedFor': {
+ 'displayName': 'John Doe',
+ 'url': 'https://spsprodcus5.vssps.visualstudio.com/A1def512a-251e-4668-9a5d-a4bc1f0da4aa/_apis/Identities/bc538feb-9fdd-6cf8-80e1-7c56950d0289',
+ 'id': 'bc538feb-9fdd-6cf8-80e1-7c56950d0289',
+ 'uniqueName': 'john.dow@merico.dev',
+ 'imageUrl': 'https://dev.azure.com/testorg/_apis/GraphProfile/MemberAvatars/aad.YmM1MzhmZWItOWZkZC03Y2Y4LTgwZTEtN2M1Njk1MGQwMjg5',
+ 'descriptor': 'aad.YmM1MzhmZWItOWZkZC03Y2Y4LTgwZTEtN2M1Njk1MGQwMjg5'
+ },
+ 'requestedBy': {
+ 'displayName': 'John Doe',
+ 'url': 'https://spsprodcus5.vssps.visualstudio.com/A1def512a-251e-4668-9a5d-a4bc1f0da4aa/_apis/Identities/bc538feb-9fdd-6cf8-80e1-7c56950d0289',
+ 'id': 'bc538feb-9fdd-6cf8-80e1-7c56950d0289',
+ 'uniqueName': 'john.doe@merico.dev',
+ 'imageUrl': 'https://dev.azure.com/testorg/_apis/GraphProfile/MemberAvatars/aad.YmM1MzhmZWItOWZkZC03Y2Y4LTgwZTEtN2M1Njk1MGQwMjg5',
+ 'descriptor': 'aad.YmM1MzhmZWItOWZkZC03Y2Y4LTgwZTEtN2M1Njk1MGQwMjg5'
+ },
+ 'lastChangedDate': '2023-02-25T06:23:04.343Z',
+ 'lastChangedBy': {
+ 'displayName': 'Microsoft.VisualStudio.Services.TFS',
+ 'url': 'https://spsprodcus5.vssps.visualstudio.com/A1def512a-251e-4668-9a5d-a4bc1f0da4aa/_apis/Identities/00000002-0000-8888-8000-000000000000',
+ 'id': '00000002-0000-8888-8000-000000000000',
+ 'uniqueName': '00000002-0000-8888-8000-000000000000@2c895908-04e0-4952-89fd-54b0046d6288',
+ 'imageUrl': 'https://dev.azure.com/testorg/_apis/GraphProfile/MemberAvatars/s2s.MDAwMDAwMDItMDAwMC04ODg4LTgwMDAtMDAwMDAwMDAwMDAwQDJjODk1OTA4LTA0ZTAtNDk1Mi04OWZkLTU0YjAwNDZkNjI4OA',
+ 'descriptor': 's2s.MDAwMDAwMDItMDAwMC04ODg4LTgwMDAtMDAwMDAwMDAwMDAwQDJjODk1OTA4LTA0ZTAtNDk1Mi04OWZkLTU0YjAwNDZkNjI4OA'
+ },
+ 'orchestrationPlan': {'planId': 'c672e778-a9e9-444a-b1e0-92f839c061e0'},
+ 'logs': {'id': 0,
+ 'type': 'Container',
+ 'url': 'https://dev.azure.com/testorg/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/build/builds/12/logs'
+ },
+ 'repository': {
+ 'id': 'johndoe/test-repo',
+ 'type': 'GitHub'
+ },
+ 'retainedByRelease': False,
+ 'triggeredByBuild': None,
+ 'appendCommitMessageToRunName': True
+ }
+
+ expected = [
+ devops.CICDPipeline(
+ name=12,
+ status=devops.CICDStatus.DONE,
+ created_date='2023-02-25T06:22:32.8097789Z',
+ finished_date='2023-02-25T06:23:04.0061884Z',
+ result=devops.CICDResult.SUCCESS,
+ duration_sec=28,
+ environment=devops.CICDEnvironment.PRODUCTION,
+ type=devops.CICDType.DEPLOYMENT,
+ cicd_scope_id='johndoe/test-repo'
+ ),
+ devops.CiCDPipelineCommit(
+ pipeline_id=12,
+ commit_sha='40c59264e73fc5e1a6cab192f1622d26b7bd5c2a',
+ branch='refs/heads/main',
+ repo_id='johndoe/test-repo',
+ repo='https://github.com/johndoe/test-repo'
+ )
+ ]
+
+ assert_convert(AzureDevOpsPlugin, 'builds', raw, expected)
+
+
+def test_jobs_stream():
+ raw = {
+ 'previousAttempts': [],
+ 'id': 'cfa20e98-6997-523c-4233-f0a7302c929f',
+ 'parentId': '9ecf18fe-987d-5811-7c63-300aecae35da',
+ 'type': 'Job',
+ 'name': 'job_2',
+ 'build_id': 12, # Added by collector,
+ 'repo_id': 'johndoe/test-repo', # Added by collector,
+ 'startTime': '2023-02-25T06:22:36.8066667Z',
+ 'finishTime': '2023-02-25T06:22:43.2333333Z',
+ 'currentOperation': None,
+ 'percentComplete': None,
+ 'state': 'completed',
+ 'result': 'succeeded',
+ 'resultCode': None,
+ 'changeId': 18,
+ 'lastModified': '0001-01-01T00:00:00',
+ 'workerName': 'Hosted Agent',
+ 'queueId': 9,
+ 'order': 1,
+ 'details': None,
+ 'errorCount': 0,
+ 'warningCount': 0,
+ 'url': None,
+ 'log': {'id': 10,
+ 'type': 'Container',
+ 'url': 'https://dev.azure.com/johndoe/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/build/builds/12/logs/10'},
+ 'task': None,
+ 'attempt': 1,
+ 'identifier': 'job_2.__default'
+ }
+
+ expected = devops.CICDTask(
+ id='cfa20e98-6997-523c-4233-f0a7302c929f',
+ name='job_2',
+ pipeline_id=12,
+ status=devops.CICDStatus.DONE,
+ created_date='2023-02-25T06:22:36.8066667Z',
+ finished_date='2023-02-25T06:22:43.2333333Z',
+ result=devops.CICDResult.SUCCESS,
+ type=devops.CICDType.BUILD,
+ duration_sec=7,
+ environment=devops.CICDEnvironment.PRODUCTION,
+ cicd_scope_id='johndoe/test-repo'
+ )
+
+ assert_convert(AzureDevOpsPlugin, 'jobs', raw, expected)
diff --git a/backend/python/pydevlake/README.md b/backend/python/pydevlake/README.md
index c30a5bf8d..7c7d2b36c 100644
--- a/backend/python/pydevlake/README.md
+++ b/backend/python/pydevlake/README.md
@@ -111,7 +111,7 @@ class Users(Stream):
def collect(self, state, context) -> Iterable[Tuple[object, dict]]:
pass
- def convert(self, user: ToolUser) -> Iterable[DomainUser]:
+ def convert(self, user: ToolUser, context) -> Iterable[DomainUser]:
pass
```
@@ -130,7 +130,7 @@ The `convert` method takes a tool-specific user model and convert it into domain
Here the two models align quite well, the conversion is trivial:
```python
-def convert(self, user: ToolUser) -> Iterable[DomainUser]:
+def convert(self, user: ToolUser, context: Context) -> Iterable[DomainUser]:
yield DomainUser(
id=user.id,
name=user.name
diff --git a/backend/python/pydevlake/poetry.lock b/backend/python/pydevlake/poetry.lock
index 7c89577ec..f5dd4e9a7 100644
--- a/backend/python/pydevlake/poetry.lock
+++ b/backend/python/pydevlake/poetry.lock
@@ -4,7 +4,7 @@
name = "attrs"
version = "22.2.0"
description = "Classes Without Boilerplate"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.6"
files = [
@@ -133,7 +133,7 @@ files = [
name = "colorama"
version = "0.4.6"
description = "Cross-platform colored terminal text."
-category = "dev"
+category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
files = [
@@ -145,7 +145,7 @@ files = [
name = "exceptiongroup"
version = "1.1.0"
description = "Backport of PEP 654 (exception groups)"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -280,7 +280,7 @@ testing = ["flake8 (<5)", "pygments", "pytest (>=6)", "pytest-black (>=0.3.7)",
name = "iniconfig"
version = "2.0.0"
description = "brain-dead simple config-ini parsing"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -309,7 +309,7 @@ files = [
name = "packaging"
version = "23.0"
description = "Core utilities for Python packages"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -321,7 +321,7 @@ files = [
name = "pluggy"
version = "1.0.0"
description = "plugin and hook calling mechanisms for python"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.6"
files = [
@@ -399,14 +399,14 @@ files = [
[[package]]
name = "pytest"
-version = "7.2.1"
+version = "7.2.2"
description = "pytest: simple powerful testing with Python"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
- {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"},
- {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"},
+ {file = "pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"},
+ {file = "pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"},
]
[package.dependencies]
@@ -511,7 +511,7 @@ greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platfo
[package.extras]
aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
-aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"]
+aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"]
asyncio = ["greenlet (!=0.4.17)"]
asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"]
mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"]
@@ -521,14 +521,14 @@ mssql-pyodbc = ["pyodbc"]
mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"]
mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"]
mysql-connector = ["mysql-connector-python"]
-oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"]
+oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"]
postgresql = ["psycopg2 (>=2.7)"]
postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"]
postgresql-psycopg2binary = ["psycopg2-binary"]
postgresql-psycopg2cffi = ["psycopg2cffi"]
pymysql = ["pymysql", "pymysql (<1)"]
-sqlcipher = ["sqlcipher3_binary"]
+sqlcipher = ["sqlcipher3-binary"]
[[package]]
name = "sqlalchemy2-stubs"
@@ -581,7 +581,7 @@ tests = ["pytest", "pytest-cov"]
name = "tomli"
version = "2.0.1"
description = "A lil' TOML parser"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.7"
files = [
@@ -621,4 +621,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
-content-hash = "e4293f87235f9c63a69951ac8ea600dd64719fa7f28c059decad45974adc5222"
+content-hash = "cbf8b5f2a4bafcf413a43bd03dd6849fda956818b66bed0f3dd747968006fc7c"
diff --git a/backend/python/pydevlake/pydevlake/api.py b/backend/python/pydevlake/pydevlake/api.py
index bc29fc474..14772d0c9 100644
--- a/backend/python/pydevlake/pydevlake/api.py
+++ b/backend/python/pydevlake/pydevlake/api.py
@@ -17,7 +17,6 @@
from __future__ import annotations
from typing import Optional, Union
-from urllib.parse import urljoin
from http import HTTPStatus
import json
import time
@@ -91,8 +90,8 @@ class APIBase:
return self._session
@property
- def base_url(self):
- pass
+ def base_url(self) -> Optional[str]:
+ return None
def send(self, request: Request):
request = self._apply_hooks(request, self.request_hooks())
@@ -126,8 +125,10 @@ class APIBase:
target = result
return target
- def get(self, path, **query_args):
- req = Request(urljoin(self.base_url, path), query_args)
+ def get(self, *path_args, **query_args):
+ parts = [self.base_url, *path_args] if self.base_url else path_args
+ url = "/".join([a.strip('/') for a in parts])
+ req = Request(url, query_args)
return self.send(req)
def request_hooks(self):
diff --git a/backend/python/pydevlake/pydevlake/context.py b/backend/python/pydevlake/pydevlake/context.py
index 043bb0636..3133ec79f 100644
--- a/backend/python/pydevlake/pydevlake/context.py
+++ b/backend/python/pydevlake/pydevlake/context.py
@@ -23,12 +23,12 @@ from pydevlake.model import Connection, TransformationRule
class Context:
def __init__(self,
db_url: str,
- scope_id: str,
+ scope: str,
connection: Connection,
transformation_rule: TransformationRule = None,
options: dict = None):
self.db_url = db_url
- self.scope_id = scope_id
+ self.scope = scope
self.connection = connection
self.transformation_rule = transformation_rule
self.options = options or {}
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/code.py b/backend/python/pydevlake/pydevlake/domain_layer/code.py
new file mode 100644
index 000000000..0ca38bab9
--- /dev/null
+++ b/backend/python/pydevlake/pydevlake/domain_layer/code.py
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from datetime import datetime
+from typing import Optional
+
+from sqlmodel import Field
+
+from pydevlake.model import DomainModel, NoPKModel
+
+
+class PullRequest(DomainModel, table=True):
+ __tablename__ = 'pull_requests'
+ base_repo_id: str
+ head_repo_id: str
+ status: str
+ title: str
+ description: str
+ url: str
+ author_name: str
+ author_id: str
+ parent_pr_id: Optional[str]
+ pull_request_key: int
+ created_date: datetime
+ merged_date: Optional[datetime]
+ closed_date: datetime
+ type: str
+ component: str
+ merge_commit_sha: str
+ head_ref: str
+ base_ref: str
+ base_commit_sha: str
+ head_commit_sha: str
+
+
+class PullRequestLabels(NoPKModel, table=True):
+ __tablename__ = 'pull_request_labels'
+ pull_request_id: str = Field(primary_key=True)
+ label_name: str
+
+
+class PullRequestCommit(NoPKModel, table=True):
+ __tablename__ = 'pull_request_commits'
+ commit_sha: str = Field(primary_key=True)
+ pull_request_id: str = Field(primary_key=True)
+
+
+class PullRequestComment(DomainModel, table=True):
+ __tablename__ = 'cicd_scopes'
+ pull_request_id: str
+ body: str
+ account_id: str
+ created_date: datetime
+ commit_sha: str
+ position: int
+ type: str
+ review_id: str
+ status: str
+
+
+class Commit(NoPKModel, table=True):
+ __tablename__ = 'commits'
+ sha: str = Field(primary_key=True)
+ additions: str
+ deletions: str = Optional[str]
+ dev_eq: Optional[str]
+ message: str
+ author_name: str
+ author_email: int
+ authored_date: datetime
+ author_id: str
+ committer_name: str
+ committer_email: str
+ committed_date: datetime
+ committer_id: str
+
+
+class CommitParent(NoPKModel, table=True):
+ __tablename__ = 'commit_parents'
+ commit_sha: str = Field(primary_key=True)
+ parent_commit_sha: str
+
+
+class CommitsDiff(DomainModel, table=True):
+ __tablename__ = 'commits_diffs'
+ new_commit_sha: str = Field(primary_key=True)
+ old_commit_sha: str = Field(primary_key=True)
+ commit_sha: str = Field(primary_key=True)
+ sorting_index: int
+
+
+class RefCommit(NoPKModel, table=True):
+ __tablename__ = 'ref_commits'
+ new_ref_id: str = Field(primary_key=True)
+ old_ref_id: str = Field(primary_key=True)
+ new_commit_sha: str
+ old_commit_sha: str
+
+
+class FinishedCommitsDiff(NoPKModel, table=True):
+ __tablename__ = 'finished_commits_diffs'
+ new_commit_sha: str = Field(primary_key=True)
+ old_commit_sha: str = Field(primary_key=True)
+
+
+class Component(NoPKModel, table=True):
+ __tablename__ = 'components'
+ repo_id: str
+ name: str = Field(primary_key=True)
+ path_regex: str
+
+
+class Ref(DomainModel, table=True):
+ __tablename__ = "refs"
+ repo_id: str
+ name: str
+ commit_sha: str
+ is_default: bool
+ ref_type: str
+ created_date: datetime
+
+
+class RefsPrCherryPick(DomainModel, table=True):
+ __tablename__ = "refs_pr_cherrypicks"
+ repo_name: str
+ parent_pr_key: int
+ cherrypick_base_branches: str
+ cherrypick_pr_keys: str
+ parent_pr_url: str
+ parent_pr_id: str = Field(primary_key=True)
+
+
+class Repo(DomainModel, table=True):
+ __tablename__ = "repos"
+ name: str
+ url: str
+ description: Optional[str]
+ owner_id: Optional[str]
+ language: Optional[str]
+ forked_from: Optional[str]
+ created_date: Optional[datetime]
+ updated_date: Optional[datetime]
+ deleted: bool
+
+
+class RepoLanguage(NoPKModel, table=True):
+ __tablename__ = "repo_languages"
+ repo_id: str = Field(primary_key=True)
+ language: str
+ bytes: int
+
+
+class RepoCommit(NoPKModel, table=True):
+ __tablename__ = "repo_commits"
+ repo_id: str = Field(primary_key=True)
+ commit_sha: str = Field(primary_key=True)
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py b/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
index b001531df..3d7d83e7a 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
+++ b/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
@@ -13,12 +13,108 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from datetime import datetime
-from pydevlake.model import DomainModel
+from sqlmodel import Field
+from pydevlake.model import DomainModel, NoPKModel
+
+
+class Account(DomainModel, table=True):
+ __tablename__ = "accounts"
+ email: str
+ full_name: str
+ user_name: str
+ avatar_url: str
+ organization: str
+ created_date: datetime
+ status: int
+
+
+class BoardRepo(NoPKModel, table=True):
+ __tablename__ = 'board_repos'
+ board_id: str = Field(primary_key=True)
+ repo_id: str = Field(primary_key=True)
+
+
+class IssueCommit(NoPKModel, table=True):
+ __tablename__ = 'issue_commits'
+ issue_id: str = Field(primary_key=True)
+ commit_sha: str = Field(primary_key=True)
+
+
+class IssueRepoCommit(NoPKModel, table=True):
+ __tablename__ = 'issue_repo_commits'
+ issue_id: str = Field(primary_key=True)
+ repo_url: str = Field(primary_key=True)
+ commit_sha: str = Field(primary_key=True)
+
+
+class ProjectIssueMetric(NoPKModel, table=True):
+ __tablename__ = "project_issue_metrics"
+ project_name: str = Field(primary_key=True)
+ deployment_id: str
+
+
+class ProjectMapping(NoPKModel, table=True):
+ __tablename__ = "project_mappings"
+ project_name: str = Field(primary_key=True)
+ table: str = Field(primary_key=True)
+ row_id: str = Field(primary_key=True)
-class User(DomainModel, table=True):
- __table_name__ = 'users'
+class ProjectPrMetric(DomainModel, table=True):
+ __tablename__ = "project_pr_metrics"
+ project_name: str = Field(primary_key=True)
+ first_commit_sha: str
+ pr_coding_time: int
+ first_review_id: str
+ pr_pick_time: int
+ pr_review_time: int
+ deployment_id: str
+ pr_deploy_time: int
+ pr_cycle_time: int
+
+
+class PullRequestIssue(NoPKModel, table=True):
+ __tablename__ = "pull_request_issues"
+ pull_request_id: str = Field(primary_key=True)
+ issue_id: str = Field(primary_key=True)
+ pull_request_key: int
+ issue_key: int
+
+
+class RefsIssuesDiffs(NoPKModel, table=True):
+ __tablename__ = "refs_issues_diffs"
+ new_ref_id: str = Field(primary_key=True)
+ old_ref_id: str = Field(primary_key=True)
+ new_ref_commit_sha: str
+ old_ref_commit_sha: str
+ issue_number: str
+ issue_id: str = Field(primary_key=True)
+
+
+class Team(DomainModel, table=True):
+ __tablename__ = "teams"
+ name: str
+ alias: str
+ parent_id: str
+ sorting_index: int
+
+
+class TeamUser(NoPKModel, table=True):
+ __tablename__ = "team_users"
+ team_id: str = Field(primary_key=True)
+ user_id: str = Field(primary_key=True)
+
+
+class User(DomainModel, table=True):
+ __tablename__ = 'users'
name: str
email: str
+
+
+class UserAccount(NoPKModel, table=True):
+ __tablename__ = 'user_accounts'
+ user_id: str
+ account_id: str = Field(primary_key=True)
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/devops.py b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
index 4a5fffbdf..2a12d2348 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/devops.py
+++ b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
@@ -18,53 +18,76 @@ from typing import Optional
from datetime import datetime
from enum import Enum
-from sqlmodel import Field, Relationship
+from sqlmodel import Field
-from pydevlake.model import DomainModel, DomainScope, NoPKModel
+from pydevlake.model import DomainModel, NoPKModel
-class CicdScope(DomainScope, table=True):
- __tablename__ = 'cicd_scopes'
+class CICDResult(Enum):
+ SUCCESS = "SUCCESS"
+ FAILURE = "FAILURE"
+ ABORT = "ABORT"
+ MANUAL = "MANUAL"
- name: str
- description: Optional[str]
- url: Optional[str]
- createdDate: Optional[datetime]
- updatedDate: Optional[datetime]
+class CICDStatus(Enum):
+ IN_PROGRESS = "IN_PROGRESS"
+ DONE = "DONE"
-class CICDPipeline(DomainModel, table=True):
- __table_name__ = 'cicd_pipelines'
- class Result(Enum):
- SUCCESS = "SUCCESS"
- FAILURE = "FAILURE"
- ABORT = "ABORT"
- MANUAL = "MANUAL"
+class CICDType(Enum):
+ TEST = "TEST"
+ LINT = "LINT"
+ BUILD = "BUILD"
+ DEPLOYMENT = "DEPLOYMENT"
+
- class Status(Enum):
- IN_PROGRESS = "IN_PROGRESS"
- DONE = "DONE"
- MANUAL = "MANUAL"
+class CICDEnvironment(Enum):
+ PRODUCTION = "PRODUCTION"
+ STAGING = "STAGING"
+ TESTING = "TESTING"
- class Type(Enum):
- CI = "CI"
- CD = "CD"
+class CICDPipeline(DomainModel, table=True):
+ __tablename__ = 'cicd_pipelines'
name: str
- status: Status
- created_date: datetime
+ status: Optional[CICDStatus]
+ created_date: Optional[datetime]
finished_date: Optional[datetime]
- result: Optional[Result]
+ result: Optional[CICDResult]
duration_sec: Optional[int]
environment: Optional[str]
- type: Optional[Type] #Unused
+ type: Optional[CICDType]
+ cicd_scope_id: Optional[str]
+
- # parent_pipelines: list["CICDPipeline"] = Relationship(back_populates="child_pipelines", link_model="CICDPipelineRelationship")
- # child_pipelines: list["CICDPipeline"] = Relationship(back_populates="parent_pipelines", link_model="CICDPipelineRelationship")
+class CiCDPipelineCommit(NoPKModel, table=True):
+ __tablename__ = 'cicd_pipeline_commits'
+ pipeline_id: str = Field(primary_key=True)
+ commit_sha: str = Field(primary_key=True)
+ branch: str
+ repo_id: str
+ repo: str
-class CICDPipelineRelationship(NoPKModel):
- __table_name__ = 'cicd_pipeline_relationships'
- parent_pipeline_id: str = Field(primary_key=True, foreign_key=CICDPipeline.id)
- child_pipeline_id: str = Field(primary_key=True, foreign_key=CICDPipeline.id)
+class CicdScope(DomainModel):
+ __tablename__ = 'cicd_scopes'
+ name: str
+ description: Optional[str]
+ url: Optional[str]
+ createdDate: Optional[datetime]
+ updatedDate: Optional[datetime]
+
+
+class CICDTask(DomainModel, table=True):
+ __tablename__ = 'cicd_tasks'
+ name: str
+ pipeline_id: str
+ result: str = Optional[CICDResult]
+ status: Optional[CICDStatus]
+ type: Optional[CICDType]
+ environment: Optional[CICDEnvironment]
+ duration_sec: int
+ started_date: Optional[datetime]
+ finished_date: Optional[datetime]
+ cicd_scope_id: str
diff --git a/backend/python/pydevlake/pydevlake/ipc.py b/backend/python/pydevlake/pydevlake/ipc.py
index aa2811481..a105d5047 100644
--- a/backend/python/pydevlake/pydevlake/ipc.py
+++ b/backend/python/pydevlake/pydevlake/ipc.py
@@ -71,9 +71,9 @@ class PluginCommands:
self._plugin.test_connection(connection)
@plugin_method
- def make_pipeline(self, scopes: list[dict]):
+ def make_pipeline(self, scopes: list[dict], connection: dict):
s = [self._plugin.tool_scope_type(**data) for data in scopes]
- return self._plugin.make_pipeline(s)
+ return self._plugin.make_pipeline(s, connection['id'])
@plugin_method
def run_migrations(self, force: bool):
@@ -93,11 +93,11 @@ class PluginCommands:
def _mk_context(self, data: dict):
db_url = data['db_url']
- scope_id = data['scope_id']
+ scope = self._plugin.tool_scope_type(**data['scope'])
connection = self._plugin.connection_type(**data['connection'])
if self._plugin.transformation_rule_type:
transformation_rule = self._plugin.transformation_rule_type(**data['transformation_rule'])
else:
transformation_rule = None
options = data.get('options', {})
- return Context(db_url, scope_id, connection, transformation_rule, options)
+ return Context(db_url, scope, connection, transformation_rule, options)
diff --git a/backend/python/pydevlake/pydevlake/plugin.py b/backend/python/pydevlake/pydevlake/plugin.py
index 9b21f6e6a..14af2000a 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -118,11 +118,11 @@ class Plugin(ABC):
else:
return self.remote_scope_groups(connection)
- def make_pipeline(self, tool_scopes: list[ToolScope]):
+ def make_pipeline(self, tool_scopes: list[ToolScope], connection_id: int):
"""
Make a simple pipeline using the scopes declared by the plugin.
"""
- plan = self.make_pipeline_plan(tool_scopes)
+ plan = self.make_pipeline_plan(tool_scopes, connection_id)
domain_scopes = [
msg.DynamicDomainScope(
type_name=type(scope).__name__,
@@ -136,10 +136,10 @@ class Plugin(ABC):
scopes=domain_scopes
)
- def make_pipeline_plan(self, scopes: list[ToolScope]) -> list[list[msg.PipelineTask]]:
- return [self.make_pipeline_stage(scope) for scope in scopes]
+ def make_pipeline_plan(self, scopes: list[ToolScope], connection_id: int) -> list[list[msg.PipelineTask]]:
+ return [self.make_pipeline_stage(scope, connection_id) for scope in scopes]
- def make_pipeline_stage(self, scope: ToolScope) -> list[msg.PipelineTask]:
+ def make_pipeline_stage(self, scope: ToolScope, connection_id: int) -> list[msg.PipelineTask]:
return [
msg.PipelineTask(
plugin=self.name,
@@ -147,7 +147,8 @@ class Plugin(ABC):
subtasks=[t.name for t in self.subtasks],
options={
"scopeId": scope.id,
- "scopeName": scope.name
+ "scopeName": scope.name,
+ "connectionId": connection_id
}
)
]
diff --git a/backend/python/pydevlake/pydevlake/stream.py b/backend/python/pydevlake/pydevlake/stream.py
index 4072777cc..108dc106e 100644
--- a/backend/python/pydevlake/pydevlake/stream.py
+++ b/backend/python/pydevlake/pydevlake/stream.py
@@ -18,6 +18,7 @@ from typing import Iterable, Type
from abc import abstractmethod
from enum import Enum
+from pydevlake.context import Context
from pydevlake.subtasks import Collector, Extractor, Convertor, SubstreamCollector
from pydevlake.model import RawModel, ToolModel, DomainModel
@@ -87,7 +88,7 @@ class Stream:
def extract(self, raw_data: dict) -> ToolModel:
return self.tool_model(**raw_data)
- def convert(self, tool_model: ToolModel) -> DomainModel:
+ def convert(self, tool_model: ToolModel, context: Context) -> DomainModel:
pass
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py b/backend/python/pydevlake/pydevlake/subtasks.py
index 865b930ab..e6ff57fa0 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -152,7 +152,7 @@ class Collector(Subtask):
def _params(self, ctx: Context) -> str:
return json.dumps({
"connection_id": ctx.connection.id,
- "scope_id": ctx.scope_id
+ "scope_id": ctx.scope.id
})
def delete(self, session, ctx):
@@ -196,9 +196,9 @@ class Convertor(Subtask):
yield item, state
def process(self, tool_model: ToolModel, session: Session, ctx: Context):
- res = self.stream.convert(tool_model)
+ res = self.stream.convert(tool_model, ctx)
if isinstance(res, Generator):
- for each in self.stream.convert(tool_model):
+ for each in res:
self._save(tool_model, each, session, ctx.connection.id)
else:
self._save(tool_model, res, session, ctx.connection.id)
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py b/backend/python/pydevlake/pydevlake/testing/__init__.py
similarity index 84%
copy from backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
copy to backend/python/pydevlake/pydevlake/testing/__init__.py
index b001531df..2404776e3 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
+++ b/backend/python/pydevlake/pydevlake/testing/__init__.py
@@ -13,12 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import pytest
+pytest.register_assert_rewrite('pydevlake.testing')
-from pydevlake.model import DomainModel
-
-
-class User(DomainModel, table=True):
- __table_name__ = 'users'
-
- name: str
- email: str
+from .testing import assert_convert
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py b/backend/python/pydevlake/pydevlake/testing/testing.py
similarity index 55%
copy from backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
copy to backend/python/pydevlake/pydevlake/testing/testing.py
index b001531df..88138bff8 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
+++ b/backend/python/pydevlake/pydevlake/testing/testing.py
@@ -13,12 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import pytest
-from pydevlake.model import DomainModel
+from typing import Union, Type, Iterable
+from pydevlake.plugin import Plugin
+from pydevlake.model import DomainModel
-class User(DomainModel, table=True):
- __table_name__ = 'users'
- name: str
- email: str
+def assert_convert(plugin: Union[Plugin, Type[Plugin]], stream_name: str, raw: dict, expected: Union[DomainModel, Iterable[DomainModel]]):
+ if isinstance(plugin, type):
+ plugin = plugin()
+ stream = plugin.get_stream(stream_name)
+ tool_model = stream.extract(raw)
+ domain_models = stream.convert(tool_model, None)
+ if not isinstance(expected, Iterable):
+ expected = [expected]
+ if not isinstance(domain_models, Iterable):
+ domain_models = [domain_models]
+ for res, exp in zip(domain_models, expected):
+ assert res == exp
diff --git a/backend/python/pydevlake/pyproject.toml b/backend/python/pydevlake/pyproject.toml
index e46ee96e5..25341c1f2 100644
--- a/backend/python/pydevlake/pyproject.toml
+++ b/backend/python/pydevlake/pyproject.toml
@@ -30,6 +30,7 @@ inflect = "^6.0.2"
fire = "^0.4.0"
pydantic = "^1.10.2"
pydevd-pycharm = "^231.6471.3"
+pytest = "^7.2.2"
[tool.poetry.group.dev.dependencies]
diff --git a/backend/python/pydevlake/test/remote_test.go b/backend/python/pydevlake/test/remote_test.go
deleted file mode 100644
index 3c11ba971..000000000
--- a/backend/python/pydevlake/test/remote_test.go
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package test
-
-import (
- "testing"
-
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/e2ehelper"
- "github.com/apache/incubator-devlake/server/services/remote"
- "github.com/apache/incubator-devlake/server/services/remote/bridge"
- "github.com/apache/incubator-devlake/server/services/remote/models"
- plg "github.com/apache/incubator-devlake/server/services/remote/plugin"
-)
-
-type CircleCIConnection struct {
- ID uint64 `json:"id"`
- Token string `json:"token" encrypt:"yes"`
-}
-
-func CreateRemotePlugin(t *testing.T) models.RemotePlugin {
- // TODO: Create a dummy plugin for tests instead of using CircleCI plugin
- pluginCmdPath := "../../plugins/circle_ci/circle_ci/main.py"
- invoker := bridge.NewPythonPoetryCmdInvoker(pluginCmdPath)
-
- pluginInfo := models.PluginInfo{}
- err := invoker.Call("plugin-info", bridge.DefaultContext).Get(&pluginInfo)
-
- if err != nil {
- t.Error("Cannot get plugin info", err)
- return nil
- }
-
- remotePlugin, err := remote.NewRemotePlugin(&pluginInfo)
- if err != nil {
- t.Error("Cannot create remote plugin", err)
- return nil
- }
-
- return remotePlugin
-}
-
-func TestCreateRemotePlugin(t *testing.T) {
- _ = CreateRemotePlugin(t)
-}
-
-func TestRunSubTask(t *testing.T) {
- remotePlugin := CreateRemotePlugin(t)
- dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", remotePlugin)
- subtask := remotePlugin.SubTaskMetas()[0]
- options := make(map[string]interface{})
- taskData := plg.RemotePluginTaskData{
- DbUrl: bridge.DefaultContext.GetConfig("db_url"),
- ScopeId: "gh/circleci/bond",
- Connection: CircleCIConnection{ID: 1},
- Options: options,
- }
- dataflowTester.Subtask(subtask, taskData)
-}
-
-func TestTestConnection(t *testing.T) {
- remotePlugin := CreateRemotePlugin(t)
-
- var handler plugin.ApiResourceHandler
- for resource, endpoints := range remotePlugin.ApiResources() {
- if resource == "test" {
- handler = endpoints["POST"]
- }
- }
-
- if handler == nil {
- t.Error("Missing test connection API resource")
- }
-
- input := plugin.ApiResourceInput{
- Body: map[string]interface{}{
- "token": "secret",
- },
- }
- _, err := handler(&input)
- if err != nil {
- t.Error(err)
- }
-}
diff --git a/backend/python/pydevlake/test/stream_test.py b/backend/python/pydevlake/test/stream_test.py
index 3aac3c724..4cee8f51b 100644
--- a/backend/python/pydevlake/test/stream_test.py
+++ b/backend/python/pydevlake/test/stream_test.py
@@ -20,7 +20,7 @@ import pytest
from sqlmodel import Session, Field
from pydevlake import Stream, Connection, Context, DomainType
-from pydevlake.model import ToolModel, DomainModel
+from pydevlake.model import ToolModel, DomainModel, ToolScope
class DummyToolModel(ToolModel, table=True):
@@ -47,7 +47,7 @@ class DummyStream(Stream):
name=raw["n"]
)
- def convert(self, tm):
+ def convert(self, tm, ctx):
return DummyDomainModel(
ID=tm.id,
Name=tm.name,
@@ -68,14 +68,19 @@ def raw_data():
@pytest.fixture
def connection(raw_data):
- return DummyConnection(id=11, raw_data=raw_data)
+ return DummyConnection(id=11, name='dummy connection', raw_data=raw_data)
@pytest.fixture
-def ctx(connection):
+def scope():
+ return ToolScope(id='scope_id', name='scope_name')
+
+
+@pytest.fixture
+def ctx(connection, scope):
return Context(
db_url="sqlite+pysqlite:///:memory:",
- scope_id="1",
+ scope=scope,
connection=connection,
options={}
)
@@ -123,7 +128,7 @@ def test_convert_data(stream, raw_data, ctx):
id=each["i"],
name=each["n"],
raw_data_table="_raw_dummy_model",
- raw_data_params=json.dumps({"connection_id": ctx.connection.id, "scope_id": ctx.scope_id})
+ raw_data_params=json.dumps({"connection_id": ctx.connection.id, "scope_id": ctx.scope.id})
)
)
session.commit()
diff --git a/backend/server/services/remote/models/models.go b/backend/server/services/remote/models/models.go
index 7dbcfb659..16887a336 100644
--- a/backend/server/services/remote/models/models.go
+++ b/backend/server/services/remote/models/models.go
@@ -69,9 +69,9 @@ type ScopeModel struct {
type TransformationModel struct {
Id uint64 `gorm:"primaryKey" json:"id"`
+ Name string `json:"name"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
- Name string
}
type SubtaskMeta struct {
diff --git a/backend/server/services/remote/plugin/plugin_extensions.go b/backend/server/services/remote/plugin/plugin_extensions.go
index 61b1e436e..b60beb4b3 100644
--- a/backend/server/services/remote/plugin/plugin_extensions.go
+++ b/backend/server/services/remote/plugin/plugin_extensions.go
@@ -60,7 +60,7 @@ func (p remoteDatasourcePlugin) MakeDataSourcePipelinePlanV200(connectionId uint
}
plan_data := models.PipelineData{}
- err = p.invoker.Call("make-pipeline", bridge.DefaultContext, toolScopes).Get(&plan_data)
+ err = p.invoker.Call("make-pipeline", bridge.DefaultContext, toolScopes, connection.Unwrap()).Get(&plan_data)
if err != nil {
return nil, nil, err
}
diff --git a/backend/server/services/remote/plugin/plugin_impl.go b/backend/server/services/remote/plugin/plugin_impl.go
index 80df3650c..e22b32817 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -18,6 +18,8 @@ limitations under the License.
package plugin
import (
+ "fmt"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -42,7 +44,7 @@ type (
}
RemotePluginTaskData struct {
DbUrl string `json:"db_url"`
- ScopeId string `json:"scope_id"`
+ Scope interface{} `json:"scope"`
Connection interface{} `json:"connection"`
TransformationRule interface{} `json:"transformation_rule"`
Options map[string]interface{} `json:"options"`
@@ -115,14 +117,27 @@ func (p *remotePluginImpl) PrepareTaskData(taskCtx plugin.TaskContext, options m
return nil, errors.BadInput.New("missing scopeId")
}
+ db := taskCtx.GetDal()
+ wrappedScope := p.scopeTabler.New()
+ err = api.CallDB(db.First, wrappedScope, dal.Where("connection_id = ? AND id = ?", connectionId, scopeId))
+ if err != nil {
+ return nil, errors.BadInput.New("Invalid scope id")
+ }
+ var scope models.ScopeModel
+ err = wrappedScope.To(&scope)
+ if err != nil {
+ return nil, err
+ }
+
var txRule interface{}
- txRuleId, ok := options["transformation_rule_id"].(uint64)
- if ok {
+ if scope.TransformationRuleId > 0 {
+ if p.transformationRuleTabler == nil {
+ return nil, errors.Default.New(fmt.Sprintf("Cannot load transformation rule %v: plugin %s has no transformation rule model", scope.TransformationRuleId, p.name))
+ }
wrappedTxRule := p.transformationRuleTabler.New()
- db := taskCtx.GetDal()
- err = db.First(&wrappedTxRule, dal.Where("id = ?", txRuleId))
+ err = api.CallDB(db.First, wrappedTxRule, dal.From(p.transformationRuleTabler.TableName()), dal.Where("id = ?", scope.TransformationRuleId))
if err != nil {
- return nil, errors.BadInput.New("invalid transformation rule id")
+ return nil, err
}
txRule = wrappedTxRule.Unwrap()
} else {
@@ -131,7 +146,7 @@ func (p *remotePluginImpl) PrepareTaskData(taskCtx plugin.TaskContext, options m
return RemotePluginTaskData{
DbUrl: dbUrl,
- ScopeId: scopeId,
+ Scope: wrappedScope.Unwrap(),
Connection: connection,
TransformationRule: txRule,
Options: options,
diff --git a/backend/server/services/remote/plugin/remote_scope_api.go b/backend/server/services/remote/plugin/remote_scope_api.go
index e0fb2d183..341e0d52d 100644
--- a/backend/server/services/remote/plugin/remote_scope_api.go
+++ b/backend/server/services/remote/plugin/remote_scope_api.go
@@ -26,13 +26,6 @@ import (
"github.com/apache/incubator-devlake/server/services/remote/bridge"
)
-type ScopeItem struct {
- ScopeId string `json:"scopeId"`
- ScopeName string `json:"scopeName"`
- ConnectionId uint64 `json:"connectionId"`
- TransformationRuleId uint64 `json:"transformationRuleId,omitempty"`
-}
-
type RemoteScopesOutput struct {
Children []RemoteScopesTreeNode `json:"children"`
}
diff --git a/backend/server/services/remote/plugin/scope_api.go b/backend/server/services/remote/plugin/scope_api.go
index d27e02bd6..7477f2aa3 100644
--- a/backend/server/services/remote/plugin/scope_api.go
+++ b/backend/server/services/remote/plugin/scope_api.go
@@ -127,6 +127,13 @@ func (pa *pluginAPI) ListScopes(input *plugin.ApiResourceInput) (*plugin.ApiReso
if err != nil {
return nil, err
}
+ if pa.txRuleType == nil {
+ var apiScopes []apiScopeResponse
+ for _, scope := range scopeMap {
+ apiScopes = append(apiScopes, apiScopeResponse{Scope: scope})
+ }
+ return &plugin.ApiResourceOutput{Body: apiScopes, Status: http.StatusOK}, nil
+ }
var ruleIds []uint64
for _, scopeModel := range scopeMap {
if tid := uint64(scopeModel["transformation_rule_id"].(float64)); tid > 0 {
diff --git a/backend/test/helper/client.go b/backend/test/helper/client.go
index 471e528de..492a49dd0 100644
--- a/backend/test/helper/client.go
+++ b/backend/test/helper/client.go
@@ -118,7 +118,7 @@ func ConnectLocalServer(t *testing.T, sbConfig *LocalClientConfig) *DevlakeClien
d.dropDB()
}
if sbConfig.CreateServer {
- cfg.Set("PORT", fmt.Sprintf(":%d", sbConfig.ServerPort))
+ cfg.Set("PORT", sbConfig.ServerPort)
cfg.Set("PLUGIN_DIR", throwawayDir)
cfg.Set("LOGGING_DIR", throwawayDir)
go func() {
diff --git a/backend/test/remote/fakeplugin/fakeplugin/main.py b/backend/test/remote/fakeplugin/fakeplugin/main.py
index 4a1fbb485..7130530f9 100644
--- a/backend/test/remote/fakeplugin/fakeplugin/main.py
+++ b/backend/test/remote/fakeplugin/fakeplugin/main.py
@@ -57,14 +57,18 @@ class FakeStream(Stream):
for p in self.fake_pipelines:
yield dict(p)
- def convert(self, pipeline: FakePipeline):
+ def convert(self, pipeline: FakePipeline, ctx):
+ if ctx.transformationRule:
+ env = ctx.transformationRule.env
+ else:
+ env = "unknown"
yield CICDPipeline(
name=pipeline.id,
status=self.convert_status(pipeline.state),
finished_date=pipeline.finished_at,
result=self.convert_result(pipeline.state),
duration_sec=self.duration(pipeline),
- environment=[],
+ environment=env,
type=CICDPipeline.Type.CI
)
@@ -99,7 +103,7 @@ class FakeProject(ToolScope):
class FakeTransformationRule(TransformationRule):
- tx1: str
+ env: str
class FakePlugin(Plugin):
@@ -111,6 +115,10 @@ class FakePlugin(Plugin):
def tool_scope_type(self):
return FakeProject
+ @property
+ def transformation_rule_type(self):
+ return FakeTransformationRule
+
def domain_scopes(self, project: FakeProject):
yield CicdScope(
id=1,
diff --git a/backend/test/remote/remote_test.go b/backend/test/remote/remote_test.go
index 95e3d8e60..fae7079c6 100644
--- a/backend/test/remote/remote_test.go
+++ b/backend/test/remote/remote_test.go
@@ -41,6 +41,19 @@ type FakePluginConnection struct {
Token string `json:"token"`
}
+type FakeProject struct {
+ Id string `json:"id"`
+ Name string `json:"name"`
+ ConnectionId uint64 `json:"connection_id"`
+ TransformationRuleId uint64 `json:"transformation_rule_id"`
+}
+
+type FakeTxRule struct {
+ Id uint64 `json:"id"`
+ Name string `json:"name"`
+ Env string `json:"env"`
+}
+
func setupEnv() {
fmt.Println("Setup test env")
helper.LocalInit()
@@ -94,6 +107,28 @@ func CreateTestConnection(client *helper.DevlakeClient) *helper.Connection {
return connection
}
+func CreateTestScope(client *helper.DevlakeClient, connectionId uint64) any {
+ res := client.CreateTransformRule(PLUGIN_NAME, FakeTxRule{Name: "Tx rule", Env: "test env"})
+ rule, ok := res.(map[string]interface{})
+ if !ok {
+ panic("Cannot cast transform rule")
+ }
+ ruleId := uint64(rule["id"].(float64))
+
+ scope := client.CreateScope(PLUGIN_NAME,
+ connectionId,
+ FakeProject{
+ Id: "12345",
+ Name: "Test project",
+ ConnectionId: connectionId,
+ TransformationRuleId: ruleId,
+ },
+ )
+
+ client.SetTimeout(1)
+ return scope
+}
+
func TestCreateConnection(t *testing.T) {
setupEnv()
buildPython(t)
@@ -106,12 +141,26 @@ func TestCreateConnection(t *testing.T) {
require.Equal(t, TOKEN, conns[0].Token)
}
+func TestCreateScope(t *testing.T) {
+ setupEnv()
+ buildPython(t)
+ client := connectLocalServer(t)
+ var connectionId uint64 = 1
+
+ CreateTestScope(client, connectionId)
+
+ scopes := client.ListScopes(PLUGIN_NAME, connectionId)
+ require.Equal(t, 1, len(scopes))
+}
+
func TestRunPipeline(t *testing.T) {
setupEnv()
buildPython(t)
client := connectLocalServer(t)
conn := CreateTestConnection(client)
+ CreateTestScope(client, conn.ID)
+
pipeline := client.RunPipeline(models.NewPipeline{
Name: "remote_test",
Plan: []plugin.PipelineStage{
@@ -121,7 +170,7 @@ func TestRunPipeline(t *testing.T) {
Subtasks: nil,
Options: map[string]interface{}{
"connectionId": conn.ID,
- "scopeId": "org/project",
+ "scopeId": "12345",
},
},
},
@@ -144,11 +193,7 @@ func TestBlueprintV200(t *testing.T) {
ProjectName: projectName,
})
- client.CreateScope("fake", connection.ID, map[string]interface{}{
- "id": "12345",
- "connection_id": connection.ID,
- "name": "fake project",
- })
+ CreateTestScope(client, connection.ID)
blueprint := client.CreateBasicBlueprintV2(
"Test blueprint",
@@ -161,7 +206,7 @@ func TestBlueprintV200(t *testing.T) {
Id: "12345",
Name: "Test scope",
Entities: []string{
- plugin.DOMAIN_TYPE_CROSS,
+ plugin.DOMAIN_TYPE_CICD,
},
},
},