You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampark.apache.org by be...@apache.org on 2022/12/07 04:07:04 UTC
[incubator-streampark] branch git-call updated: [Bug] project git clone bug fixed
This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch git-call
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/git-call by this push:
new b1f57c65c [Bug] project git clone bug fixed
b1f57c65c is described below
commit b1f57c65c36583917b0d52e6243604e67f541c9d
Author: benjobs <be...@apache.org>
AuthorDate: Wed Dec 7 12:06:49 2022 +0800
[Bug] project git clone bug fixed
---
.../streampark-console-service/pom.xml | 4 +-
.../streampark/console/base/util/GitUtils.java | 123 +++++++++++++++++++++
.../streampark/console/core/entity/Project.java | 49 +++-----
.../streampark/console/core/enums/GitProtocol.java | 37 ++++++-
.../core/service/impl/ProjectServiceImpl.java | 13 ++-
.../console/core/task/ProjectBuildTask.java | 22 +---
6 files changed, 193 insertions(+), 55 deletions(-)
diff --git a/streampark-console/streampark-console-service/pom.xml b/streampark-console/streampark-console-service/pom.xml
index 312da0450..f8a801eae 100644
--- a/streampark-console/streampark-console-service/pom.xml
+++ b/streampark-console/streampark-console-service/pom.xml
@@ -56,7 +56,7 @@
<lombok.version>1.18.24</lombok.version>
<xml-apis.version>1.4.01</xml-apis.version>
<ivy.version>2.5.0</ivy.version>
- <eclipse.jgit.version>5.13.0.202109080827-r</eclipse.jgit.version>
+ <eclipse.jgit.version>5.13.1.202206130422-r</eclipse.jgit.version>
</properties>
<dependencyManagement>
@@ -328,7 +328,7 @@
<dependency>
<groupId>org.eclipse.jgit</groupId>
- <artifactId>org.eclipse.jgit</artifactId>
+ <artifactId>org.eclipse.jgit.ssh.jsch</artifactId>
<version>${eclipse.jgit.version}</version>
</dependency>
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
new file mode 100644
index 000000000..dd6ce0584
--- /dev/null
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
@@ -0,0 +1,123 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Code Technology Studio
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+ * the Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.apache.streampark.console.base.util;
+
+import org.apache.streampark.console.core.entity.Project;
+import org.apache.streampark.console.core.enums.GitProtocol;
+
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.jgit.api.CloneCommand;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.LsRemoteCommand;
+import org.eclipse.jgit.api.TransportCommand;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.transport.JschConfigSessionFactory;
+import org.eclipse.jgit.transport.OpenSshConfig;
+import org.eclipse.jgit.transport.SshTransport;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+import org.eclipse.jgit.util.FS;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ *
+ */
+public class GitUtils {
+
+ public static Git clone(Project project) throws GitAPIException {
+ CloneCommand cloneCommand = Git.cloneRepository()
+ .setURI(project.getUrl())
+ .setDirectory(project.getAppSource());
+
+ if (project.getBranches() != null) {
+ cloneCommand.setBranch(Constants.R_HEADS + project.getBranches());
+ cloneCommand.setBranchesToClone(Collections.singletonList(Constants.R_HEADS + project.getBranches()));
+ }
+ setCredentials(cloneCommand, project);
+ return cloneCommand.call();
+ }
+
+ private static void setCredentials(TransportCommand<?, ?> transportCommand, Project project) {
+ GitProtocol gitProtocol = GitProtocol.of(project.getGitProtocol());
+ switch (gitProtocol) {
+ case HTTP:
+ UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(project.getUserName(), project.getPassword());
+ transportCommand.setCredentialsProvider(credentialsProvider);
+ break;
+ case SSH:
+ transportCommand.setTransportConfigCallback(transport -> {
+ SshTransport sshTransport = (SshTransport) transport;
+ sshTransport.setSshSessionFactory(new JschConfigSessionFactory() {
+ @Override
+ protected void configure(OpenSshConfig.Host hc, Session session) {
+ session.setConfig("StrictHostKeyChecking", "no");
+ }
+
+ @Override
+ protected JSch createDefaultJSch(FS fs) throws JSchException {
+ JSch jSch = super.createDefaultJSch(fs);
+ if (project.getRsaPath() == null) {
+ return jSch;
+ }
+ if (StringUtils.isEmpty(project.getPassword())) {
+ jSch.addIdentity(project.getRsaPath());
+ } else {
+ jSch.addIdentity(project.getRsaPath(), project.getPassword());
+ }
+ return jSch;
+ }
+ });
+ });
+ break;
+ default:
+ throw new IllegalStateException("[StreamPark] git setCredentials: unsupported protocol type");
+ }
+ }
+
+ public static List<String> getBranchList(Project project) throws GitAPIException {
+ LsRemoteCommand command = Git.lsRemoteRepository()
+ .setRemote(project.getUrl())
+ .setHeads(true);
+ setCredentials(command, project);
+ Collection<Ref> refList = command.call();
+ List<String> branchList = new ArrayList<>(4);
+ for (Ref ref : refList) {
+ String refName = ref.getName();
+ if (refName.startsWith(Constants.R_HEADS)) {
+ String branchName = refName.replace(Constants.R_HEADS, "");
+ branchList.add(branchName);
+ }
+ }
+ return branchList;
+ }
+
+}
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
index fc1b99bed..0584fa204 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
@@ -21,11 +21,14 @@ import org.apache.streampark.common.conf.CommonConfig;
import org.apache.streampark.common.conf.Workspace;
import org.apache.streampark.common.util.CommandUtils;
import org.apache.streampark.console.base.util.CommonUtils;
+import org.apache.streampark.console.base.util.GitUtils;
import org.apache.streampark.console.base.util.WebUtils;
import org.apache.streampark.console.core.enums.GitAuthorizedError;
import org.apache.streampark.console.core.service.SettingService;
+import com.baomidou.mybatisplus.annotation.FieldStrategy;
import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -33,16 +36,11 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
-import org.eclipse.jgit.api.Git;
-import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.transport.CredentialsProvider;
-import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+import org.eclipse.jgit.lib.Constants;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
@@ -67,9 +65,17 @@ public class Project implements Serializable {
private Date lastBuild;
+ private Integer gitProtocol;
+
+ @TableField(updateStrategy = FieldStrategy.IGNORED)
private String userName;
+ @TableField(updateStrategy = FieldStrategy.IGNORED)
private String password;
+
+ @TableField(updateStrategy = FieldStrategy.IGNORED)
+ private String rsaPath;
+
/**
* 1:git 2:svn
*/
@@ -134,15 +140,10 @@ public class Project implements Serializable {
return new File(Workspace.APP_LOCAL_DIST(), id.toString());
}
- @JsonIgnore
- public CredentialsProvider getCredentialsProvider() {
- return new UsernamePasswordCredentialsProvider(this.userName, this.password);
- }
-
@JsonIgnore
public File getGitRepository() {
File home = getAppSource();
- return new File(home, ".git");
+ return new File(home, Constants.DOT_GIT);
}
public void delete() throws IOException {
@@ -153,22 +154,7 @@ public class Project implements Serializable {
@JsonIgnore
public List<String> getAllBranches() {
try {
- Collection<Ref> refList;
- if (CommonUtils.notEmpty(userName, password)) {
- UsernamePasswordCredentialsProvider pro = new UsernamePasswordCredentialsProvider(userName, password);
- refList = Git.lsRemoteRepository().setRemote(url).setCredentialsProvider(pro).call();
- } else {
- refList = Git.lsRemoteRepository().setRemote(url).call();
- }
- List<String> branchList = new ArrayList<>(4);
- for (Ref ref : refList) {
- String refName = ref.getName();
- if (refName.startsWith("refs/heads/")) {
- String branchName = refName.replace("refs/heads/", "");
- branchList.add(branchName);
- }
- }
- return branchList;
+ return GitUtils.getBranchList(this);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
@@ -177,12 +163,7 @@ public class Project implements Serializable {
public GitAuthorizedError gitCheck() {
try {
- if (CommonUtils.notEmpty(userName, password)) {
- UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(userName, password);
- Git.lsRemoteRepository().setRemote(url).setCredentialsProvider(credentialsProvider).call();
- } else {
- Git.lsRemoteRepository().setRemote(url).call();
- }
+ GitUtils.getBranchList(this);
return GitAuthorizedError.SUCCESS;
} catch (Exception e) {
String err = e.getMessage();
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/enums/GitProtocol.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/enums/GitProtocol.java
index 42ff8760d..f05de6b5c 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/enums/GitProtocol.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/enums/GitProtocol.java
@@ -1,2 +1,37 @@
-package org.apache.streampark.console.core.enums;public enum GitProtocol {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.console.core.enums;
+
+import java.util.Arrays;
+
+public enum GitProtocol {
+
+ HTTP(1),
+ SSH(2);
+
+ private final int value;
+
+ GitProtocol(int value) {
+ this.value = value;
+ }
+
+ public static GitProtocol of(Integer value) {
+ return Arrays.stream(values()).filter(x -> x.value == value).findFirst().orElse(null);
+ }
+
}
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
index 7abe4154e..0b7eafda6 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -27,6 +27,7 @@ import org.apache.streampark.common.util.ThreadUtils;
import org.apache.streampark.console.base.domain.ResponseCode;
import org.apache.streampark.console.base.domain.RestRequest;
import org.apache.streampark.console.base.domain.RestResponse;
+import org.apache.streampark.console.base.exception.ApiAlertException;
import org.apache.streampark.console.base.mybatis.pager.MybatisPager;
import org.apache.streampark.console.base.util.CommonUtils;
import org.apache.streampark.console.base.util.FileUtils;
@@ -34,6 +35,7 @@ import org.apache.streampark.console.base.util.GZipUtils;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildState;
+import org.apache.streampark.console.core.enums.GitProtocol;
import org.apache.streampark.console.core.enums.LaunchState;
import org.apache.streampark.console.core.mapper.ProjectMapper;
import org.apache.streampark.console.core.service.ApplicationService;
@@ -104,7 +106,7 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
return response.message("Add project failed").data(false);
}
} else {
- return response.message("A project with this name already exists, adding a task failed").data(false);
+ throw new ApiAlertException("project name already exists,add project failed");
}
}
@@ -119,11 +121,18 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
project.setName(projectParam.getName());
project.setUrl(projectParam.getUrl());
project.setBranches(projectParam.getBranches());
+ project.setGitProtocol(projectParam.getGitProtocol());
+ project.setRsaPath(projectParam.getRsaPath());
project.setUserName(projectParam.getUserName());
project.setPassword(projectParam.getPassword());
project.setPom(projectParam.getPom());
project.setDescription(projectParam.getDescription());
project.setBuildArgs(projectParam.getBuildArgs());
+ if (GitProtocol.SSH.equals(GitProtocol.of(project.getGitProtocol()))) {
+ project.setUserName(null);
+ } else {
+ project.setRsaPath(null);
+ }
if (projectParam.getBuildState() != null) {
project.setBuildState(projectParam.getBuildState());
if (BuildState.of(projectParam.getBuildState()).equals(BuildState.NEED_REBUILD)) {
@@ -196,7 +205,7 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
List<String> list = new ArrayList<>();
File[] files = appHome.listFiles();
if (CommonUtils.notEmpty(files)) {
- for (File file: files) {
+ for (File file : files) {
list.add(file.getName());
}
}
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
index 110aa8b86..7340fdd6a 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
@@ -19,7 +19,7 @@ package org.apache.streampark.console.core.task;
import org.apache.streampark.common.util.CommandUtils;
import org.apache.streampark.common.util.Utils;
-import org.apache.streampark.console.base.util.CommonUtils;
+import org.apache.streampark.console.base.util.GitUtils;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildState;
@@ -28,7 +28,6 @@ import org.apache.streampark.console.core.mapper.ProjectMapper;
import org.apache.streampark.console.core.service.ApplicationService;
import lombok.extern.slf4j.Slf4j;
-import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.lib.StoredConfig;
@@ -78,7 +77,7 @@ public class ProjectBuildTask extends AbstractLogFileTask {
this.baseMapper.updateBuildTime(project.getId());
this.deploy(project);
List<Application> applications = this.applicationService.getByProjectId(project.getId());
- // Update the deploy state
+ // Update the deployment state
FlinkTrackingTask.refreshTracking(() -> applications.forEach((app) -> {
fileLogger.info("update deploy by project: {}, appName:{}", project.getName(), app.getJobName());
app.setLaunch(LaunchState.NEED_LAUNCH.get());
@@ -102,22 +101,13 @@ public class ProjectBuildTask extends AbstractLogFileTask {
project.cleanCloned();
fileLogger.info("clone {}, {} starting...", project.getName(), project.getUrl());
fileLogger.info(project.getLog4CloneStart());
- CloneCommand cloneCommand = Git.cloneRepository()
- .setURI(project.getUrl())
- .setDirectory(project.getAppSource())
- .setBranch(project.getBranches());
-
- if (CommonUtils.notEmpty(project.getUserName(), project.getPassword())) {
- cloneCommand.setCredentialsProvider(project.getCredentialsProvider());
- }
- Git git = cloneCommand.call();
+ Git git = GitUtils.clone(project);
StoredConfig config = git.getRepository().getConfig();
config.setBoolean("http", project.getUrl(), "sslVerify", false);
config.setBoolean("https", project.getUrl(), "sslVerify", false);
config.save();
-
File workTree = git.getRepository().getWorkTree();
- gitWorkTree(project.getId(), workTree, "");
+ printWorkTree(workTree, "");
String successMsg = String.format(
"[StreamPark] project [%s] git clone successful!\n",
project.getName()
@@ -137,7 +127,7 @@ public class ProjectBuildTask extends AbstractLogFileTask {
}
}
- private void gitWorkTree(Long id, File workTree, String space) {
+ private void printWorkTree(File workTree, String space) {
File[] files = workTree.listFiles();
for (File file : Objects.requireNonNull(files)) {
if (!file.getName().startsWith(".git")) {
@@ -147,7 +137,7 @@ public class ProjectBuildTask extends AbstractLogFileTask {
fileLogger.info("{} / {}", space, file.getName());
} else if (file.isDirectory()) {
fileLogger.info("{} / {}", space, file.getName());
- gitWorkTree(id, file, space.concat("/").concat(file.getName()));
+ printWorkTree(file, space.concat("/").concat(file.getName()));
}
}
}