You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/30 07:54:14 UTC
[ignite-teamcity-bot] branch ignite-9848-3 updated: IGNITE-9848
Chain run report using Fat Build paradigm: dependencies migration to
background threads
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-3
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-3 by this push:
new b912422 IGNITE-9848 Chain run report using Fat Build paradigm: dependencies migration to background threads
b912422 is described below
commit b9124229bd0919c66b602a229db79f5baeeb01c8
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Oct 30 10:54:12 2018 +0300
IGNITE-9848 Chain run report using Fat Build paradigm: dependencies migration to background threads
---
.../ci/tcbot/builds/CompareBuildsService.java | 2 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 28 +++++++++++++++-------
2 files changed, 20 insertions(+), 10 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
index 45b26ab..71715c8 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
@@ -80,7 +80,7 @@ public class CompareBuildsService {
MultBuildRunCtx buildCtx = new MultBuildRunCtx(build, compactor);
final FatBuildCompacted fatBuild = tcIgnited.getFatBuild(build.getId());
- buildCtx.addBuild(bcp.loadTestsAndProblems(fatBuild, buildCtx, tcIgnited));
+ buildCtx.addBuild(bcp.loadTestsAndProblems(fatBuild, tcIgnited));
for (String testName : buildCtx.tests())
tests.add(extractTestName(testName));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 5b90a4b..7a0027c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -45,15 +45,14 @@ import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
-import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
-import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import org.jvnet.hk2.internal.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,12 +99,20 @@ public class BuildChainProcessor {
.filter(id -> !builds.containsKey(id)) //load and propagate only new entry points
.map(id -> builds.computeIfAbsent(id, teamcityIgnited::getFatBuild));
- Stream<FatBuildCompacted> uniqueBuildsInvolved = entryPointsFatBuilds
- .flatMap(ref -> dependencies(teamcityIgnited, builds, ref))
- .flatMap(ref -> dependencies(teamcityIgnited, builds, ref));
-
final ExecutorService svc = tcUpdatePool.getService();
+ final Stream<FatBuildCompacted> depsFirstLevel = entryPointsFatBuilds
+ .map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, ref)))
+ .collect(Collectors.toList())
+ .stream()
+ .flatMap(fut -> FutureUtil.getResult(fut));
+
+ Stream<FatBuildCompacted> uniqueBuildsInvolved = depsFirstLevel
+ .map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, ref)))
+ .collect(Collectors.toList())
+ .stream()
+ .flatMap(fut -> FutureUtil.getResult(fut));
+
final List<Future<Stream<FatBuildCompacted>>> phase1Submitted = uniqueBuildsInvolved
.map((buildRef) -> svc.submit(
() -> replaceWithRecent(teamcityIgnited, includeLatestRebuild, builds, buildRef, entryPoints.size())))
@@ -156,7 +163,8 @@ public class BuildChainProcessor {
@SuppressWarnings("WeakerAccess")
@NotNull
@AutoProfiling
- protected Stream<FatBuildCompacted> processBuildList(ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap,
+ protected Stream<FatBuildCompacted> processBuildList(ITeamcityIgnited teamcityIgnited,
+ Map<String, MultBuildRunCtx> buildsCtxMap,
Stream<FatBuildCompacted> list) {
list.forEach((FatBuildCompacted buildCompacted) -> {
final BuildRef ref = buildCompacted.toBuildRef(compactor);
@@ -164,7 +172,7 @@ public class BuildChainProcessor {
MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId,
k -> new MultBuildRunCtx(ref, compactor));
- ctx.addBuild(loadTestsAndProblems(buildCompacted, ctx, teamcityIgnited));
+ ctx.addBuild(loadTestsAndProblems(buildCompacted, teamcityIgnited));
});
return list;
@@ -178,7 +186,7 @@ public class BuildChainProcessor {
* @return Full context.
*/
public SingleBuildRunCtx loadTestsAndProblems(@Nonnull FatBuildCompacted buildCompacted,
- @Deprecated MultBuildRunCtx mCtx, ITeamcityIgnited tcIgnited) {
+ ITeamcityIgnited tcIgnited) {
SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildCompacted, compactor);
ctx.setChanges(tcIgnited.getAllChanges(buildCompacted.changes()));
@@ -287,5 +295,7 @@ public class BuildChainProcessor {
IntStream.of(build.snapshotDependencies())
.filter(id -> !builds.containsKey(id)) //load and propagate only new dependencies
.mapToObj(id -> builds.computeIfAbsent(id, teamcityIgnited::getFatBuild)));
+
+ //todo race here: double build id may appear
}
}