You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/30 10:57:04 UTC
[ignite-teamcity-bot] branch ignite-9848-3 updated: IGNITE-9848:
Loading builds chain in accordance with fat build approach
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-3
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-3 by this push:
new 6bb3ca7 IGNITE-9848: Loading builds chain in accordance with fat build approach
6bb3ca7 is described below
commit 6bb3ca7d52d51585aea1b15057f290bf71372e05
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Oct 30 13:57:02 2018 +0300
IGNITE-9848: Loading builds chain in accordance with fat build approach
---
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 60 +++++++++++-----------
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 5 +-
.../ci/teamcity/ignited/change/ChangeDao.java | 17 +++---
.../ci/teamcity/ignited/change/ChangeSync.java | 2 +
4 files changed, 41 insertions(+), 43 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 7a0027c..5675db2 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -52,7 +52,6 @@ import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import org.jvnet.hk2.internal.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -107,25 +106,21 @@ public class BuildChainProcessor {
.stream()
.flatMap(fut -> FutureUtil.getResult(fut));
- Stream<FatBuildCompacted> uniqueBuildsInvolved = depsFirstLevel
+ Stream<FatBuildCompacted> secondLevelDeps = depsFirstLevel
.map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, ref)))
.collect(Collectors.toList())
.stream()
.flatMap(fut -> FutureUtil.getResult(fut));
- final List<Future<Stream<FatBuildCompacted>>> phase1Submitted = uniqueBuildsInvolved
- .map((buildRef) -> svc.submit(
- () -> replaceWithRecent(teamcityIgnited, includeLatestRebuild, builds, buildRef, entryPoints.size())))
+ // builds may became non unique because of rece in filtering and acquiring deps
+ final List<Future<Stream<FatBuildCompacted>>> phase3Submitted = secondLevelDeps
+ .map((fatBuild) -> svc.submit(
+ () -> replaceWithRecent(teamcityIgnited, includeLatestRebuild, builds, fatBuild, entryPoints.size())))
.collect(Collectors.toList());
-
- final List<Future<? extends Stream<FatBuildCompacted>>> phase2Submitted = phase1Submitted.stream()
- .map(FutureUtil::getResult)
- .map((s) -> svc.submit(
- () -> processBuildList(teamcityIgnited, buildsCtxMap, s)))
- .collect(Collectors.toList());
-
- phase2Submitted.forEach(FutureUtil::getResult);
+ phase3Submitted.stream()
+ .flatMap(fut -> FutureUtil.getResult(fut))
+ .forEach((fatBuild) -> createCxt(teamcityIgnited, buildsCtxMap, fatBuild));
ArrayList<MultBuildRunCtx> contexts = new ArrayList<>(buildsCtxMap.values());
@@ -160,22 +155,17 @@ public class BuildChainProcessor {
return fullChainRunCtx;
}
+
@SuppressWarnings("WeakerAccess")
- @NotNull
@AutoProfiling
- protected Stream<FatBuildCompacted> processBuildList(ITeamcityIgnited teamcityIgnited,
- Map<String, MultBuildRunCtx> buildsCtxMap,
- Stream<FatBuildCompacted> list) {
- list.forEach((FatBuildCompacted buildCompacted) -> {
- final BuildRef ref = buildCompacted.toBuildRef(compactor);
+ protected void createCxt(ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap,
+ FatBuildCompacted buildCompacted) {
+ final BuildRef ref = buildCompacted.toBuildRef(compactor);
- MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId,
- k -> new MultBuildRunCtx(ref, compactor));
+ final MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId,
+ k -> new MultBuildRunCtx(ref, compactor));
- ctx.addBuild(loadTestsAndProblems(buildCompacted, teamcityIgnited));
- });
-
- return list;
+ ctx.addBuild(loadTestsAndProblems(buildCompacted, teamcityIgnited));
}
/**
@@ -290,12 +280,20 @@ public class BuildChainProcessor {
private Stream<FatBuildCompacted> dependencies(
ITeamcityIgnited teamcityIgnited,
Map<Integer, FatBuildCompacted> builds,
- FatBuildCompacted build) {
- return Stream.concat(Stream.of(build),
- IntStream.of(build.snapshotDependencies())
- .filter(id -> !builds.containsKey(id)) //load and propagate only new dependencies
- .mapToObj(id -> builds.computeIfAbsent(id, teamcityIgnited::getFatBuild)));
+ FatBuildCompacted build) {
+ return Stream.concat(
+ Stream.of(build),
+ IntStream.of(build.snapshotDependencies())
+ .mapToObj(id -> {
+ if (builds.containsKey(id))
+ return null; //load and propagate only new dependencies
+
+ FatBuildCompacted buildLoaded = teamcityIgnited.getFatBuild(id);
+
+ FatBuildCompacted prevVal = builds.putIfAbsent(id, buildLoaded);
- //todo race here: double build id may appear
+ return prevVal == null ? buildLoaded : null;
+ }))
+ .filter(Objects::nonNull);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 9ff34ab..67ea0c2 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -189,9 +189,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return savedVer == null ? existingBuild : savedVer;
}
- @Override
@AutoProfiling
- public Collection<ChangeCompacted> getAllChanges(int[] changeIds) {
+ @Override public Collection<ChangeCompacted> getAllChanges(int[] changeIds) {
final Map<Long, ChangeCompacted> all = changesDao.getAll(srvIdMaskHigh, changeIds);
final Map<Integer, ChangeCompacted> changes = new HashMap<>();
@@ -205,7 +204,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
for (int changeId : changeIds) {
if (!changes.containsKey(changeId)) {
- final ChangeCompacted change = changeSync.change(srvIdMaskHigh, changeId, conn);
+ final ChangeCompacted change = changeSync.reloadChange(srvIdMaskHigh, changeId, conn);
changes.put(changeId, change);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java
index f4a7d9a..6227265 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java
@@ -17,6 +17,11 @@
package org.apache.ignite.ci.teamcity.ignited.change;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
+import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
@@ -24,10 +29,6 @@ import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.configuration.CacheConfiguration;
-import javax.inject.Inject;
-import javax.inject.Provider;
-import java.util.*;
-
public class ChangeDao {
/** Cache name */
public static final String TEAMCITY_CHANGE_CACHE_NAME = "teamcityChange";
@@ -92,14 +93,12 @@ public class ChangeDao {
return changesCache.get(changeIdToCacheKey(srvId, changeId));
}
+ @AutoProfiling
public Map<Long, ChangeCompacted> getAll(int srvIdMaskHigh, int[] changeIds) {
final Set<Long> collect = new HashSet<>();
- for (int changeId : changeIds) {
+ for (int changeId : changeIds)
collect.add(changeIdToCacheKey(srvIdMaskHigh, changeId));
- }
-
- final Map<Long, ChangeCompacted> all = changesCache.getAll(collect);
- return all;
+ return changesCache.getAll(collect);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java
index 739f119..88d27a0 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.teamcity.ignited.change;
import com.google.common.base.Throwables;
+import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.changes.Change;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
@@ -48,6 +49,7 @@ public class ChangeSync {
}
@NotNull
+ @AutoProfiling
public ChangeCompacted reloadChange(int srvId, int changeId, ITeamcityConn conn) {
Change change;
try {