You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/21 17:08:08 UTC
[ignite-teamcity-bot] 01/02: Trusted tests & suite history
performance fixes: suite history implementation
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 6f8463859e49ae6bbdfda5e0290a946923415f31
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jun 21 19:25:58 2019 +0300
Trusted tests & suite history performance fixes: suite history implementation
---
.../ignite/ci/tcbot/issue/IssueDetector.java | 18 ++-
.../tcbot/engine/chain/BuildChainProcessor.java | 5 +-
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 13 +-
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 2 +-
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 17 ++-
.../apache/ignite/tcignited/ITeamcityIgnited.java | 2 +
.../ignite/tcignited/TeamcityIgnitedImpl.java | 16 ++-
.../apache/ignite/tcignited/build/FatBuildDao.java | 2 +-
.../ignite/tcignited/build/SuiteHistory.java | 15 ++-
.../ignite/tcignited/history/HistoryCollector.java | 137 +++++++++------------
.../tcignited/history/RunHistCompactedDao.java | 4 +-
.../ignite/tcignited/history/SuiteInvocation.java | 8 +-
12 files changed, 130 insertions(+), 109 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index 0dc8d98..ead0965 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -261,13 +261,14 @@ public class IssueDetector {
final String trackedBranch = res.getTrackedBranch();
+ String suiteId = suiteCurrentStatus.suiteId;
for (DsTestFailureUi testFailure : suiteCurrentStatus.testFailures) {
- if (registerTestFailIssues(tcIgnited, srvCode, normalizeBranch, testFailure, trackedBranch,
+ if (registerTestFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, testFailure, trackedBranch,
suiteCurrentStatus.tags))
newIssues++;
}
- if (registerSuiteFailIssues(tcIgnited, srvCode, normalizeBranch, suiteCurrentStatus, trackedBranch))
+ if (registerSuiteFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, suiteCurrentStatus, trackedBranch))
newIssues++;
}
}
@@ -286,13 +287,15 @@ public class IssueDetector {
*/
private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
+ String suiteId,
String normalizeBranch,
DsSuiteUi suiteFailure,
String trackedBranch) {
- String suiteId = suiteFailure.suiteId;
+ Integer btId = compactor.getStringIdIfPresent(suiteId);
+ Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch);
- IRunHistory runStat = tcIgnited.getSuiteRunHist(suiteId, normalizeBranch);
+ IRunHistory runStat = tcIgnited.getSuiteRunHist(btId, brNormId);
if (runStat == null)
return false;
@@ -361,14 +364,17 @@ public class IssueDetector {
private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
+ String suiteId,
String normalizeBranch,
DsTestFailureUi testFailure,
String trackedBranch,
@Nonnull Set<String> suiteTags) {
String name = testFailure.name;
- Integer tname = compactor.getStringIdIfPresent(name);
+ int tname = compactor.getStringId(name);
+ Integer btId = compactor.getStringIdIfPresent(suiteId);
+ Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch);
- IRunHistory runStat = tcIgnited.getTestRunHist(name, normalizeBranch);
+ IRunHistory runStat = tcIgnited.getTestRunHist(tname, btId, brNormId);
if (runStat == null)
return false;
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index ff88d77..f8ee591 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -185,9 +185,8 @@ public class BuildChainProcessor {
});
Function<MultBuildRunCtx, Float> function = ctx -> {
-
- //todo cache RunStat instance into suite context to compare
- IRunHistory runStat = tcIgn.getSuiteRunHist(ctx.suiteId(), RunHistSync.normalizeBranch(failRateBranch));
+ IRunHistory runStat = ctx.history(tcIgn,
+ compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch)));
if (runStat == null)
return 0f;
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index 5fb0b2d..6d45206 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -27,6 +27,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.OptionalDouble;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
@@ -34,7 +35,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
@@ -67,6 +67,8 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();
+ private java.util.Map<Integer, IRunHistory> historyCacheMap = new ConcurrentHashMap<>();
+
public void addBuild(SingleBuildRunCtx ctx) {
builds.add(ctx);
}
@@ -636,4 +638,13 @@ public class MultBuildRunCtx implements ISuiteResults {
public Optional<SingleBuildRunCtx> firstBuild() {
return builds.stream().findFirst();
}
+
+ /**
+ * @param tcIgn Tc ign.
+ * @param baseBranchId Base branch id.
+ */
+ public IRunHistory history(ITeamcityIgnited tcIgn, Integer baseBranchId) {
+ return historyCacheMap.computeIfAbsent(baseBranchId,
+ (k)-> tcIgn.getSuiteRunHist(buildTypeIdId(), k));
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index 0e93b57..2da9f5e 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -259,7 +259,7 @@ public class PrChainsProcessor {
.map((ctx) -> {
String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch);
Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch);
- IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(ctx.suiteId(), normalizedBaseBranch);
+ IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId);
Integer suiteId = compactor.getStringIdIfPresent(ctx.suiteId()); // can be inlined
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 8a1b003..c823fd9 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -168,7 +168,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
String curBranchNormalized = normalizeBranch(suite.branchName());
Integer curBranchId = compactor.getStringIdIfPresent(curBranchNormalized);
- IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId());
+ IRunHistory baseBranchHist = initSuiteStat(tcIgnited, baseBranchId, curBranchId, suite);
Set<String> collect = suite.lastChangeUsers().collect(Collectors.toSet());
@@ -264,13 +264,13 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
private IRunHistory initSuiteStat(ITeamcityIgnited tcIgnited,
- String failRateNormalizedBranch,
- String curBranchNormalized,
- String suiteId) {
- if (Strings.isNullOrEmpty(suiteId))
+ Integer failRateNormalizedBranch,
+ Integer curBranchNormalized,
+ MultBuildRunCtx suite) {
+ if (suite.buildTypeIdId() == null)
return null;
- final IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(suiteId, failRateNormalizedBranch);
+ IRunHistory statInBaseBranch = suite.history(tcIgnited, failRateNormalizedBranch);
if (statInBaseBranch != null) {
failures = statInBaseBranch.getFailuresCount();
@@ -289,9 +289,8 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
IRunHistory latestRunsSrc = null;
- if (!failRateNormalizedBranch.equals(curBranchNormalized)) {
-
- final IRunHistory statForStripe = tcIgnited.getSuiteRunHist(suiteId, curBranchNormalized);
+ if (!Objects.equals(failRateNormalizedBranch, curBranchNormalized)) {
+ IRunHistory statForStripe = suite.history(tcIgnited, curBranchNormalized);
latestRunsSrc = statForStripe;
latestRuns = statForStripe != null ? statForStripe.getLatestRunResults() : null;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index f19a4ad..7cd950e 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -203,8 +203,10 @@ public interface ITeamcityIgnited {
@Deprecated
@Nullable public IRunHistory getTestRunHist(String testName, @Nullable String branch);
+ @Deprecated
@Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch);
+ @Nullable public IRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
/**
* V.3.0 run history implementation based on scan of fat builds.
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 18ea52b..81a74c8 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -16,10 +16,8 @@
*/
package org.apache.ignite.tcignited;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.io.File;
-import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -34,7 +32,6 @@ import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -51,7 +48,6 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
@@ -72,7 +68,6 @@ import org.apache.ignite.tcignited.history.RunHistSync;
import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
-import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.model.agent.Agent;
import org.apache.ignite.tcservice.model.conf.Project;
@@ -441,6 +436,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
+ @Nullable @Override public IRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch) {
+ if (buildTypeId == null || normalizedBaseBranch == null)
+ return null;
+
+ if (buildTypeId < 0 || normalizedBaseBranch < 0)
+ return null;
+
+ return histCollector.getSuiteRunHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId,
@Nullable Integer normalizedBaseBranch) {
if (buildTypeId == null || normalizedBaseBranch == null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 36f355c..b40e204 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -134,7 +134,7 @@ public class FatBuildDao {
public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
- histCollector.invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
+ histCollector.invalidateHistoryInMem(srvIdMaskHigh, newBuild);
}
public static int[] extractChangeIds(@Nonnull ChangesList changesList) {
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index 394b75d..cfd24fe 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -20,7 +20,6 @@ package org.apache.ignite.tcignited.build;
import java.util.HashMap;
import java.util.Map;
-import java.util.TreeMap;
import org.apache.ignite.Ignite;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
@@ -33,7 +32,9 @@ import org.apache.ignite.tcignited.history.SuiteInvocation;
*/
public class SuiteHistory {
/** Tests history: Test name ID->RunHistory */
- Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
+ private Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
+
+ private RunHistCompacted suiteHist = new RunHistCompacted();
public int size(Ignite ignite) {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
@@ -51,4 +52,14 @@ public class SuiteHistory {
public void addTestInvocation(Integer tName, Invocation invocation) {
getOrAddTestsHistory(tName).innerAddInvocation(invocation);
}
+
+ public void addSuiteInvocation(SuiteInvocation suiteInv) {
+ suiteInv.tests().forEach(this::addTestInvocation);
+
+ suiteHist.innerAddInvocation(suiteInv.suiteInvocation());
+ }
+
+ public RunHistCompacted getSuiteHist() {
+ return suiteHist;
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index b204eae..4a62534 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.inject.Inject;
@@ -47,11 +48,16 @@ import org.apache.ignite.tcignited.build.SuiteHistory;
import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
public class HistoryCollector {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(HistoryCollector.class);
+
/** History DAO. */
@Inject private SuiteInvocationHistoryDao histDao;
@@ -84,16 +90,25 @@ public class HistoryCollector {
.softValues()
.build();
+ /** Biggest build ID, which out of history scope (MAX days + 2). */
+ private final ConcurrentMap<Integer, AtomicInteger> biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>();
+
/**
* @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
* @param testName Test name.
* @param buildTypeId Suite (Build type) id.
* @param normalizedBaseBranch Branch name.
*/
- @AutoProfiling
public IRunHistory getTestRunHist(int srvIdMaskHigh, int testName, int buildTypeId,
int normalizedBaseBranch) {
+ SuiteHistory hist = getSuiteHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+
+ return hist.getTestRunHist(testName);
+ }
+
+ @AutoProfiling
+ protected SuiteHistory getSuiteHist(int srvIdMaskHigh, int buildTypeId, int normalizedBaseBranch) {
RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
SuiteHistory hist;
@@ -105,11 +120,9 @@ public class HistoryCollector {
throw ExceptionUtil.propagateException(e);
}
- return hist.getTestRunHist(testName);
+ return hist;
}
- ConcurrentMap<Integer, AtomicInteger> biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>();
-
/**
* Latest actual Build ids supplier. This supplier should handle all equivalent branches in
* it.
@@ -123,14 +136,14 @@ public class HistoryCollector {
int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
String btId = compactor.getStringFromId(buildTypeId);
String branchId = compactor.getStringFromId(normalizedBaseBranch);
- List<BuildRefCompacted> compacted = buildRefDao.getAllBuildsCompacted(srvId, btId,
+ List<BuildRefCompacted> bRefsList = buildRefDao.getAllBuildsCompacted(srvId, btId,
branchEquivalence.branchForQuery(branchId));
AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
long curTs = System.currentTimeMillis();
- Set<Integer> buildIds = compacted.stream()
+ Set<Integer> buildIds = bRefsList.stream()
.filter(b -> b.id() > outOfScopeBuildId)
.filter(this::applicableForHistory)
.map(BuildRefCompacted::id)
@@ -140,31 +153,28 @@ public class HistoryCollector {
+ compactor.getStringFromId(buildTypeId)
+ " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" );
- Map<Integer, Long> buildStartTimeFromSeparatedCache = getStartTimeFromSpecialCache(srvId, buildIds);
+ Map<Integer, Long> buildStartTimes = getStartTimeFromSpecialCache(srvId, buildIds);
- HashSet<Integer> notFoundKeys = new HashSet<>(buildIds);
- notFoundKeys.removeAll(buildStartTimeFromSeparatedCache.keySet());
+ Set<Integer> notFoundKeys = new HashSet<>(buildIds);
+ notFoundKeys.removeAll(buildStartTimes.keySet());
if (!notFoundKeys.isEmpty()) {
Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, notFoundKeys);
- buildStartTimeFromSeparatedCache.putAll(buildStartTimeFromFatBuild);
+ buildStartTimes.putAll(buildStartTimeFromFatBuild);
- runHistCompactedDao.setBuildsStartTime(srvId, buildStartTimeFromFatBuild);
+ runHistCompactedDao.setBuildsStartTimeAsync(srvId, buildStartTimeFromFatBuild);
}
- //todo filter queued, cancelled and so on
+
Set<Integer> buildInScope = buildIds.stream().filter(
bId -> {
- //todo getAll
- //todo getStartTime From FatBuild
- Long startTime = buildStartTimeFromSeparatedCache.get(bId);
- //todo cache in runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
+ Long startTime = buildStartTimes.get(bId);
if (startTime == null)
return false;
long ageInDays = Duration.ofMillis(curTs - startTime).toDays();
- if(ageInDays>InvocationData.MAX_DAYS + 2) {
+ if (ageInDays > InvocationData.MAX_DAYS + 2) {
AtomicInteger integer = biggestBuildIdOutOfHistoryScope.computeIfAbsent(srvId,
s -> {
AtomicInteger atomicInteger = new AtomicInteger();
@@ -175,7 +185,7 @@ public class HistoryCollector {
int newBorder = integer.accumulateAndGet(bId, Math::max);
if (newBorder == bId)
- System.err.println(" New border for server was set " + bId);
+ logger.info("History Collector: New border for server was set " + bId);
}
return ageInDays < InvocationData.MAX_DAYS;
@@ -183,7 +193,7 @@ public class HistoryCollector {
).collect(Collectors.toSet());
System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " +
- buildInScope.size() + " from " + buildIds.size());
+ buildInScope.size() + " from " + bRefsList.size());
return buildInScope;
}
@@ -206,7 +216,7 @@ public class HistoryCollector {
}
@AutoProfiling
- public SuiteHistory loadSuiteHistory(int srvId,
+ protected SuiteHistory loadSuiteHistory(int srvId,
int buildTypeId,
int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
@@ -229,45 +239,50 @@ public class HistoryCollector {
SuiteHistory sumary = new SuiteHistory();
- suiteRunHist.forEach((buildId, suiteInv) -> suiteInv.tests().forEach(sumary::addTestInvocation));
+ suiteRunHist.forEach((buildId, suiteInv) -> sumary.addSuiteInvocation(suiteInv));
- System.err.println("***** History for suite "
- + compactor.getStringFromId(buildTypeId)
- + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- sumary.size(igniteProvider.get()) + " bytes");
+ if (logger.isDebugEnabled()) {
+ logger.debug("***** History for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ sumary.size(igniteProvider.get()) + " bytes");
+ }
return sumary;
}
- public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
- Iterable<RunHistKey> objects =
- stream
- .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
- .collect(Collectors.toSet());
+ /**
+ * @param srvId Server id.
+ * @param b Build ref to invalidate.
+ */
+ public void invalidateHistoryInMem(int srvId, BuildRefCompacted b) {
+ RunHistKey inv = new RunHistKey(srvId, b.buildTypeId(), b.branchName());
- runHistInMemCache.invalidateAll(objects);
+ runHistInMemCache.invalidate(inv);
}
@AutoProfiling
- public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
+ protected Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
System.err.println("GET ALL: " + missedBuildsIds.size());
- Iterables.partition(missedBuildsIds, 32*10).forEach(
- chunk->{
+ Iterables.partition(missedBuildsIds, 32 * 10).forEach(
+ chunk -> {
fatBuildDao.getAllFatBuilds(srvId, chunk).forEach((buildCacheKey, fatBuildCompacted) -> {
- if(!applicableForHistory(fatBuildCompacted))
+ if (!applicableForHistory(fatBuildCompacted))
return;
- SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
+ BiPredicate<Integer, Integer> paramsFilter = (k, v) -> false;
+
+ SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, paramsFilter);
Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
tests.forEach(
testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, paramsFilter, successStatusStrId);
sinv.addTest(testCompacted.testName(), invocation);
}
@@ -288,45 +303,13 @@ public class HistoryCollector {
return suiteRunHist;
}
- /*
- @AutoProfiling
- protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
- Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
-
- int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
-
- CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new FatBuildDao.HistoryCollectProcessor(successStatusStrId);
-
- Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
-
- SuiteHistory hist = new SuiteHistory();
-
- map.values().forEach(
- res -> {
- if (res == null)
- return;
-
- Map<Integer, Invocation> invocationMap = res.get();
-
- if (invocationMap == null)
- return;
-
- invocationMap.forEach((k, v) -> {
- RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
- k_ -> new RunHistCompacted());
-
- compacted.innerAddInvocation(v);
- });
-
- }
- );
-
- System.err.println("Suite history: tests in scope "
- + hist.testsHistory.size()
- + " for " +buildIds.size() + " builds checked"
- + " size " + hist.size(igniteProvider.get()));
-
- return hist;
+ /**
+ * @param srvId Server id.
+ * @param buildTypeId Build type id.
+ * @param normalizedBaseBranch Normalized base branch.
+ */
+ public IRunHistory getSuiteRunHist(int srvId, int buildTypeId, int normalizedBaseBranch) {
+ return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch).getSuiteHist();
}
- */
+
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index ffc0c47..a5bb997 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -260,7 +260,7 @@ public class RunHistCompactedDao {
return res;
}
- public void setBuildsStartTime(int srvId, Map<Integer, Long> builds) {
+ public void setBuildsStartTimeAsync(int srvId, Map<Integer, Long> builds) {
Map<Long, Long> res = new HashMap<>();
builds.forEach((buildId, ts) -> {
@@ -268,6 +268,6 @@ public class RunHistCompactedDao {
res.put(buildIdToCacheKey(srvId, buildId), ts);
});
- buildStartTime.putAll(res);
+ buildStartTime.putAllAsync(res);
}
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
index 8bc8d95..7342af7 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -46,9 +46,9 @@ public class SuiteInvocation {
@QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)})
private int normalizedBranchName;
- Invocation suite;
+ private Invocation suite;
- java.util.Map<Integer, Invocation> tests = new HashMap<>();
+ private Map<Integer, Invocation> tests = new HashMap<>();
Long buildStartTime;
@@ -70,4 +70,8 @@ public class SuiteInvocation {
public Map<Integer, Invocation> tests() {
return Collections.unmodifiableMap(tests);
}
+
+ public Invocation suiteInvocation() {
+ return suite;
+ }
}