You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/20 14:22:27 UTC
[ignite-teamcity-bot] 04/07: Trusted tests & suite history
performance fixes: Caching of test run history in the context
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit f01b323fd5bff8f9f54f43f7cf121cd7cf3e2943
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Jun 19 21:43:36 2019 +0300
Trusted tests & suite history performance fixes: Caching of test run history in the context
---
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 15 ++++----
.../tcbot/engine/chain/TestCompactedMult.java | 10 ++++++
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 5 ++-
.../apache/ignite/tcbot/engine/ui/DsChainUi.java | 15 ++++----
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 42 ++++++++++++----------
.../ignite/tcbot/engine/ui/DsTestFailureUi.java | 41 ++++++++++-----------
.../ignite/tcbot/engine/ui/DsTestHistoryUi.java | 2 +-
7 files changed, 70 insertions(+), 60 deletions(-)
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index dac6bf1..5fb0b2d 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -305,8 +305,8 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(logSizeBytes.entrySet().stream(), 3, comparing).stream();
}
- public Stream<? extends IMultTestOccurrence> getTopLongRunning() {
- Comparator<IMultTestOccurrence> comparing = Comparator.comparing(IMultTestOccurrence::getAvgDurationMs);
+ public Stream<TestCompactedMult> getTopLongRunning() {
+ Comparator<TestCompactedMult> comparing = Comparator.comparing(TestCompactedMult::getAvgDurationMs);
Map<Integer, TestCompactedMult> res = new HashMap<>();
@@ -609,17 +609,14 @@ public class MultBuildRunCtx implements ISuiteResults {
//todo can cache mult occurrences in ctx
builds.forEach(singleBuildRunCtx -> {
- saveToMap(res, singleBuildRunCtx.getAllTests()
- .filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
+ saveToMap(res,
+ singleBuildRunCtx.getAllTests().filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
Integer suiteName = buildTypeIdId();
- // res.clear(); //todo enable feature back
-
- //todo can cache fail rate in mult occur
- res.keySet().forEach((testNameId) -> {
- IRunHistory stat = tcIgnited.getTestRunHist(testNameId, suiteName, branchName);
+ res.forEach((testNameId, compactedMult) -> {
+ IRunHistory stat = compactedMult.history(tcIgnited, suiteName, branchName);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
boolean b = testBlockerComment != null;
if (b) // this test will be considered as blocker if will fail
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
index 792d5c6..b1ab48f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
@@ -20,11 +20,14 @@ package org.apache.ignite.tcbot.engine.chain;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.history.IRunHistSummary;
+import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
@@ -35,6 +38,7 @@ public class TestCompactedMult implements IMultTestOccurrence {
private final List<TestCompacted> occurrences = new ArrayList<>();
private IStringCompactor compactor;
private long avgDuration = -1;
+ private java.util.Map<Integer, IRunHistory> historyCacheMap = new ConcurrentHashMap<>();
public TestCompactedMult(IStringCompactor compactor) {
this.compactor = compactor;
@@ -113,4 +117,10 @@ public class TestCompactedMult implements IMultTestOccurrence {
public void add(TestCompacted next) {
occurrences.add(next);
}
+
+
+ public IRunHistory history(ITeamcityIgnited ignited, Integer buildTypeIdId, Integer baseBranchId) {
+ return historyCacheMap.computeIfAbsent(baseBranchId,
+ (k)-> ignited.getTestRunHist(testName(), buildTypeIdId, k));
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index b3fb4bd..0e93b57 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -265,14 +265,13 @@ public class PrChainsProcessor {
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
List<DsTestFailureUi> failures = ctx.getFailedTests().stream().map(occurrence -> {
- IRunHistory stat = tcIgnited.getTestRunHist(occurrence.testName(), suiteId, baseBranchId);
-
+ IRunHistory stat = occurrence.history(tcIgnited, suiteId, baseBranchId);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
if (!Strings.isNullOrEmpty(testBlockerComment)) {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch);
+ failure.initFromOccurrence(occurrence, ctx.buildTypeIdId(), tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch, baseBranchId);
return failure;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
index 913c08a..d8c8746 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
@@ -24,13 +24,12 @@ import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;
import javax.annotation.Nullable;
-
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.tcbot.common.util.CollectionUtil;
import org.apache.ignite.tcbot.common.util.UrlUtil;
import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
-import org.apache.ignite.tcbot.common.util.CollectionUtil;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcservice.model.conf.BuildType;
@@ -208,18 +207,18 @@ public class DsChainUi {
webToHist = buildWebLink(tcIgnited, ctx);
webToBuild = buildWebLinkToBuild(tcIgnited, ctx);
- Stream<T2<MultBuildRunCtx, IMultTestOccurrence>> allLongRunning = ctx.suites().flatMap(
+ Stream<T2<MultBuildRunCtx, TestCompactedMult>> allLongRunning = ctx.suites().flatMap(
suite -> suite.getTopLongRunning().map(t -> new T2<>(suite, t))
);
- Comparator<T2<MultBuildRunCtx, IMultTestOccurrence>> durationComp
+ Comparator<T2<MultBuildRunCtx, TestCompactedMult>> durationComp
= Comparator.comparing((pair) -> pair.get2().getAvgDurationMs());
CollectionUtil.top(allLongRunning, 3, durationComp).forEach(
pairCtxAndOccur -> {
MultBuildRunCtx suite = pairCtxAndOccur.get1();
- IMultTestOccurrence longRunningOccur = pairCtxAndOccur.get2();
+ TestCompactedMult longRunningOccur = pairCtxAndOccur.get2();
- DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, longRunningOccur, baseBranchTc);
+ DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, longRunningOccur, baseBranchTc);
failure.testName = "[" + suite.suiteName() + "] " + failure.testName; //may be separate field
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 9f2b021..8a1b003 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -29,20 +29,18 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.tcbot.common.util.UrlUtil;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
import org.apache.ignite.tcbot.engine.ui.BotUrls.GetBuildLog;
-import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
-import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
+import org.apache.ignite.tcignited.history.IRunHistory;
-import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable;
+import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
@@ -165,9 +163,11 @@ public class DsSuiteUi extends DsHistoryStatUi {
name = suite.suiteName();
String failRateNormalizedBranch = normalizeBranch(baseBranch);
- String curBranchNormalized = normalizeBranch(suite.branchName());
Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch);
+ String curBranchNormalized = normalizeBranch(suite.branchName());
+ Integer curBranchId = compactor.getStringIdIfPresent(curBranchNormalized);
+
IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId());
Set<String> collect = suite.lastChangeUsers().collect(Collectors.toSet());
@@ -188,26 +188,28 @@ public class DsSuiteUi extends DsHistoryStatUi {
webToHistBaseBranch = buildWebLink(tcIgnited, suite, baseBranch);
webToBuild = buildWebLinkToBuild(tcIgnited, suite);
+ Integer buildTypeIdId = suite.buildTypeIdId();
if (includeTests) {
List<TestCompactedMult> tests = suite.getFailedTests();
- Function<TestCompactedMult, Float> function = foccur -> {
- IRunHistory apply = tcIgnited.getTestRunHist(foccur.testName(), suite.buildTypeIdId(), baseBranchId);
+ Function<TestCompactedMult, Float> function = testCompactedMult -> {
+ IRunHistory res = testCompactedMult.history(tcIgnited, buildTypeIdId, baseBranchId);
- return apply == null ? 0f : apply.getFailRate();
+ return res == null ? 0f : res.getFailRate();
};
tests.sort(Comparator.comparing(function).reversed());
tests.forEach(occurrence -> {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), baseBranch);
- failure.initStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized);
+ failure.initFromOccurrence(occurrence, buildTypeIdId, tcIgnited, suite.projectId(),
+ suite.branchName(), baseBranch, baseBranchId);
+ failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, curBranchId);
testFailures.add(failure);
});
suite.getTopLongRunning().forEach(occurrence -> {
- final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, occurrence, baseBranch);
+ final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, occurrence, baseBranch);
topLongRunning.add(failure);
});
@@ -317,16 +319,18 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
@Nonnull public static DsTestFailureUi createOrrucForLongRun(ITeamcityIgnited tcIgnited,
- @Nonnull MultBuildRunCtx suite,
- final IMultTestOccurrence occurrence,
- @Nullable final String failRateBranch) {
+ IStringCompactor compactor, @Nonnull MultBuildRunCtx suite,
+ final TestCompactedMult occurrence,
+ @Nullable final String failRateBranch) {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), failRateBranch);
+ Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(failRateBranch));
+ Integer buildTypeIdId = suite.buildTypeIdId();
+ failure.initFromOccurrence(occurrence, buildTypeIdId, tcIgnited, suite.projectId(), suite.branchName(),
+ failRateBranch, baseBranchId);
- failure.initStat(tcIgnited,
- normalizeBranch(failRateBranch),
- normalizeBranch(suite.branchName()));
+ failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId,
+ compactor.getStringIdIfPresent(normalizeBranch(suite.branchName())));
return failure;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
index b0cb5d9..9c146f9 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
@@ -25,17 +25,15 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.tcbot.common.util.UrlUtil;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildlog.LogMsgToWarn;
import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcignited.ITeamcityIgnited;
-import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable;
+import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
@@ -92,16 +90,20 @@ public class DsTestFailureUi {
/**
* @param failure test ocurrence (probably multiple)
+ * @param buildTypeIdId
* @param tcIgn Teamcity.
* @param projectId project ID.
- * @param branchName
- * @param baseBranchName base branch name (e.g. master).
+ * @param branchName current branch name.
+ * @param baseBranchName base branch name (e.g. master), without normalization.
+ * @param baseBranchId Normalized base branch ID (from compactor).
*/
- public void initFromOccurrence(@Nonnull final IMultTestOccurrence failure,
+ public void initFromOccurrence(@Nonnull final TestCompactedMult failure,
+ Integer buildTypeIdId,
@Nonnull final ITeamcityIgnited tcIgn,
@Nullable final String projectId,
@Nullable final String branchName,
- @Nullable final String baseBranchName) {
+ @Nullable final String baseBranchName,
+ Integer baseBranchId) {
name = failure.getName();
investigated = failure.isInvestigated();
curFailures = failure.failuresCount();
@@ -144,8 +146,7 @@ public class DsTestFailureUi {
webUrlBaseBranch = buildWebLink(tcIgn, full.test.id, projectId, baseBranchName);
});
- final IRunHistory stat = tcIgn.getTestRunHist(name, normalizeBranch(baseBranchName));
-
+ final IRunHistory stat = failure.history(tcIgn, buildTypeIdId, baseBranchId);
blockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
}
@@ -191,22 +192,22 @@ public class DsTestFailureUi {
}
/**
+ * @param occurrence
+ * @param buildTypeIdId
* @param tcIgnited TC service as Run stat supplier.
- * @param failRateNormalizedBranch Base branch: Fail rate and flakyness detection normalized branch.
+ * @param baseBranchId Base branch: Fail rate and flakyness detection normalized branch.
* @param curBranchNormalized Cur branch normalized.
*/
- public void initStat(ITeamcityIgnited tcIgnited,
- String failRateNormalizedBranch,
- String curBranchNormalized) {
-
- final IRunHistory stat = tcIgnited.getTestRunHist(name, failRateNormalizedBranch);
-
+ public void initStat(TestCompactedMult occurrence, Integer buildTypeIdId, ITeamcityIgnited tcIgnited,
+ @Nullable Integer baseBranchId,
+ @Nullable Integer curBranchNormalized) {
+ final IRunHistory stat = occurrence.history(tcIgnited, buildTypeIdId, baseBranchId);
histBaseBranch.init(stat);
- IRunHistory statForProblemsDetection = null;
+ IRunHistory statForProblemsDetection;
- if (!curBranchNormalized.equals(failRateNormalizedBranch)) {
- statForProblemsDetection = tcIgnited.getTestRunHist(name, curBranchNormalized);
+ if (!Objects.equals(curBranchNormalized, baseBranchId)) {
+ statForProblemsDetection = occurrence.history(tcIgnited, buildTypeIdId, curBranchNormalized);
if (statForProblemsDetection != null) {
histCurBranch = new DsTestHistoryUi();
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
index 8b8597f..e0fe975 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
@@ -27,7 +27,7 @@ import org.apache.ignite.tcignited.history.IRunHistory;
*/
public class DsTestHistoryUi {
/** 'All the time' runs history statistic. */
- public DsHistoryStatUi allTime = new DsHistoryStatUi();
+ @Deprecated public DsHistoryStatUi allTime = new DsHistoryStatUi();
/** Latest runs history statistic. */
public DsHistoryStatUi recent = new DsHistoryStatUi();