You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/07 14:39:19 UTC
[ignite-teamcity-bot] branch master updated: Trusted and total
tests count: initial implementation (#128)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 912f8f5 Trusted and total tests count: initial implementation (#128)
912f8f5 is described below
commit 912f8f50502b5f61604cf5f01b1b76df44a2ac1a
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jun 7 17:39:14 2019 +0300
Trusted and total tests count: initial implementation (#128)
---
.../ignite/ci/analysis/IMultTestOccurrence.java | 2 -
.../apache/ignite/ci/analysis/ISuiteResults.java | 15 ---
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 89 +++++++++++--
.../ignite/ci/analysis/SingleBuildRunCtx.java | 36 +++--
.../ignite/ci/analysis/TestCompactedMult.java | 41 +++---
.../ignite/ci/runners/RemoteClientTmpHelper.java | 2 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 5 +-
.../ignite/ci/tcbot/chain/PrChainsProcessor.java | 13 +-
.../tcbot/chain/TrackedBranchChainsProcessor.java | 5 +-
.../ignite/ci/tcbot/issue/IssueDetector.java | 8 +-
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../model/current/ChainAtServerCurrentStatus.java | 39 +++++-
.../ci/web/model/current/SuiteCurrentStatus.java | 33 ++++-
.../ignite/ci/web/model/current/TestFailure.java | 3 +-
.../ci/web/model/current/TestFailuresSummary.java | 4 +-
.../ignite/ci/web/model/current/UpdateInfo.java | 4 +-
.../ignite/ci/web/rest/GetChainResultsAsHtml.java | 2 +-
.../ci/web/rest/build/GetBuildTestFailures.java | 2 +-
.../rest/tracked/GetTrackedBranchTestResults.java | 36 ++---
ignite-tc-helper-web/src/main/webapp/current.html | 6 +
.../src/main/webapp/js/testfails-2.2.js | 53 +++++---
.../ci/tcbot/chain/TrackedBranchProcessorTest.java | 4 +-
.../ci/teamcity/ignited/BuildKeyUnitTest.java | 2 +-
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 4 +-
.../IgnitedTcInMemoryIntegrationTest.java | 4 +-
.../teamcity/ignited/change/ChangeCompacted.java | 2 +-
.../teamcity/ignited/change/RevisionCompacted.java | 2 +-
.../ignited/fatbuild/FatBuildCompacted.java | 44 +++++--
.../teamcity/ignited/fatbuild/TestCompacted.java | 9 +-
.../teamcity/ignited/runhist/RunHistCompacted.java | 33 +++--
.../ci/teamcity/ignited/runhist/RunHistKey.java | 6 +-
.../apache/ignite/tcignited/ITeamcityIgnited.java | 10 ++
.../ignite/tcignited/TeamcityIgnitedImpl.java | 115 ++++++++++++----
.../ignite/tcignited/TeamcityIgnitedModule.java | 4 +-
.../fatbuild => tcignited/build}/FatBuildDao.java | 145 ++++++++++++++++++++-
.../build}/ProactiveFatBuildSync.java | 5 +-
.../ignite/tcignited/build/SuiteHistory.java | 29 ++---
.../ignite/tcignited/buildref/BuildRefDao.java | 122 ++++++++++++-----
.../ignite/tcignited/buildref/BuildRefSync.java | 2 +-
.../ignite/tcignited/history/IRunHistSummary.java | 30 ++---
.../ignite/tcignited/history/IRunHistory.java | 11 +-
.../apache/ignite/tcignited/history/IRunStat.java | 3 +-
.../tcignited/history/RunHistCompactedDao.java | 15 ++-
.../ignite/tcignited/history/RunHistSync.java | 7 +-
44 files changed, 751 insertions(+), 257 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
index 095301c..a403219 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
@@ -33,6 +33,4 @@ public interface IMultTestOccurrence {
public long getAvgDurationMs();
Iterable<TestOccurrenceFull> getOccurrences();
-
- String getPossibleBlockerComment(IRunHistory baseBranchStat);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ISuiteResults.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ISuiteResults.java
index b4b49b7..d981ff7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ISuiteResults.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ISuiteResults.java
@@ -42,21 +42,6 @@ public interface ISuiteResults {
/** */
public boolean hasBuildMessageProblem();
- default boolean hasCriticalProblem() {
- return hasJvmCrashProblem()
- || hasTimeoutProblem()
- || hasCompilationProblem()
- || hasMetricProblem();
- }
-
- default boolean hasSuiteIncompleteFailure() {
- return hasJvmCrashProblem()
- || hasTimeoutProblem()
- || hasOomeProblem()
- || hasExitCodeProblem()
- || hasCompilationProblem()
- || hasMetricProblem();
- }
public String suiteId();
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index 516a980..5fdfb8e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -18,12 +18,28 @@
package org.apache.ignite.ci.analysis;
import com.google.common.base.Strings;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalDouble;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcignited.history.IRunHistory;
@@ -33,13 +49,6 @@ import org.apache.ignite.tcservice.model.result.stat.Statistics;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import javax.annotation.Nonnull;
-import java.util.*;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
/**
* Run configuration execution results loaded from different API URLs. Includes tests and problem occurrences; if logs
* processing is done also contains last started test
@@ -47,10 +56,12 @@ import java.util.stream.Stream;
public class MultBuildRunCtx implements ISuiteResults {
/** Cancelled. */
public static final String CANCELLED = "CANCELLED";
+ public static final int LOG_CONSUMER_BORDER_BYTES = 1024 * 1024;
/** First build info. */
@Nonnull private final BuildRef firstBuildInfo;
+ /** String Compactor. */
private IStringCompactor compactor;
/** Builds: Single execution. */
@@ -113,6 +124,23 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().flatMap(SingleBuildRunCtx::getProblemsStream);
}
+
+ /**
+ * @param problemCode Problem code.
+ * @return count of builds having particular problem type
+ */
+ public long buildsCntHavingBuildProblem(String problemCode) {
+ int problemCodeId = compactor.getStringId(problemCode);
+ int cnt = 0;
+
+ for (SingleBuildRunCtx ctx : builds) {
+ if (ctx.hasBuildProblemType(problemCodeId))
+ cnt++;
+ }
+
+ return cnt;
+ }
+
public List<SingleBuildRunCtx> getBuilds() {
return builds;
}
@@ -155,7 +183,7 @@ public class MultBuildRunCtx implements ISuiteResults {
}
public long getJvmCrashProblemCount() {
- return buildsStream().filter(ISuiteResults::hasJvmCrashProblem).count();
+ return buildsCntHavingBuildProblem(ProblemOccurrence.TC_JVM_CRASH);
}
public boolean hasOomeProblem() {
@@ -264,9 +292,8 @@ public class MultBuildRunCtx implements ISuiteResults {
(testName, logCheckResult) -> {
//todo may be it is better to find avg
long bytes = (long)logCheckResult.getLogSizeBytes();
- if (bytes > 1024 * 1024) {
+ if (bytes > LOG_CONSUMER_BORDER_BYTES)
logSizeBytes.merge(testName, bytes, Math::max);
- }
}
);
});
@@ -309,7 +336,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return firstBuildInfo.getId();
}
- boolean isFailed() {
+ public boolean isFailed() {
return failedTests() != 0 || hasAnyBuildProblemExceptTestOrSnapshot() || onlyCancelledBuilds();
}
@@ -560,4 +587,44 @@ public class MultBuildRunCtx implements ISuiteResults {
return res;
}
+
+ public boolean hasCriticalProblem() {
+ return hasJvmCrashProblem()
+ || hasTimeoutProblem()
+ || hasCompilationProblem()
+ || hasMetricProblem();
+ }
+
+ public Integer totalTests() {
+ return (int)buildsStream().mapToInt(SingleBuildRunCtx::totalNotMutedTests).average().orElse(0.0);
+ }
+
+ public Integer trustedTests(ITeamcityIgnited tcIgnited,
+ String normalizedBaseBranch) {
+
+ AtomicInteger trustedCnt = new AtomicInteger();
+ Map<Integer, TestCompactedMult> res = new HashMap<>();
+
+ //todo can cache mult occurrences in ctx
+ builds.forEach(singleBuildRunCtx -> {
+ saveToMap(res, singleBuildRunCtx.getAllTests()
+ .filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
+ });
+ Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
+ Integer suiteName = compactor.getStringIdIfPresent( buildTypeId());
+
+ // res.clear(); //todo enable feature back
+
+ //todo can cache fail rate in mult occur
+ res.keySet().forEach((testNameId) -> {
+ IRunHistory stat = tcIgnited.getTestRunHist(testNameId, suiteName, branchName);
+ String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
+ boolean b = testBlockerComment != null;
+ if (b) // this test will be considered as blocker if will fail
+ trustedCnt.addAndGet(1);
+ });
+
+ return trustedCnt.get();
+ }
+
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
index 683072f..6854920 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
@@ -32,20 +32,19 @@ import java.util.concurrent.Future;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
-
-import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
-import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
-import org.apache.ignite.tcservice.ITeamcity;
-import org.apache.ignite.tcbot.common.conf.IBuildParameterSpec;
-import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
-import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
-import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.util.FutureUtil;
+import org.apache.ignite.tcbot.common.conf.IBuildParameterSpec;
+import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
+import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
+import org.apache.ignite.tcservice.ITeamcity;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
import org.jetbrains.annotations.Nullable;
/**
@@ -95,6 +94,7 @@ public class SingleBuildRunCtx implements ISuiteResults {
return getProblemsStream().anyMatch(p -> p.isCompilationError(compactor));
}
+
public boolean hasTimeoutProblem() {
return getExecutionTimeoutCount() > 0;
}
@@ -272,6 +272,9 @@ public class SingleBuildRunCtx implements ISuiteResults {
this.tags.add(lb);
}
+ /**
+ * @return Build tags associated with this run. May be Java version, area of responibilty - any text mark.
+ */
public Set<String> tags() {
return tags;
}
@@ -325,4 +328,21 @@ public class SingleBuildRunCtx implements ISuiteResults {
return propVal;
}
+
+ public boolean hasBuildProblemType(int id) {
+ return buildCompacted.hasBuildProblemType(id);
+ }
+
+ public boolean hasSuiteIncompleteFailure() {
+ return hasJvmCrashProblem()
+ || hasTimeoutProblem()
+ || hasOomeProblem()
+ || hasExitCodeProblem()
+ || hasCompilationProblem()
+ || hasMetricProblem();
+ }
+
+ public int totalNotMutedTests() {
+ return buildCompacted.totalNotMutedTests();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
index f0dece6..45b548f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
@@ -21,11 +21,15 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
-import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
-import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.history.IRunHistSummary;
+import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
+/**
+ * Test occurrence merged from several runs.
+ */
public class TestCompactedMult implements IMultTestOccurrence {
private final List<TestCompacted> occurrences = new ArrayList<>();
private IStringCompactor compactor;
@@ -78,22 +82,29 @@ public class TestCompactedMult implements IMultTestOccurrence {
.collect(Collectors.toList());
}
- @Override public String getPossibleBlockerComment(IRunHistory baseBranchStat) {
- if (baseBranchStat == null)
- return "History for base branch is absent.";
+ /**
+ * @param baseBranchStat Base branch statistics.
+ * @return non null comment in case test failure is a blocker for merge into base branch.
+ */
+ public static String getPossibleBlockerComment(IRunHistSummary baseBranchStat) {
+ if (baseBranchStat == null)
+ return "History for base branch is absent.";
- String flakyComments = baseBranchStat.getFlakyComments();
+ boolean flaky = baseBranchStat.isFlaky();
- boolean lowFailureRate = baseBranchStat.getFailRate() * 100.0f < 4.;
+ float failRate = baseBranchStat.getFailRate();
+ boolean lowFailureRate = failRate * 100.0f < 4.;
- if (lowFailureRate && flakyComments == null) {
- return "Test has low fail rate in base branch "
- + baseBranchStat.getFailPercentPrintable()
- + "% and is not flaky";
- }
+ if (lowFailureRate && !flaky) {
+ String runStatPrintable = IRunStat.getPercentPrintable(failRate * 100.0f);
- return null;
- }
+ return "Test has low fail rate in base branch "
+ + runStatPrintable
+ + "% and is not flaky";
+ }
+
+ return null;
+ }
public void add(TestCompacted next) {
occurrences.add(next);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
index 9735909..de23a26 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
@@ -38,7 +38,7 @@ import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.ci.user.TcHelperUser;
import org.apache.ignite.tcservice.util.XmlUtil;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 7f5333e..8d0bfbb 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -361,13 +361,14 @@ public class BuildChainProcessor {
protected void analyzeTests(MultBuildRunCtx outCtx, ITeamcityIgnited teamcity,
ProcessLogsMode procLog) {
for (SingleBuildRunCtx ctx : outCtx.getBuilds()) {
- if ((procLog == ProcessLogsMode.SUITE_NOT_COMPLETE && ctx.hasSuiteIncompleteFailure())
+ boolean incompleteFailure = ctx.hasSuiteIncompleteFailure();
+ if ((procLog == ProcessLogsMode.SUITE_NOT_COMPLETE && incompleteFailure)
|| procLog == ProcessLogsMode.ALL)
ctx.setLogCheckResFut(
CompletableFuture.supplyAsync(
() -> buildLogProcessor.analyzeBuildLog(teamcity,
ctx.buildId(),
- ctx.hasSuiteIncompleteFailure()),
+ incompleteFailure),
tcUpdatePool.getService()));
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java
index 9247de2..eaf0b7a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.ignite.ci.analysis.FullChainRunCtx;
+import org.apache.ignite.ci.analysis.TestCompactedMult;
import org.apache.ignite.ci.analysis.mode.LatestRebuildMode;
import org.apache.ignite.ci.analysis.mode.ProcessLogsMode;
import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited;
@@ -149,7 +150,7 @@ public class PrChainsProcessor {
runningUpdates.addAndGet(cnt0);
//fail rate reference is always default (master)
- chainStatus.initFromContext(tcIgnited, ctx, baseBranch, compactor);
+ chainStatus.initFromContext(tcIgnited, ctx, baseBranch, compactor, false); // don't need for PR
chainStatus.initJiraAndGitInfo(ticketMatcher, jiraIntegration, gitHubConnIgnited);
}
@@ -185,8 +186,8 @@ public class PrChainsProcessor {
String baseBranch = ITeamcity.DEFAULT;
- final FullChainRunCtx ctx = buildChainProcessor.loadFullChainContext(
- tcIgnited,
+ FullChainRunCtx ctx = buildChainProcessor.loadFullChainContext(
+ tcIgnited,
hist,
LatestRebuildMode.LATEST,
ProcessLogsMode.SUITE_NOT_COMPLETE,
@@ -217,10 +218,10 @@ public class PrChainsProcessor {
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
- List<TestFailure> failures = ctx.getFailedTests().stream().map(occurrence -> {
+ List<TestFailure> failures = ctx.getFailedTests().stream().map(occurrence -> {
IRunHistory stat = tcIgnited.getTestRunHist(occurrence.getName(), normalizedBaseBranch);
- String testBlockerComment = occurrence.getPossibleBlockerComment(stat);
+ String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
if (!Strings.isNullOrEmpty(testBlockerComment)) {
final TestFailure failure = new TestFailure();
@@ -239,7 +240,7 @@ public class PrChainsProcessor {
SuiteCurrentStatus suiteUi = new SuiteCurrentStatus();
suiteUi.testFailures = failures;
- suiteUi.initFromContext(tcIgnited, ctx, baseBranch, compactor, false);
+ suiteUi.initFromContext(tcIgnited, ctx, baseBranch, compactor, false, false);
return suiteUi;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java
index 7289f9b..4283bdf 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java
@@ -63,7 +63,8 @@ public class TrackedBranchChainsProcessor {
@Nullable Boolean checkAllLogs,
int buildResMergeCnt,
ITcBotUserCreds creds,
- SyncMode syncMode) {
+ SyncMode syncMode,
+ boolean calcTrustedTests) {
final TestFailuresSummary res = new TestFailuresSummary();
final AtomicInteger runningUpdates = new AtomicInteger();
@@ -118,7 +119,7 @@ public class TrackedBranchChainsProcessor {
if (cnt > 0)
runningUpdates.addAndGet(cnt);
- chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc, compactor);
+ chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc, compactor, calcTrustedTests);
return chainStatus;
})
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index 4119c66..d309443 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -48,7 +48,6 @@ import org.apache.ignite.ci.mail.SlackSender;
import org.apache.ignite.ci.tcbot.chain.TrackedBranchChainsProcessor;
import org.apache.ignite.ci.tcbot.conf.INotificationChannel;
import org.apache.ignite.ci.tcbot.conf.ITcBotConfig;
-import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.ci.tcbot.conf.NotificationsConfig;
import org.apache.ignite.ci.tcbot.user.IUserStorage;
import org.apache.ignite.tcignited.history.IRunHistory;
@@ -504,15 +503,16 @@ public class IssueDetector {
false,
buildsToQry,
creds,
- SyncMode.RELOAD_QUEUED);
+ SyncMode.RELOAD_QUEUED,
+ false);
TestFailuresSummary failures =
tbProc.getTrackedBranchTestFailures(brachName,
false,
1,
creds,
- SyncMode.RELOAD_QUEUED
- );
+ SyncMode.RELOAD_QUEUED,
+ false);
String issRes = registerIssuesAndNotifyLater(failures, backgroundOpsCreds);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index 29df77b..7457916 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
/** TC Bot Version. */
- public static final String VERSION = "20190605";
+ public static final String VERSION = "20190607";
/** Java version, where Web App is running. */
public String javaVer;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
index ba20658..1e62199 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
@@ -33,11 +33,11 @@ import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited;
import org.apache.ignite.ci.github.pure.IGitHubConnection;
import org.apache.ignite.ci.jira.ignited.IJiraIgnited;
import org.apache.ignite.ci.tcbot.visa.BranchTicketMatcher;
-import org.apache.ignite.tcservice.model.conf.BuildType;
-import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcservice.model.conf.BuildType;
import static org.apache.ignite.ci.util.UrlUtil.escape;
import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchForLink;
@@ -88,7 +88,15 @@ public class ChainAtServerCurrentStatus {
/** Suites involved in chain. */
public List<SuiteCurrentStatus> suites = new ArrayList<>();
+ /** Count of failed tests not muted tests. In case several runs are used, overall by all runs. */
public Integer failedTests;
+
+ /** Total (Not Muted/Not Ignored) tests in all suites. */
+ public Integer totalTests;
+
+ /** Tests which will be considered as a blocker. */
+ public Integer trustedTests;
+
/** Count of suites with critical build problems found */
public Integer failedToFinish;
@@ -191,9 +199,13 @@ public class ChainAtServerCurrentStatus {
public void initFromContext(ITeamcityIgnited tcIgnited,
FullChainRunCtx ctx,
- @Nullable String baseBranchTc, IStringCompactor compactor) {
+ @Nullable String baseBranchTc,
+ IStringCompactor compactor,
+ boolean calcTrustedTests) {
failedTests = 0;
failedToFinish = 0;
+ totalTests = 0;
+ trustedTests = 0;
//todo mode with not failed
Stream<MultBuildRunCtx> stream = ctx.failedChildSuites();
@@ -201,9 +213,11 @@ public class ChainAtServerCurrentStatus {
suite -> {
final SuiteCurrentStatus suiteCurStatus = new SuiteCurrentStatus();
- suiteCurStatus.initFromContext(tcIgnited, suite, baseBranchTc, compactor, true);
+ suiteCurStatus.initFromContext(tcIgnited, suite, baseBranchTc, compactor, true, calcTrustedTests);
- failedTests += suiteCurStatus.failedTests;
+ failedTests += suiteCurStatus.failedTests != null ? suiteCurStatus.failedTests : 0;
+ totalTests += suiteCurStatus.totalTests != null ? suiteCurStatus.totalTests : 0;
+ trustedTests += suiteCurStatus.trustedTests != null ? suiteCurStatus.trustedTests : 0;
if (suite.hasAnyBuildProblemExceptTestOrSnapshot() || suite.onlyCancelledBuilds())
failedToFinish++;
@@ -211,6 +225,19 @@ public class ChainAtServerCurrentStatus {
}
);
+ if(calcTrustedTests) {
+ //todo odd convertion
+ ctx.suites().filter(s -> !s.isFailed()).forEach(suite -> {
+
+ final SuiteCurrentStatus suiteCurStatus = new SuiteCurrentStatus();
+
+ suiteCurStatus.initFromContext(tcIgnited, suite, baseBranchTc, compactor, true, calcTrustedTests);
+
+ totalTests += suiteCurStatus.totalTests != null ? suiteCurStatus.totalTests : 0;
+ trustedTests += suiteCurStatus.trustedTests != null ? suiteCurStatus.trustedTests : 0;
+ });
+ }
+
totalBlockers = suites.stream().mapToInt(SuiteCurrentStatus::totalBlockers).sum();
durationPrintable = ctx.getDurationPrintable();
testsDurationPrintable = ctx.getTestsDurationPrintable();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
index 398b7c8..dc04d07 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
@@ -46,9 +46,9 @@ import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrint
import static org.apache.ignite.ci.util.UrlUtil.escape;
/**
- * Represent Suite result
+ * Represent Suite result, UI class for REST responses, so it contains public fields
*/
-@SuppressWarnings("WeakerAccess") public class SuiteCurrentStatus extends FailureSummary {
+@SuppressWarnings({"WeakerAccess", "PublicField"}) public class SuiteCurrentStatus extends FailureSummary {
/** Suite Name */
public String name;
@@ -100,18 +100,28 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
/** TcHelperUser commits, comma separated string. */
public String userCommits = "";
+ /** Count of failed tests not muted tests. In case several runs are used, overall by all runs. */
public Integer failedTests;
+ /** Count of executed tests during current run. In case several runs are used, average number. */
+ public Integer totalTests;
+
+ /** Tests which will not considered as a blocker and not filtered out. */
+ public Integer trustedTests;
+
/** Duration printable. */
public String durationPrintable;
/** Duration net time printable. */
public String durationNetTimePrintable;
+ /** Source update duration printable. */
public String sourceUpdateDurationPrintable;
+ /** Artifcact publishing duration printable. */
public String artifcactPublishingDurationPrintable;
+ /** Dependecies resolving duration printable. */
public String dependeciesResolvingDurationPrintable;
/** Tests duration printable. */
@@ -125,6 +135,7 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
*/
@Nullable public ProblemRef problemRef;
+ /** Tags for build. */
@Nonnull public Set<String> tags = new HashSet<>();
/**
@@ -133,11 +144,20 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
*/
@Nullable public String blockerComment;
+ /**
+ * @param tcIgnited Tc ignited.
+ * @param suite Suite.
+ * @param baseBranch Base branch.
+ * @param compactor String Compactor.
+ * @param includeTests Include tests - usually {@code true}, but it may be disabled for speeding up VISA collection.
+ * @param calcTrustedTests
+ */
public SuiteCurrentStatus initFromContext(ITeamcityIgnited tcIgnited,
@Nonnull final MultBuildRunCtx suite,
@Nullable final String baseBranch,
@Nonnull IStringCompactor compactor,
- boolean includeTests) {
+ boolean includeTests,
+ boolean calcTrustedTests) {
name = suite.suiteName();
@@ -201,7 +221,6 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
(testName, logCheckResult) -> {
if (logCheckResult.hasWarns())
this.findFailureAndAddWarning(testName, logCheckResult);
-
}
);
}
@@ -210,6 +229,12 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
suite.getTopLogConsumers().forEach(
(entry) -> logConsumers.add(createOccurForLogConsumer(entry))
);
+
+
+ totalTests = suite.totalTests();
+
+ if(calcTrustedTests)
+ trustedTests = suite.trustedTests(tcIgnited, failRateNormalizedBranch);
}
suite.getBuildsWithThreadDump().forEach(buildId -> {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
index c50b6a7..2e6f269 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
@@ -26,6 +26,7 @@ import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.ignite.ci.analysis.IMultTestOccurrence;
+import org.apache.ignite.ci.analysis.TestCompactedMult;
import org.apache.ignite.ci.issue.EventTemplates;
import org.apache.ignite.ci.issue.ProblemRef;
import org.apache.ignite.tcignited.buildlog.LogMsgToWarn;
@@ -146,7 +147,7 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
final IRunHistory stat = tcIgn.getTestRunHist(name, normalizeBranch(baseBranchName));
- blockerComment = failure.getPossibleBlockerComment(stat);
+ blockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailuresSummary.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailuresSummary.java
index 1f92eb6..634053b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailuresSummary.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailuresSummary.java
@@ -24,9 +24,9 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
/**
- * Summary of failures from all servers.
+ * Summary of failures from all servers. UI model, so it contains public fields.
*/
-@SuppressWarnings("WeakerAccess") public class TestFailuresSummary extends UpdateInfo {
+@SuppressWarnings({"WeakerAccess", "PublicField"}) public class TestFailuresSummary extends UpdateInfo {
/** Servers (Services) and their chain results. */
public List<ChainAtServerCurrentStatus> servers = new ArrayList<>();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java
index 240e704..9830f41 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java
@@ -21,9 +21,9 @@ import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited;
import org.apache.ignite.ci.jira.ignited.IJiraIgnited;
/**
- *
+ * General update information for JS data updating requests. UI model, so it contains public fields.
*/
-@SuppressWarnings("WeakerAccess") public class UpdateInfo {
+@SuppressWarnings({"WeakerAccess", "PublicField"}) public class UpdateInfo {
/** TeamCity auth token availability flag. */
public static final int TEAMCITY_FLAG = 1;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java
index 91b6f98..43c2b5c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java
@@ -89,7 +89,7 @@ public class GetChainResultsAsHtml {
status.chainName = ctx.suiteName();
- status.initFromContext(tcIgn, ctx, failRateBranch, injector.getInstance(IStringCompactor.class));
+ status.initFromContext(tcIgn, ctx, failRateBranch, injector.getInstance(IStringCompactor.class), false);
res.append(showChainAtServerData(status));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
index 4ec3894..6525ba1 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
@@ -139,7 +139,7 @@ public class GetBuildTestFailures {
if (cnt > 0)
runningUpdates.addAndGet(cnt);
- chainStatus.initFromContext(tcIgnited, ctx, failRateBranch, injector.getInstance(IStringCompactor.class));
+ chainStatus.initFromContext(tcIgnited, ctx, failRateBranch, injector.getInstance(IStringCompactor.class), false);
res.addChainOnServer(chainStatus);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
index c631a7f..a1961f3 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
@@ -30,15 +30,15 @@ import javax.ws.rs.core.MediaType;
import org.apache.ignite.ci.tcbot.chain.TrackedBranchChainsProcessor;
import org.apache.ignite.ci.tcbot.conf.ITcBotConfig;
import org.apache.ignite.ci.tcbot.visa.TcBotTriggerAndSignOffService;
-import org.apache.ignite.tcservice.model.mute.MuteInfo;
-import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
-import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.ci.web.CtxListener;
import org.apache.ignite.ci.web.model.current.TestFailuresSummary;
import org.apache.ignite.ci.web.model.current.UpdateInfo;
import org.apache.ignite.ci.web.rest.parms.FullQueryParams;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
+import org.apache.ignite.tcignited.SyncMode;
+import org.apache.ignite.tcservice.model.mute.MuteInfo;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -60,45 +60,51 @@ public class GetTrackedBranchTestResults {
@GET
@Path("updates")
public UpdateInfo getTestFailsUpdates(@Nullable @QueryParam("branch") String branchOrNull,
- @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs) {
- return new UpdateInfo().copyFrom(getTestFailsResultsNoSync(branchOrNull, checkAllLogs));
+ @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs,
+ @QueryParam("trustedTests") @Nullable Boolean trustedTests) {
+ return new UpdateInfo().copyFrom(getTestFailsResultsNoSync(branchOrNull, checkAllLogs, trustedTests));
}
@GET
@Path("results/txt")
@Produces(MediaType.TEXT_PLAIN)
public String getTestFailsText(@Nullable @QueryParam("branch") String branchOrNull,
- @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs) {
- return getTestFailsResultsNoSync(branchOrNull, checkAllLogs).toString();
+ @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs,
+ @QueryParam("trustedTests") @Nullable Boolean trustedTests) {
+ return getTestFailsResultsNoSync(branchOrNull, checkAllLogs, trustedTests).toString();
}
@GET
@Path("resultsNoSync")
public TestFailuresSummary getTestFailsResultsNoSync(
- @Nullable @QueryParam("branch") String branch,
- @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs) {
- return latestBuildResults(branch, checkAllLogs, SyncMode.NONE);
+ @Nullable @QueryParam("branch") String branch,
+ @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs,
+ @QueryParam("trustedTests") @Nullable Boolean trustedTests) {
+ return latestBuildResults(branch, checkAllLogs, trustedTests, SyncMode.NONE);
}
@GET
@Path("results")
@NotNull
public TestFailuresSummary getTestFailsNoCache(
- @Nullable @QueryParam("branch") String branch,
- @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs) {
- return latestBuildResults(branch, checkAllLogs, SyncMode.RELOAD_QUEUED);
+ @Nullable @QueryParam("branch") String branch,
+ @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs,
+ @QueryParam("trustedTests") @Nullable Boolean trustedTests) {
+ return latestBuildResults(branch, checkAllLogs, trustedTests, SyncMode.RELOAD_QUEUED);
}
@NotNull public TestFailuresSummary latestBuildResults(
@QueryParam("branch") @Nullable String branch,
@QueryParam("checkAllLogs") @Nullable Boolean checkAllLogs,
+ @QueryParam("trustedTests") @Nullable Boolean trustedTests,
SyncMode mode) {
ITcBotUserCreds creds = ITcBotUserCreds.get(req);
Injector injector = CtxListener.getInjector(ctx);
return injector.getInstance(TrackedBranchChainsProcessor.class)
- .getTrackedBranchTestFailures(branch, checkAllLogs, 1, creds, mode);
+ .getTrackedBranchTestFailures(branch, checkAllLogs, 1, creds, mode,
+ Boolean.TRUE.equals(trustedTests));
}
@GET
@@ -136,7 +142,7 @@ public class GetTrackedBranchTestResults {
Injector injector = CtxListener.getInjector(ctx);
return injector.getInstance(TrackedBranchChainsProcessor.class)
- .getTrackedBranchTestFailures(branchOpt, checkAllLogs, cntLimit, creds, mode);
+ .getTrackedBranchTestFailures(branchOpt, checkAllLogs, cntLimit, creds, mode, false);
}
/**
diff --git a/ignite-tc-helper-web/src/main/webapp/current.html b/ignite-tc-helper-web/src/main/webapp/current.html
index 69b14de..882a8b0 100644
--- a/ignite-tc-helper-web/src/main/webapp/current.html
+++ b/ignite-tc-helper-web/src/main/webapp/current.html
@@ -37,6 +37,12 @@ function parmsForRest() {
if (checkAllLogs != null) {
curReqParms += "&checkAllLogs=" + checkAllLogs;
}
+
+ var trustedTests = findGetParameter("trustedTests");
+ if (trustedTests != null) {
+ curReqParms += "&trustedTests=" + trustedTests;
+ }
+
return curReqParms;
}
diff --git a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js
index 8c87e3d..ca60cfc 100644
--- a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js
+++ b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js
@@ -159,7 +159,7 @@ function showChainCurrentStatusData(chain, settings) {
res += "Long running tests report";
res += "</a>";
- var mInfo = "";
+ var moreInfoTxt = "";
var cntFailed = 0;
var suitesFailedList = "";
@@ -179,19 +179,19 @@ function showChainCurrentStatusData(chain, settings) {
}
if (suitesFailedList.length !== 0 && isDefinedAndFilled(chain.tcServerCode) && isDefinedAndFilled(chain.branchName)) {
- mInfo += "Trigger failed " + cntFailed + " builds";
- mInfo += " <a href='javascript:void(0);' ";
- mInfo += " onClick='triggerBuilds(\"" + chain.tcServerCode + "\", \"" + parentSuitId + "\", " +
+ moreInfoTxt += "Trigger failed " + cntFailed + " builds";
+ moreInfoTxt += " <a href='javascript:void(0);' ";
+ moreInfoTxt += " onClick='triggerBuilds(\"" + chain.tcServerCode + "\", \"" + parentSuitId + "\", " +
"\"" + suitesFailedList + "\", \"" + chain.branchName + "\", false, false, null, \"" + chain.prNum + "\")' ";
- mInfo += " title='trigger builds'>in queue</a> ";
+ moreInfoTxt += " title='trigger builds'>in queue</a> ";
- mInfo += " <a href='javascript:void(0);' ";
- mInfo += " onClick='triggerBuilds(\"" + chain.tcServerCode + "\", \"" + parentSuitId + "\", " +
+ moreInfoTxt += " <a href='javascript:void(0);' ";
+ moreInfoTxt += " onClick='triggerBuilds(\"" + chain.tcServerCode + "\", \"" + parentSuitId + "\", " +
"\"" + suitesFailedList + "\", \"" + chain.branchName + "\", true, false, null, \"" + chain.prNum + "\")' ";
- mInfo += " title='trigger builds'>on top</a><br>";
+ moreInfoTxt += " title='trigger builds'>on top</a><br>";
}
- mInfo += "Duration: " + chain.durationPrintable + " " +
+ moreInfoTxt += "Duration: " + chain.durationPrintable + " " +
"(Net Time: " + chain.durationNetTimePrintable + "," +
" Tests: " + chain.testsDurationPrintable + "," +
" Src. Update: " + chain.sourceUpdateDurationPrintable + "," +
@@ -199,31 +199,39 @@ function showChainCurrentStatusData(chain, settings) {
" Dependecies Resolving: " + chain.dependeciesResolvingDurationPrintable + "," +
" Timeouts: " + chain.lostInTimeouts + ")<br>";
+ if(isDefinedAndFilled(chain.totalTests))
+ moreInfoTxt += " <span title='Not muted and not ignored tests'>Total tests: " + chain.totalTests + "</span>";
+
+ if(isDefinedAndFilled(chain.trustedTests))
+ moreInfoTxt += " <span title='Tests which not filtered out because of flakyness'>Trusted tests: " + chain.trustedTests + "</span>";
+
+ moreInfoTxt += "<br>";
+
if (isDefinedAndFilled(chain.topLongRunning) && chain.topLongRunning.length > 0) {
- mInfo += "Top long running:<br>";
+ moreInfoTxt += "Top long running:<br>";
- mInfo += "<table>";
+ moreInfoTxt += "<table>";
for (var j = 0; j < chain.topLongRunning.length; j++) {
- mInfo += showTestFailData(chain.topLongRunning[j], false, settings);
+ moreInfoTxt += showTestFailData(chain.topLongRunning[j], false, settings);
}
- mInfo += "</table>";
+ moreInfoTxt += "</table>";
}
if (isDefinedAndFilled(chain.logConsumers) && chain.logConsumers.length > 0) {
- mInfo += "Top Log Consumers:<br>";
+ moreInfoTxt += "Top Log Consumers:<br>";
- mInfo += "<table>";
+ moreInfoTxt += "<table>";
for (var k = 0; k < chain.logConsumers.length; k++) {
- mInfo += showTestFailData(chain.logConsumers[k], false, settings);
+ moreInfoTxt += showTestFailData(chain.logConsumers[k], false, settings);
}
- mInfo += "</table>";
+ moreInfoTxt += "</table>";
}
if(!isDefinedAndFilled(findGetParameter("reportMode"))) {
res += "<span class='container'>";
res += " <a href='javascript:void(0);' class='header'>" + more + "</a>";
- res += "<div class='content'>" + mInfo + "</div></span>";
+ res += "<div class='content'>" + moreInfoTxt + "</div></span>";
}
res += "</td><td>";
@@ -571,6 +579,15 @@ function showSuiteData(suite, settings, prNum) {
" Dependecies Resolving: " + suite.dependeciesResolvingDurationPrintable + "," +
" Timeouts: " + suite.lostInTimeouts + ")<br>";
+
+ if(isDefinedAndFilled(suite.totalTests))
+ moreInfoTxt += " <span title='Not muted and not ignored tests'>Total tests: " + suite.totalTests + "</span>";
+
+ if(isDefinedAndFilled(suite.trustedTests))
+ moreInfoTxt += " <span title='Tests which not filtered out because of flakyness'>Trusted tests: " + suite.trustedTests + "</span>";
+
+ moreInfoTxt += "<br>";
+
var res = "";
res += "<tr bgcolor='#FAFAFF'><td align='right' valign='top'>";
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java
index 578e211..40ad6b4 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java
@@ -117,8 +117,8 @@ public class TrackedBranchProcessorTest {
TestFailuresSummary failures = tbProc.getTrackedBranchTestFailures(BRACH_NAME,
false,
1,
- mock, SyncMode.RELOAD_QUEUED
- );
+ mock, SyncMode.RELOAD_QUEUED,
+ false);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
System.out.println(gson.toJson(failures));
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/BuildKeyUnitTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/BuildKeyUnitTest.java
index f11b6e5..67a369f 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/BuildKeyUnitTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/BuildKeyUnitTest.java
@@ -18,7 +18,7 @@
package org.apache.ignite.ci.teamcity.ignited;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.junit.Test;
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
index f5f3bca..4f9eacd 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
@@ -31,6 +31,7 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.SyncMode;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.jetbrains.annotations.NotNull;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
@@ -132,6 +133,7 @@ public class TeamcityIgnitedMock {
Map<Integer, FatBuildCompacted> builds, int srvId) {
Map<RunHistKey, RunHistCompacted> histCache = new ConcurrentHashMap<>();
+ int successStatusStrId = c.getStringId(TestOccurrence.STATUS_SUCCESS);
for (FatBuildCompacted build : builds.values()) {
if (!build.isFinished(c))
continue;
@@ -141,7 +143,7 @@ public class TeamcityIgnitedMock {
final RunHistCompacted hist = histCache.computeIfAbsent(histKey, RunHistCompacted::new);
- Invocation inv = testCompacted.toInvocation(c, build, (k, v) -> true);
+ Invocation inv = testCompacted.toInvocation(build, (k, v) -> true, successStatusStrId);
hist.addInvocation(inv);
});
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java
index 6f4feab..6bd753f 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java
@@ -34,8 +34,8 @@ import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.configuration.IgniteConfiguration;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeCompacted.java
index 0c114ba..45aafd8 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeCompacted.java
@@ -27,7 +27,7 @@ import org.apache.ignite.tcbot.persistence.IVersionedEntity;
import org.apache.ignite.tcbot.persistence.Persisted;
import org.apache.ignite.tcservice.model.changes.Change;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.tcignited.build.FatBuildDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/RevisionCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/RevisionCompacted.java
index 6b3e8e4..b918605 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/RevisionCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/RevisionCompacted.java
@@ -26,7 +26,7 @@ import org.apache.ignite.tcbot.persistence.Persisted;
import org.apache.ignite.tcservice.model.vcs.Revision;
import org.apache.ignite.tcservice.model.vcs.VcsRootInstance;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.tcignited.build.FatBuildDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index e8a9d27..eaeeb54 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections;
import java.util.Date;
+import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.TreeMap;
@@ -30,7 +31,14 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
+import org.apache.ignite.ci.teamcity.ignited.change.RevisionCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcbot.persistence.IVersionedEntity;
import org.apache.ignite.tcbot.persistence.Persisted;
import org.apache.ignite.tcservice.ITeamcity;
@@ -49,15 +57,6 @@ import org.apache.ignite.tcservice.model.user.User;
import org.apache.ignite.tcservice.model.vcs.Revision;
import org.apache.ignite.tcservice.model.vcs.Revisions;
import org.apache.ignite.tcservice.model.vcs.VcsRootInstance;
-import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
-import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
-import org.apache.ignite.ci.teamcity.ignited.change.RevisionCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
-import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
/**
* Composed data from {@link Build} and other classes, compressed for storage.
@@ -687,4 +686,29 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
@Nullable public ParametersCompacted parameters() {
return buildParameters;
}
+
+ public boolean hasBuildProblemType(int id) {
+ if (problems == null)
+ return false;
+
+ for (ProblemCompacted next : problems) {
+ if (next.type() == id)
+ return true;
+ }
+
+ return false;
+ }
+
+ public int totalNotMutedTests() {
+ if(tests == null)
+ return 0;
+
+ int cnt = 0;
+ for (TestCompacted next : tests) {
+ if (!next.isMutedTest() && !next.isIgnoredTest())
+ cnt++;
+ }
+
+ return cnt;
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index 053d4db..92a85df 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -406,15 +406,14 @@ public class TestCompacted {
}
/**
- * @param compactor
* @param build
* @param paramsFilter parameters filter to find out parameters to be saved in RunHistory (for future filtering).
+ * @param successStatusStrId
* @return
*/
- public Invocation toInvocation(IStringCompactor compactor,
- FatBuildCompacted build,
- BiPredicate<Integer, Integer> paramsFilter) {
- final boolean failedTest = isFailedTest(compactor);
+ public Invocation toInvocation(FatBuildCompacted build,
+ BiPredicate<Integer, Integer> paramsFilter, int successStatusStrId) {
+ final boolean failedTest = successStatusStrId != status;
final int failCode = failedTest
? (isIgnoredTest() || isMutedTest())
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
index 75f5abc..4a0cd2e 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
@@ -22,12 +22,11 @@ import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
-
+import org.apache.ignite.tcbot.persistence.IVersionedEntity;
+import org.apache.ignite.tcbot.persistence.Persisted;
import org.apache.ignite.tcignited.history.ChangesState;
import org.apache.ignite.tcignited.history.IEventTemplate;
import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcbot.persistence.IVersionedEntity;
-import org.apache.ignite.tcbot.persistence.Persisted;
import org.apache.ignite.tcignited.history.RunStatus;
/**
@@ -37,6 +36,7 @@ import org.apache.ignite.tcignited.history.RunStatus;
public class RunHistCompacted implements IVersionedEntity, IRunHistory {
/** Latest version. */
private static final int LATEST_VERSION = 1;
+ public static final int FLAKYNESS_STATUS_CHANGE_BORDER = 1;
/** Entity fields version. */
@SuppressWarnings("FieldCanBeLocal")
@@ -90,6 +90,16 @@ public class RunHistCompacted implements IVersionedEntity, IRunHistory {
/** {@inheritDoc} */
@Override public String getFlakyComments() {
+ int statusChange = getStatusChangesWithoutCodeModification();
+
+ if (statusChange < FLAKYNESS_STATUS_CHANGE_BORDER)
+ return null;
+
+ return "Test seems to be flaky: " +
+ "changed its status [" + statusChange + "/" + data.invocations().count() + "] without code modifications";
+ }
+
+ public int getStatusChangesWithoutCodeModification() {
int statusChange = 0;
Invocation prev = null;
@@ -105,12 +115,12 @@ public class RunHistCompacted implements IVersionedEntity, IRunHistory {
}
prev = cur;
}
+ return statusChange;
+ }
- if (statusChange < 1)
- return null;
-
- return "Test seems to be flaky: " +
- "changed its status [" + statusChange + "/" + latestRuns.size() + "] without code modifications";
+ /** {@inheritDoc} */
+ @Override public boolean isFlaky() {
+ return getStatusChangesWithoutCodeModification() >= FLAKYNESS_STATUS_CHANGE_BORDER;
}
/** {@inheritDoc} */
@@ -215,4 +225,11 @@ public class RunHistCompacted implements IVersionedEntity, IRunHistory {
@Override public int hashCode() {
return Objects.hash(_ver, data);
}
+
+ /**
+ * @param v Invocation.
+ */
+ public void innerAddInvocation(Invocation v) {
+ data.innerAdd(v);
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistKey.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistKey.java
index ddb910f..dd199f5 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistKey.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistKey.java
@@ -39,12 +39,12 @@ public class RunHistKey {
/**
* @param srvId Server id.
- * @param testName Test or suite name.
+ * @param testOrSuiteName Test or suite name.
* @param branchName Branch name.
*/
- public RunHistKey(int srvId, int testName, int branchName) {
+ public RunHistKey(int srvId, int testOrSuiteName, int branchName) {
this.srvId = srvId;
- this.testOrSuiteName = testName;
+ this.testOrSuiteName = testOrSuiteName;
this.branch = branchName;
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 548a610..11a4671 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -204,6 +204,16 @@ public interface ITeamcityIgnited {
@Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch);
+
+ /**
+ * V.3.0 run history implementation based on scan of fat builds.
+ *
+ * @param testName Test name.
+ * @param suiteName Suite name.
+ * @param branchName Branch name.
+ */
+ @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName);
+
/**
* @param suiteBuildTypeId Suite id.
* @return run statistics of recent runls on all branches.
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 9ee9e6d..b35db62 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -18,31 +18,58 @@ package org.apache.ignite.tcignited;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.OptionalInt;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildCondition;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
-import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
-import org.apache.ignite.tcignited.buildref.BuildRefDao;
-import org.apache.ignite.tcignited.buildref.BuildRefSync;
-import org.apache.ignite.ci.teamcity.ignited.buildtype.*;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeCompacted;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeDao;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefDao;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeSync;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.ignited.mute.MuteDao;
import org.apache.ignite.ci.teamcity.ignited.mute.MuteSync;
-import org.apache.ignite.tcignited.history.RunHistCompactedDao;
-import org.apache.ignite.tcignited.history.RunHistSync;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.buildref.BuildRefSync;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcignited.history.RunHistCompactedDao;
+import org.apache.ignite.tcignited.history.RunHistSync;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.model.agent.Agent;
@@ -53,16 +80,6 @@ import org.apache.ignite.tcservice.model.result.Build;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import java.io.File;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-
import static org.apache.ignite.tcservice.model.hist.BuildRef.STATUS_UNKNOWN;
/**
@@ -348,9 +365,21 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (stateQueuedId == null)
return Collections.emptyList();
- List<BuildRefCompacted> builds = buildRefDao.getBuildsForBranch(srvIdMaskHigh, branchForQuery(branchName));
+ Set<Integer> branchNameIds = branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str))
+ .filter(Objects::nonNull).collect(Collectors.toSet());
+
+ List<BuildRefCompacted> res = new ArrayList<>();
+
+ branchNameIds.forEach(br -> {
+ List<BuildRefCompacted> builds = buildRefDao.getBuildsForBranch(srvIdMaskHigh, br);
- return builds.stream().filter(b -> b.state() == stateQueuedId).collect(Collectors.toList());
+ if(builds.isEmpty())
+ return;
+
+ builds.stream().filter(buildRef -> buildRef.state() == stateQueuedId).forEach(res::add);
+ });
+
+ return res;
}
@@ -378,7 +407,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
.collect(Collectors.toList());
if (chains.isEmpty()) {
- // probably there are no not-cacelled builds at all
+ // probably there are no not-cacelled builds at all, check for cancelled
chains = hist.stream()
.filter(ref -> !ref.isFakeStub())
.filter(t -> t.isFinished(compactor))
@@ -405,6 +434,34 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return runHistCompactedDao.getSuiteRunHist(srvIdMaskHigh, suiteId, branch);
}
+ @Nullable @Override
+ public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName) {
+ if (suiteName == null || branchName == null)
+ return null;
+
+ Supplier<Set<Integer>> supplier = () -> {
+ String btId = compactor.getStringFromId(suiteName);
+ String branchId = compactor.getStringFromId(branchName);
+ List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId);
+ long curTs = System.currentTimeMillis();
+ Set<Integer> buildIds = compacted.stream().filter(
+ bRef -> {
+ Long startTime = getBuildStartTime(bRef.id());
+ if (startTime == null)
+ return false;
+
+ return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
+ }
+ ).map(BuildRefCompacted::id).collect(Collectors.toSet());
+
+ System.err.println("Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+
+ return buildIds;
+ };
+
+ return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, suiteName, branchName);
+ }
+
/** {@inheritDoc} */
@Nullable @Override public IRunStat getSuiteRunStatAllBranches(String suiteBuildTypeId) {
return runHistCompactedDao.getSuiteRunStatAllBranches(srvIdMaskHigh, suiteBuildTypeId);
@@ -511,16 +568,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return buildStartTime != null ? new Date(buildStartTime) : null;
}
- @GuavaCached(maximumSize = 2000, cacheNullRval = false)
+ @GuavaCached(maximumSize = 5000, cacheNullRval = false)
@AutoProfiling
public Long getBuildStartTs(int buildId) {
- Long buildStartTime = runHistCompactedDao.getBuildStartTime(srvIdMaskHigh, buildId);
+ Long buildStartTime = getBuildStartTime(buildId);
if (buildStartTime != null)
return buildStartTime;
- String msg = "Loading build [" + buildId + "] start date";
-
- logger.info(msg);
+ if(logger.isDebugEnabled()) {
+ String msg = "Loading build [" + buildId + "] start date";
+ logger.debug(msg);
+ }
FatBuildCompacted highBuild = getFatBuild(buildId, SyncMode.LOAD_NEW);
if (highBuild == null || highBuild.isFakeStub())
@@ -531,6 +589,11 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return ts > 0 ? ts : null;
}
+ @GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true)
+ public Long getBuildStartTime(int buildId) {
+ return runHistCompactedDao.getBuildStartTime(srvIdMaskHigh, buildId);
+ }
+
/** {@inheritDoc} */
@GuavaCached(maximumSize = 500, expireAfterAccessSecs = 30, softValues = true)
@Override public FatBuildCompacted getFatBuild(int buildId, SyncMode mode) {
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
index a9b5ed3..66fbf78 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
@@ -27,8 +27,8 @@ import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefDao;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeSync;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.ignited.mute.MuteDao;
import org.apache.ignite.ci.teamcity.ignited.mute.MuteSync;
import org.apache.ignite.tcignited.buildlog.BuildLogProcessorModule;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
similarity index 56%
rename from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
rename to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index dd341e8..ad08fe3 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -15,32 +15,50 @@
* limitations under the License.
*/
-package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.cache.Cache;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
+import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcservice.model.changes.ChangesList;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull;
-import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,10 +78,26 @@ public class FatBuildDao {
/** Builds cache. */
private IgniteCache<Long, FatBuildCompacted> buildsCache;
+
+ /** Suite history cache. */
+ private IgniteCache<RunHistKey, SuiteHistory> suiteHistory;
+
/** Compactor. */
@Inject private IStringCompactor compactor;
/**
+ * Non persistence cache for all suite RunHistory for particular branch.
+ * RunHistKey(ServerId||BranchId||suiteId)-> Build reference
+ */
+ private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache
+ = CacheBuilder.newBuilder()
+ .maximumSize(8000)
+ .expireAfterAccess(16, TimeUnit.MINUTES)
+ .softValues()
+ .build();
+
+
+ /**
*
*/
public FatBuildDao init() {
@@ -120,6 +154,8 @@ public class FatBuildDao {
@AutoProfiling
public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
+
+ invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
}
public static int[] extractChangeIds(@Nonnull ChangesList changesList) {
@@ -186,4 +222,109 @@ public class FatBuildDao {
.filter(entry -> entry.getValue().isOutdatedEntityVersion())
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
+
+ public IRunHistory getTestRunHist(int srvIdMaskHigh,
+ Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) {
+
+
+ RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, suiteName, branchName);
+
+ SuiteHistory history;
+
+ try {
+ history = runHistInMemCache.get(runHistKey,
+ () -> {
+ Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
+
+ return calcSuiteHistory(srvIdMaskHigh, buildIds);
+ });
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+
+ return history.testsHistory.get(testName);
+ }
+
+ @AutoProfiling
+ protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
+ Set<Long> cacheKeys = buildIds.stream().map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
+
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new HistoryCollectProcessor(successStatusStrId);
+
+ Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
+
+ SuiteHistory hist = new SuiteHistory();
+
+ map.values().forEach(
+ res-> {
+ if(res==null)
+ return;
+
+ Map<Integer, Invocation> invocationMap = res.get();
+
+ if(invocationMap == null)
+ return;
+
+ invocationMap.forEach((k, v) -> {
+ RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
+ k_ -> new RunHistCompacted());
+
+ compacted.innerAddInvocation(v);
+ });
+
+ }
+ );
+
+ System.err.println("Suite history: tests in scope "
+ + hist.testsHistory.size()
+ + " for " +buildIds.size() + " builds checked"
+ + " size " + hist.size(igniteProvider.get()));
+
+ return hist;
+ }
+
+ @AutoProfiling
+ protected Set<Integer> determineLatestBuilds(Supplier<Set<Integer>> buildIdsSupplier) {
+ return buildIdsSupplier.get();
+ }
+
+ public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
+ Iterable<RunHistKey> objects =
+ stream
+ .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
+ .collect(Collectors.toSet());
+
+ runHistInMemCache.invalidateAll(objects);
+ }
+
+
+ private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
+ private final int successStatusStrId;
+
+ public HistoryCollectProcessor(int successStatusStrId) {
+ this.successStatusStrId = successStatusStrId;
+ }
+
+ @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
+ Object... arguments) throws EntryProcessorException {
+ if (entry.getValue() == null)
+ return null;
+
+ Map<Integer, Invocation> hist = new HashMap<>();
+ FatBuildCompacted fatBuildCompacted = entry.getValue();
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+
+ hist.put(testCompacted.testName(), invocation);
+ }
+ );
+
+ return hist;
+ }
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
similarity index 99%
rename from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
rename to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
index e9bfe7c..3e01c05 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
@@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+package org.apache.ignite.tcignited.build;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import java.util.stream.Stream;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
@@ -94,7 +95,7 @@ public class ProactiveFatBuildSync {
/**
* Invoke load fat builds later, re-load provided builds.
- * @param conn
+ * @param conn TC server connection.
* @param buildsToAskFromTc Builds to ask from tc.
*/
public void scheduleBuildsLoad(ITeamcityConn conn, Collection<Integer> buildsToAskFromTc) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
similarity index 59%
copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index 095301c..8969736 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -15,24 +15,21 @@
* limitations under the License.
*/
-package org.apache.ignite.ci.analysis;
+package org.apache.ignite.tcignited.build;
-import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
-import org.apache.ignite.tcignited.history.IRunHistory;
+import java.util.HashMap;
+import java.util.Map;
-/**
- * Multiple test occurrence. For single build context - max 1 failure
- */
-public interface IMultTestOccurrence {
- public String getName();
-
- public boolean isInvestigated();
-
- public int failuresCount();
-
- public long getAvgDurationMs();
+import org.apache.ignite.Ignite;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
+import org.apache.ignite.internal.binary.BinaryObjectExImpl;
- Iterable<TestOccurrenceFull> getOccurrences();
+public class SuiteHistory {
+ /** Tests history: Test name ID->RunHistory */
+ Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
- String getPossibleBlockerComment(IRunHistory baseBranchStat);
+ public int size(Ignite ignite) {
+ BinaryObjectExImpl binary = ignite.binary().toBinary(this);
+ return binary.length();
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index a4c73d4..f3a3e89 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -17,12 +17,20 @@
package org.apache.ignite.tcignited.buildref;
-import java.util.*;
+import com.google.common.cache.CacheBuilder;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
import javax.cache.Cache;
import javax.inject.Inject;
import javax.inject.Provider;
@@ -31,14 +39,16 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.QueryEntity;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.GridIntList;
+import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
-import org.apache.ignite.tcservice.model.hist.BuildRef;
-import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.util.GridIntList;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
/**
*
@@ -50,12 +60,23 @@ public class BuildRefDao {
/** Ignite provider. */
@Inject private Provider<Ignite> igniteProvider;
- /** Builds cache. */
+ /** Builds (Refs) cache: Long(ServerId||BuildId)-> Build reference */
private IgniteCache<Long, BuildRefCompacted> buildRefsCache;
/** Compactor. */
@Inject private IStringCompactor compactor;
+ /** Non persistence cache for all BuildRefsCompacted for particular branch.
+ * RunHistKey(ServerId||BranchId||suiteId)-> Build reference
+ */
+ private final com.google.common.cache.Cache<RunHistKey, List<BuildRefCompacted>> buildRefsInMemCache
+ = CacheBuilder.newBuilder()
+ .maximumSize(8000)
+ .expireAfterAccess(16, TimeUnit.MINUTES)
+ .softValues()
+ .build();
+
+
/** */
public BuildRefDao init() {
CacheConfiguration<Long, BuildRefCompacted> cfg = CacheConfigs.getCacheV2Config(TEAMCITY_BUILD_CACHE_NAME);
@@ -113,12 +134,24 @@ public class BuildRefDao {
}
int size = entriesToPut.size();
- if (size != 0)
+ if (size != 0) {
buildRefsCache.putAll(entriesToPut);
+ invalidateHistoryInMem(srvId, entriesToPut.values().stream());
+ }
+
return entriesToPut.keySet();
}
+ public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
+ Iterable<RunHistKey> objects =
+ stream
+ .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
+ .collect(Collectors.toSet());
+
+ buildRefsInMemCache.invalidateAll(objects);
+ }
+
/**
* @param srvId Server id mask high.
* @param buildId Build id.
@@ -137,24 +170,53 @@ public class BuildRefDao {
/**
* @param srvId Server id mask high.
- * @param buildTypeId Build type id.
+ * @param buildTypeId Build type (suite) id.
* @param bracnhNameQry Bracnh name query.
*/
@AutoProfiling
@Nonnull public List<BuildRefCompacted> getAllBuildsCompacted(int srvId,
- @Nullable String buildTypeId,
- List<String> bracnhNameQry) {
-
+ String buildTypeId,
+ List<String> bracnhNameQry) {
Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId);
if (buildTypeIdId == null)
return Collections.emptyList();
- if (bracnhNameQry.stream().map(str -> compactor.getStringIdIfPresent(str)).allMatch(Objects::isNull))
+ Set<Integer> branchNameIds = bracnhNameQry.stream().map(str -> compactor.getStringIdIfPresent(str))
+ .filter(Objects::nonNull).collect(Collectors.toSet());
+
+ if (branchNameIds.isEmpty())
return Collections.emptyList();
- return getBuildsForBranch(srvId, bracnhNameQry).stream()
- .filter(e -> e.buildTypeId() == buildTypeIdId)
- .collect(Collectors.toList());
+ List<BuildRefCompacted> res = new ArrayList<>();
+
+ branchNameIds.forEach(branchNameId -> {
+ RunHistKey runHistKey = new RunHistKey(srvId, buildTypeIdId, branchNameId);
+ try {
+ List<BuildRefCompacted> compactedBuildsForBranch =
+ buildRefsInMemCache.get(runHistKey, () -> {
+ List<BuildRefCompacted> branch = getBuildsForBranch(srvId, branchNameId);
+
+ List<BuildRefCompacted> resForBranch = branch.stream()
+ .filter(e -> e.buildTypeId() == buildTypeIdId)
+ .collect(Collectors.toList());
+
+ if (!resForBranch.isEmpty()) {
+ System.err.println("Branch " + compactor.getStringFromId(branchNameId)
+ + " Suite " + buildTypeId
+ + " builds " + resForBranch.size() + " ");
+ }
+
+ return resForBranch;
+ });
+
+ res.addAll(compactedBuildsForBranch);
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+ });
+
+ return res;
}
/**
@@ -178,27 +240,19 @@ public class BuildRefDao {
}
/**
+ * Collects all builds for branch. Short-term cached because builds from same branch may be queued several times.
+ *
* @param srvId Server id.
- * @param branchNamesQry Branch name(s).
+ * @param branchNameId Branch name - IDs from compactor.
*/
@AutoProfiling
@GuavaCached(softValues = true, maximumSize = 10000, expireAfterWriteSecs = 90)
- public List<BuildRefCompacted> getBuildsForBranch(int srvId, List<String> branchNamesQry) {
- Set<Integer> branchIds = branchNamesQry.stream().map(str -> compactor.getStringIdIfPresent(str))
- .filter(Objects::nonNull).collect(Collectors.toSet());
-
+ public List<BuildRefCompacted> getBuildsForBranch(int srvId, int branchNameId) {
List<BuildRefCompacted> list = new ArrayList<>();
- for (Integer next : branchIds)
- fillBuilds(srvId, next, list);
-
- return list;
- }
-
- private void fillBuilds(int srvId, Integer branchNameId, List<BuildRefCompacted> list) {
- try (QueryCursor<Cache.Entry<Long, BuildRefCompacted>> qryCursor = buildRefsCache.query(
+ try (QueryCursor<Cache.Entry<Long, BuildRefCompacted>> qryCursor = buildRefsCache.query(
new SqlQuery<Long, BuildRefCompacted>(BuildRefCompacted.class, "branchName = ?")
- .setArgs(branchNameId))) {
+ .setArgs((Integer)branchNameId))) {
for (Cache.Entry<Long, BuildRefCompacted> next : qryCursor) {
Long key = next.getKey();
@@ -209,6 +263,13 @@ public class BuildRefDao {
list.add(next.getValue());
}
}
+
+ if (!list.isEmpty()) {
+ System.err.println(" Branch " + compactor.getStringFromId(branchNameId)
+ + " builds " + list.size() + " (Overall) ");
+ }
+
+ return list;
}
/**
@@ -222,6 +283,7 @@ public class BuildRefDao {
if (buildPersisted == null || !buildPersisted.equals(refCompacted)) {
buildRefsCache.put(cacheKey, refCompacted);
+ invalidateHistoryInMem(srvId, Stream.of(refCompacted));
return true;
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java
index 05c3d2d..37ab754 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java
@@ -20,7 +20,7 @@ import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcignited.ITeamcityIgnited;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.tcservice.ITeamcityConn;
import javax.annotation.Nonnull;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistSummary.java
similarity index 60%
copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistSummary.java
index 095301c..1d33f73 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistSummary.java
@@ -14,25 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.ignite.ci.analysis;
-
-import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
-import org.apache.ignite.tcignited.history.IRunHistory;
+package org.apache.ignite.tcignited.history;
/**
- * Multiple test occurrence. For single build context - max 1 failure
+ *
*/
-public interface IMultTestOccurrence {
- public String getName();
-
- public boolean isInvestigated();
-
- public int failuresCount();
-
- public long getAvgDurationMs();
-
- Iterable<TestOccurrenceFull> getOccurrences();
-
- String getPossibleBlockerComment(IRunHistory baseBranchStat);
+public interface IRunHistSummary {
+ /**
+ *
+ */
+ public float getFailRate();
+
+ /**
+ *
+ */
+ public boolean isFlaky();
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java
index 6bd846a..cb7dbd5 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java
@@ -22,7 +22,7 @@ import java.util.List;
/**
* Test or Build run statistics.
*/
-public interface IRunHistory extends IRunStat {
+public interface IRunHistory extends IRunStat, IRunHistSummary {
public int getFailuresAllHist();
public int getRunsAllHist();
@@ -64,4 +64,13 @@ public interface IRunHistory extends IRunStat {
public int getCriticalFailuresCount();
+
+ @Override default float getFailRate() {
+ int runs = getRunsCount();
+
+ if (runs == 0)
+ return 0.0f;
+
+ return 1.0f * getFailuresCount() / runs;
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunStat.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunStat.java
index c1e4123..7596400 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunStat.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunStat.java
@@ -24,7 +24,8 @@ public interface IRunStat {
public int getFailuresCount();
/**
- * @return fail rate as float.
+ * Recent runs fail rate.
+ * @return fail rate as float: 0.0...1.0
*/
public default float getFailRate() {
int runs = getRunsCount();
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index 92d0544..8fed725 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -35,11 +35,11 @@ import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
-import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
-import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
@@ -93,7 +93,7 @@ public class RunHistCompactedDao {
buildStartTime = ignite.getOrCreateCache(CacheConfigs.getCacheV2Config(BUILD_START_TIME_CACHE_NAME));
}
- @GuavaCached(maximumSize = 200, expireAfterAccessSecs = 30, softValues = true)
+ @GuavaCached(maximumSize = 50000, expireAfterWriteSecs = 120, softValues = true)
public IRunHistory getTestRunHist(int srvIdMaskHigh, String name, @Nullable String branch) {
RunHistKey key = getKey(srvIdMaskHigh, name, branch);
if (key == null)
@@ -124,9 +124,14 @@ public class RunHistCompactedDao {
@AutoProfiling
public boolean buildWasProcessed(int srvId, int buildId) {
- return buildStartTime.containsKey(buildIdToCacheKey(srvId, buildId));
+ return getBuildStartTime(srvId, buildId) != null;
}
+ /**
+ * @param srvId Server id.
+ * @param buildId Build id.
+ */
+ @AutoProfiling
@Nullable public Long getBuildStartTime(int srvId, int buildId) {
return buildStartTime.get(buildIdToCacheKey(srvId, buildId));
}
@@ -180,7 +185,7 @@ public class RunHistCompactedDao {
* @param suiteId Suite id.
* @param branch Branch.
*/
- @GuavaCached(maximumSize = 200, expireAfterAccessSecs = 30, softValues = true)
+ @GuavaCached(maximumSize = 200, expireAfterWriteSecs = 120, softValues = true)
public IRunHistory getSuiteRunHist(int srvId, String suiteId, @Nullable String branch) {
RunHistKey key = getKey(srvId, suiteId, branch);
if (key == null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
index 7212243..edd603d 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
@@ -48,7 +48,8 @@ import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,12 +119,14 @@ public class RunHistSync {
int branchNameNormalized = compactor.getStringId(normalizeBranch(build.branchName(compactor)));
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
AtomicInteger cntTests = new AtomicInteger();
Map<RunHistKey, List<Invocation>> testInvMap = new HashMap<>();
build.getAllTests().forEach(t -> {
RunHistKey histKey = new RunHistKey(srvId, t.testName(), branchNameNormalized);
List<Invocation> list = testInvMap.computeIfAbsent(histKey, k -> new ArrayList<>());
- list.add(t.toInvocation(compactor, build, parmFilter));
+ list.add(t.toInvocation(build, parmFilter, successStatusStrId));
cntTests.incrementAndGet();
});