You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/20 14:22:25 UTC
[ignite-teamcity-bot] 02/07: Trusted tests & suite history
performance fixes
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 2f3cd48f1d30d8a1ce043131272ad6a1720839bf
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Jun 19 19:13:22 2019 +0300
Trusted tests & suite history performance fixes
---
.../ignite/ci/tcbot/issue/IssueDetector.java | 1 +
.../tcbot/engine/chain/BuildChainProcessor.java | 2 +-
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 24 ++++++--
.../tcbot/engine/chain/SingleBuildRunCtx.java | 4 ++
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 5 +-
.../apache/ignite/tcignited/ITeamcityIgnited.java | 1 +
.../ignite/tcignited/TeamcityIgnitedImpl.java | 1 -
.../apache/ignite/tcignited/build/FatBuildDao.java | 69 +++++++++++++++++-----
.../tcignited/history/RunHistCompactedDao.java | 1 +
.../ignite/tcignited/history/RunHistSync.java | 2 +
.../ignite/tcignited/history/SuiteInvocation.java | 25 +++++++-
11 files changed, 108 insertions(+), 27 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index e3800a7..0dc8d98 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -366,6 +366,7 @@ public class IssueDetector {
String trackedBranch,
@Nonnull Set<String> suiteTags) {
String name = testFailure.name;
+ Integer tname = compactor.getStringIdIfPresent(name);
IRunHistory runStat = tcIgnited.getTestRunHist(name, normalizeBranch);
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index 62bb209..ff88d77 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -330,7 +330,7 @@ public class BuildChainProcessor {
protected void fillBuildCounts(MultBuildRunCtx outCtx,
ITeamcityIgnited teamcityIgnited, boolean includeScheduledInfo) {
if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) {
- final List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.buildTypeId(), outCtx.branchName());
+ List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.suiteId(), outCtx.branchName());
long cntRunning = runAllBuilds
.stream()
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index 2207e2a..dac6bf1 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -98,8 +98,9 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().map(SingleBuildRunCtx::getTestLogCheckResult).filter(Objects::nonNull);
}
- public String suiteId() {
- return firstBuildInfo.suiteId();
+ /** {@inheritDoc} */
+ @Override public String suiteId() {
+ return firstBuild().map(SingleBuildRunCtx::suiteId).orElse(null);
}
/** {@inheritDoc} */
@@ -107,9 +108,6 @@ public class MultBuildRunCtx implements ISuiteResults {
return getBuildMessageProblemCount() > 0;
}
- public String buildTypeId() {
- return firstBuildInfo.buildTypeId;
- }
public boolean hasAnyBuildProblemExceptTestOrSnapshot() {
return allProblemsInAllBuilds()
@@ -170,6 +168,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().filter(ISuiteResults::hasCompilationProblem).count();
}
+ /** {@inheritDoc} */
public boolean hasTimeoutProblem() {
return getExecutionTimeoutCount() > 0;
}
@@ -178,6 +177,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().filter(SingleBuildRunCtx::hasTimeoutProblem).count();
}
+ /** {@inheritDoc} */
public boolean hasJvmCrashProblem() {
return getJvmCrashProblemCount() > 0;
}
@@ -186,10 +186,12 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsCntHavingBuildProblem(ProblemOccurrence.TC_JVM_CRASH);
}
+ /** {@inheritDoc} */
public boolean hasOomeProblem() {
return getOomeProblemCount() > 0;
}
+ /** {@inheritDoc} */
@Override public boolean hasExitCodeProblem() {
return getExitCodeProblemsCount() > 0;
}
@@ -611,7 +613,7 @@ public class MultBuildRunCtx implements ISuiteResults {
.filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
- Integer suiteName = compactor.getStringIdIfPresent( buildTypeId());
+ Integer suiteName = buildTypeIdId();
// res.clear(); //todo enable feature back
@@ -627,4 +629,14 @@ public class MultBuildRunCtx implements ISuiteResults {
return trustedCnt.get();
}
+ /**
+ * Returns suite name non compacted.
+ */
+ public Integer buildTypeIdId() {
+ return firstBuild().map(SingleBuildRunCtx::buildTypeIdId).orElse(null);
+ }
+
+ public Optional<SingleBuildRunCtx> firstBuild() {
+ return builds.stream().findFirst();
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
index dab646e..e41d65f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
@@ -346,4 +346,8 @@ public class SingleBuildRunCtx implements ISuiteResults {
public int totalNotMutedTests() {
return buildCompacted.totalNotMutedTests();
}
+
+ public int buildTypeIdId() {
+ return buildCompacted.buildTypeId();
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 4a5ca06..9f2b021 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -166,6 +166,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
String failRateNormalizedBranch = normalizeBranch(baseBranch);
String curBranchNormalized = normalizeBranch(suite.branchName());
+ Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch);
IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId());
@@ -189,8 +190,8 @@ public class DsSuiteUi extends DsHistoryStatUi {
if (includeTests) {
List<TestCompactedMult> tests = suite.getFailedTests();
- Function<IMultTestOccurrence, Float> function = foccur -> {
- IRunHistory apply = tcIgnited.getTestRunHist(foccur.getName(), failRateNormalizedBranch);
+ Function<TestCompactedMult, Float> function = foccur -> {
+ IRunHistory apply = tcIgnited.getTestRunHist(foccur.testName(), suite.buildTypeIdId(), baseBranchId);
return apply == null ? 0f : apply.getFailRate();
};
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 51429b9..38120db 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -200,6 +200,7 @@ public interface ITeamcityIgnited {
*/
public BuildTypeCompacted getBuildType(String buildTypeId);
+ @Deprecated
@Nullable public IRunHistory getTestRunHist(String testName, @Nullable String branch);
@Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch);
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index eb14115..def3851 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -421,7 +421,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** {@inheritDoc} */
@Nullable
- @AutoProfiling
@Override public IRunHistory getTestRunHist(String testName, @Nullable String branch) {
return runHistCompactedDao.getTestRunHist(srvIdMaskHigh, testName, branch);
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index f83521b..341c3b9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -230,6 +230,7 @@ public class FatBuildDao {
* @param suiteName Suite name.
* @param branchName Branch name.
*/
+ @AutoProfiling
public IRunHistory getTestRunHist(int srvIdMaskHigh,
Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) {
@@ -239,29 +240,67 @@ public class FatBuildDao {
try {
hist = runHistInMemCache.get(runHistKey,
- () -> {
- Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvIdMaskHigh, suiteName, branchName);// todo RunHistSync.normalizeBranch();
+ () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, suiteName, branchName));
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+
+ return hist.testsHistory.get(testName);
+ }
- Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
+ @AutoProfiling
+ public SuiteHistory loadSuiteHistory(int srvId,
+ Supplier<Set<Integer>> buildIdsSupplier,
+ int suiteName,
+ int branchName) {
+ Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, suiteName, branchName);// todo RunHistSync.normalizeBranch();
- HashSet<Integer> missedBuilds = new HashSet<>(buildIds);
+ Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
- missedBuilds.removeAll(suiteRunHist.keySet());
+ HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
- if (!missedBuilds.isEmpty()) {
+ missedBuildsIds.removeAll(suiteRunHist.keySet());
- }
+ if (!missedBuildsIds.isEmpty())
+ addSuiteInvocationsToHistory(srvId, suiteRunHist, missedBuildsIds);
- Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
+ SuiteHistory sumary = new SuiteHistory();
- return calcSuiteHistory(srvIdMaskHigh, buildIds);
- });
- }
- catch (ExecutionException e) {
- throw ExceptionUtil.propagateException(e);
- }
+ suiteRunHist.forEach((buildId, suiteInv) -> {
+ suiteInv.tests().forEach((tName, test) -> {
+ RunHistCompacted compacted = sumary.testsHistory.computeIfAbsent(tName,
+ k_ -> new RunHistCompacted());
- return hist.testsHistory.get(testName);
+ compacted.innerAddInvocation(test);
+ });
+
+ });
+
+ return sumary;
+ }
+
+ @AutoProfiling
+ public void addSuiteInvocationsToHistory(int srvId, Map<Integer, SuiteInvocation> suiteRunHist,
+ HashSet<Integer> missedBuildsIds) {
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ Map<Long, FatBuildCompacted> buildsCacheAll = getAllFatBuilds(srvId, missedBuildsIds);
+
+ buildsCacheAll.forEach((buildCacheKey, fatBuildCompacted) -> {
+ SuiteInvocation sinv = new SuiteInvocation(fatBuildCompacted, compactor, (k, v) -> false);
+
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+
+ sinv.addTest(testCompacted.testName(), invocation);
+ }
+ );
+
+ suiteRunHist.put(fatBuildCompacted.id(), sinv);
+ });
}
@AutoProfiling
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index 8fed725..e6478a7 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -61,6 +61,7 @@ public class RunHistCompactedDao {
private Provider<Ignite> igniteProvider;
/** Test history cache. */
+ @Deprecated
private IgniteCache<RunHistKey, RunHistCompacted> testHistCache;
/** Suite history cache. */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
index a0a90f4..9a621ee 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
@@ -205,6 +205,8 @@ public class RunHistSync {
@Nonnull protected String saveInvocationsMap(
Map<RunHistKey, List<Invocation>> buildsSaveThisRun,
Map<RunHistKey, List<Invocation>> testsSaveThisRun) {
+ if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE)))
+ return "Skipped";
if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE)))
if (testsSaveThisRun.size() > 100)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
index bd3aa80..e2cc0cd 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -16,9 +16,14 @@
*/
package org.apache.ignite.tcignited.history;
-import java.util.List;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.BiPredicate;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcbot.persistence.Persisted;
/**
@@ -41,7 +46,23 @@ public class SuiteInvocation {
Invocation suite;
- List<Invocation> tests;
+ java.util.Map<Integer, Invocation> tests = new HashMap<>();
Long buildStartTime;
+
+ public SuiteInvocation() {}
+
+ public SuiteInvocation(FatBuildCompacted buildCompacted, IStringCompactor comp,
+ BiPredicate<Integer, Integer> filter) {
+ buildStartTime = buildCompacted.getStartDateTs();
+ suite = buildCompacted.toInvocation(comp, filter);
+ }
+
+ public void addTest(int testName, Invocation invocation) {
+ tests.put(testName, invocation);
+ }
+
+ public Map<Integer, Invocation> tests() {
+ return Collections.unmodifiableMap(tests);
+ }
}