You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/20 14:22:23 UTC
[ignite-teamcity-bot] branch test-hist-performance created (now
5c777be)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git.
at 5c777be Trusted tests & suite history performance fixes: Build start time implementation using entry processor
This branch includes the following new commits:
new e78c82e Trusted tests & suite history performance fixes
new 2f3cd48 Trusted tests & suite history performance fixes
new 9a87e9c Trusted tests & suite history performance fixes: Storage into TTL persisted cache
new f01b323 Trusted tests & suite history performance fixes: Caching of test run history in the context
new 487bced Trusted tests & suite history performance fixes: skipping known builds
new 685ad52 Trusted tests & suite history performance fixes: history collection was moved to standalone class
new 5c777be Trusted tests & suite history performance fixes: Build start time implementation using entry processor
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ignite-teamcity-bot] 02/07: Trusted tests & suite history
performance fixes
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 2f3cd48f1d30d8a1ce043131272ad6a1720839bf
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Jun 19 19:13:22 2019 +0300
Trusted tests & suite history performance fixes
---
.../ignite/ci/tcbot/issue/IssueDetector.java | 1 +
.../tcbot/engine/chain/BuildChainProcessor.java | 2 +-
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 24 ++++++--
.../tcbot/engine/chain/SingleBuildRunCtx.java | 4 ++
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 5 +-
.../apache/ignite/tcignited/ITeamcityIgnited.java | 1 +
.../ignite/tcignited/TeamcityIgnitedImpl.java | 1 -
.../apache/ignite/tcignited/build/FatBuildDao.java | 69 +++++++++++++++++-----
.../tcignited/history/RunHistCompactedDao.java | 1 +
.../ignite/tcignited/history/RunHistSync.java | 2 +
.../ignite/tcignited/history/SuiteInvocation.java | 25 +++++++-
11 files changed, 108 insertions(+), 27 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index e3800a7..0dc8d98 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -366,6 +366,7 @@ public class IssueDetector {
String trackedBranch,
@Nonnull Set<String> suiteTags) {
String name = testFailure.name;
+ Integer tname = compactor.getStringIdIfPresent(name);
IRunHistory runStat = tcIgnited.getTestRunHist(name, normalizeBranch);
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index 62bb209..ff88d77 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -330,7 +330,7 @@ public class BuildChainProcessor {
protected void fillBuildCounts(MultBuildRunCtx outCtx,
ITeamcityIgnited teamcityIgnited, boolean includeScheduledInfo) {
if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) {
- final List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.buildTypeId(), outCtx.branchName());
+ List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.suiteId(), outCtx.branchName());
long cntRunning = runAllBuilds
.stream()
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index 2207e2a..dac6bf1 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -98,8 +98,9 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().map(SingleBuildRunCtx::getTestLogCheckResult).filter(Objects::nonNull);
}
- public String suiteId() {
- return firstBuildInfo.suiteId();
+ /** {@inheritDoc} */
+ @Override public String suiteId() {
+ return firstBuild().map(SingleBuildRunCtx::suiteId).orElse(null);
}
/** {@inheritDoc} */
@@ -107,9 +108,6 @@ public class MultBuildRunCtx implements ISuiteResults {
return getBuildMessageProblemCount() > 0;
}
- public String buildTypeId() {
- return firstBuildInfo.buildTypeId;
- }
public boolean hasAnyBuildProblemExceptTestOrSnapshot() {
return allProblemsInAllBuilds()
@@ -170,6 +168,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().filter(ISuiteResults::hasCompilationProblem).count();
}
+ /** {@inheritDoc} */
public boolean hasTimeoutProblem() {
return getExecutionTimeoutCount() > 0;
}
@@ -178,6 +177,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().filter(SingleBuildRunCtx::hasTimeoutProblem).count();
}
+ /** {@inheritDoc} */
public boolean hasJvmCrashProblem() {
return getJvmCrashProblemCount() > 0;
}
@@ -186,10 +186,12 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsCntHavingBuildProblem(ProblemOccurrence.TC_JVM_CRASH);
}
+ /** {@inheritDoc} */
public boolean hasOomeProblem() {
return getOomeProblemCount() > 0;
}
+ /** {@inheritDoc} */
@Override public boolean hasExitCodeProblem() {
return getExitCodeProblemsCount() > 0;
}
@@ -611,7 +613,7 @@ public class MultBuildRunCtx implements ISuiteResults {
.filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
- Integer suiteName = compactor.getStringIdIfPresent( buildTypeId());
+ Integer suiteName = buildTypeIdId();
// res.clear(); //todo enable feature back
@@ -627,4 +629,14 @@ public class MultBuildRunCtx implements ISuiteResults {
return trustedCnt.get();
}
+ /**
+ * Returns suite name non compacted.
+ */
+ public Integer buildTypeIdId() {
+ return firstBuild().map(SingleBuildRunCtx::buildTypeIdId).orElse(null);
+ }
+
+ public Optional<SingleBuildRunCtx> firstBuild() {
+ return builds.stream().findFirst();
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
index dab646e..e41d65f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
@@ -346,4 +346,8 @@ public class SingleBuildRunCtx implements ISuiteResults {
public int totalNotMutedTests() {
return buildCompacted.totalNotMutedTests();
}
+
+ public int buildTypeIdId() {
+ return buildCompacted.buildTypeId();
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 4a5ca06..9f2b021 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -166,6 +166,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
String failRateNormalizedBranch = normalizeBranch(baseBranch);
String curBranchNormalized = normalizeBranch(suite.branchName());
+ Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch);
IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId());
@@ -189,8 +190,8 @@ public class DsSuiteUi extends DsHistoryStatUi {
if (includeTests) {
List<TestCompactedMult> tests = suite.getFailedTests();
- Function<IMultTestOccurrence, Float> function = foccur -> {
- IRunHistory apply = tcIgnited.getTestRunHist(foccur.getName(), failRateNormalizedBranch);
+ Function<TestCompactedMult, Float> function = foccur -> {
+ IRunHistory apply = tcIgnited.getTestRunHist(foccur.testName(), suite.buildTypeIdId(), baseBranchId);
return apply == null ? 0f : apply.getFailRate();
};
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 51429b9..38120db 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -200,6 +200,7 @@ public interface ITeamcityIgnited {
*/
public BuildTypeCompacted getBuildType(String buildTypeId);
+ @Deprecated
@Nullable public IRunHistory getTestRunHist(String testName, @Nullable String branch);
@Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch);
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index eb14115..def3851 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -421,7 +421,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** {@inheritDoc} */
@Nullable
- @AutoProfiling
@Override public IRunHistory getTestRunHist(String testName, @Nullable String branch) {
return runHistCompactedDao.getTestRunHist(srvIdMaskHigh, testName, branch);
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index f83521b..341c3b9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -230,6 +230,7 @@ public class FatBuildDao {
* @param suiteName Suite name.
* @param branchName Branch name.
*/
+ @AutoProfiling
public IRunHistory getTestRunHist(int srvIdMaskHigh,
Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) {
@@ -239,29 +240,67 @@ public class FatBuildDao {
try {
hist = runHistInMemCache.get(runHistKey,
- () -> {
- Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvIdMaskHigh, suiteName, branchName);// todo RunHistSync.normalizeBranch();
+ () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, suiteName, branchName));
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+
+ return hist.testsHistory.get(testName);
+ }
- Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
+ @AutoProfiling
+ public SuiteHistory loadSuiteHistory(int srvId,
+ Supplier<Set<Integer>> buildIdsSupplier,
+ int suiteName,
+ int branchName) {
+ Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, suiteName, branchName);// todo RunHistSync.normalizeBranch();
- HashSet<Integer> missedBuilds = new HashSet<>(buildIds);
+ Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
- missedBuilds.removeAll(suiteRunHist.keySet());
+ HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
- if (!missedBuilds.isEmpty()) {
+ missedBuildsIds.removeAll(suiteRunHist.keySet());
- }
+ if (!missedBuildsIds.isEmpty())
+ addSuiteInvocationsToHistory(srvId, suiteRunHist, missedBuildsIds);
- Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
+ SuiteHistory sumary = new SuiteHistory();
- return calcSuiteHistory(srvIdMaskHigh, buildIds);
- });
- }
- catch (ExecutionException e) {
- throw ExceptionUtil.propagateException(e);
- }
+ suiteRunHist.forEach((buildId, suiteInv) -> {
+ suiteInv.tests().forEach((tName, test) -> {
+ RunHistCompacted compacted = sumary.testsHistory.computeIfAbsent(tName,
+ k_ -> new RunHistCompacted());
- return hist.testsHistory.get(testName);
+ compacted.innerAddInvocation(test);
+ });
+
+ });
+
+ return sumary;
+ }
+
+ @AutoProfiling
+ public void addSuiteInvocationsToHistory(int srvId, Map<Integer, SuiteInvocation> suiteRunHist,
+ HashSet<Integer> missedBuildsIds) {
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ Map<Long, FatBuildCompacted> buildsCacheAll = getAllFatBuilds(srvId, missedBuildsIds);
+
+ buildsCacheAll.forEach((buildCacheKey, fatBuildCompacted) -> {
+ SuiteInvocation sinv = new SuiteInvocation(fatBuildCompacted, compactor, (k, v) -> false);
+
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+
+ sinv.addTest(testCompacted.testName(), invocation);
+ }
+ );
+
+ suiteRunHist.put(fatBuildCompacted.id(), sinv);
+ });
}
@AutoProfiling
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index 8fed725..e6478a7 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -61,6 +61,7 @@ public class RunHistCompactedDao {
private Provider<Ignite> igniteProvider;
/** Test history cache. */
+ @Deprecated
private IgniteCache<RunHistKey, RunHistCompacted> testHistCache;
/** Suite history cache. */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
index a0a90f4..9a621ee 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
@@ -205,6 +205,8 @@ public class RunHistSync {
@Nonnull protected String saveInvocationsMap(
Map<RunHistKey, List<Invocation>> buildsSaveThisRun,
Map<RunHistKey, List<Invocation>> testsSaveThisRun) {
+ if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE)))
+ return "Skipped";
if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE)))
if (testsSaveThisRun.size() > 100)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
index bd3aa80..e2cc0cd 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -16,9 +16,14 @@
*/
package org.apache.ignite.tcignited.history;
-import java.util.List;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.BiPredicate;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcbot.persistence.Persisted;
/**
@@ -41,7 +46,23 @@ public class SuiteInvocation {
Invocation suite;
- List<Invocation> tests;
+ java.util.Map<Integer, Invocation> tests = new HashMap<>();
Long buildStartTime;
+
+ public SuiteInvocation() {}
+
+ public SuiteInvocation(FatBuildCompacted buildCompacted, IStringCompactor comp,
+ BiPredicate<Integer, Integer> filter) {
+ buildStartTime = buildCompacted.getStartDateTs();
+ suite = buildCompacted.toInvocation(comp, filter);
+ }
+
+ public void addTest(int testName, Invocation invocation) {
+ tests.put(testName, invocation);
+ }
+
+ public Map<Integer, Invocation> tests() {
+ return Collections.unmodifiableMap(tests);
+ }
}
[ignite-teamcity-bot] 04/07: Trusted tests & suite history
performance fixes: Caching of test run history in the context
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit f01b323fd5bff8f9f54f43f7cf121cd7cf3e2943
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Jun 19 21:43:36 2019 +0300
Trusted tests & suite history performance fixes: Caching of test run history in the context
---
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 15 ++++----
.../tcbot/engine/chain/TestCompactedMult.java | 10 ++++++
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 5 ++-
.../apache/ignite/tcbot/engine/ui/DsChainUi.java | 15 ++++----
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 42 ++++++++++++----------
.../ignite/tcbot/engine/ui/DsTestFailureUi.java | 41 ++++++++++-----------
.../ignite/tcbot/engine/ui/DsTestHistoryUi.java | 2 +-
7 files changed, 70 insertions(+), 60 deletions(-)
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index dac6bf1..5fb0b2d 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -305,8 +305,8 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(logSizeBytes.entrySet().stream(), 3, comparing).stream();
}
- public Stream<? extends IMultTestOccurrence> getTopLongRunning() {
- Comparator<IMultTestOccurrence> comparing = Comparator.comparing(IMultTestOccurrence::getAvgDurationMs);
+ public Stream<TestCompactedMult> getTopLongRunning() {
+ Comparator<TestCompactedMult> comparing = Comparator.comparing(TestCompactedMult::getAvgDurationMs);
Map<Integer, TestCompactedMult> res = new HashMap<>();
@@ -609,17 +609,14 @@ public class MultBuildRunCtx implements ISuiteResults {
//todo can cache mult occurrences in ctx
builds.forEach(singleBuildRunCtx -> {
- saveToMap(res, singleBuildRunCtx.getAllTests()
- .filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
+ saveToMap(res,
+ singleBuildRunCtx.getAllTests().filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
Integer suiteName = buildTypeIdId();
- // res.clear(); //todo enable feature back
-
- //todo can cache fail rate in mult occur
- res.keySet().forEach((testNameId) -> {
- IRunHistory stat = tcIgnited.getTestRunHist(testNameId, suiteName, branchName);
+ res.forEach((testNameId, compactedMult) -> {
+ IRunHistory stat = compactedMult.history(tcIgnited, suiteName, branchName);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
boolean b = testBlockerComment != null;
if (b) // this test will be considered as blocker if will fail
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
index 792d5c6..b1ab48f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
@@ -20,11 +20,14 @@ package org.apache.ignite.tcbot.engine.chain;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.history.IRunHistSummary;
+import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
@@ -35,6 +38,7 @@ public class TestCompactedMult implements IMultTestOccurrence {
private final List<TestCompacted> occurrences = new ArrayList<>();
private IStringCompactor compactor;
private long avgDuration = -1;
+ private java.util.Map<Integer, IRunHistory> historyCacheMap = new ConcurrentHashMap<>();
public TestCompactedMult(IStringCompactor compactor) {
this.compactor = compactor;
@@ -113,4 +117,10 @@ public class TestCompactedMult implements IMultTestOccurrence {
public void add(TestCompacted next) {
occurrences.add(next);
}
+
+
+ public IRunHistory history(ITeamcityIgnited ignited, Integer buildTypeIdId, Integer baseBranchId) {
+ return historyCacheMap.computeIfAbsent(baseBranchId,
+ (k)-> ignited.getTestRunHist(testName(), buildTypeIdId, k));
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index b3fb4bd..0e93b57 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -265,14 +265,13 @@ public class PrChainsProcessor {
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
List<DsTestFailureUi> failures = ctx.getFailedTests().stream().map(occurrence -> {
- IRunHistory stat = tcIgnited.getTestRunHist(occurrence.testName(), suiteId, baseBranchId);
-
+ IRunHistory stat = occurrence.history(tcIgnited, suiteId, baseBranchId);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
if (!Strings.isNullOrEmpty(testBlockerComment)) {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch);
+ failure.initFromOccurrence(occurrence, ctx.buildTypeIdId(), tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch, baseBranchId);
return failure;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
index 913c08a..d8c8746 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
@@ -24,13 +24,12 @@ import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;
import javax.annotation.Nullable;
-
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.tcbot.common.util.CollectionUtil;
import org.apache.ignite.tcbot.common.util.UrlUtil;
import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
-import org.apache.ignite.tcbot.common.util.CollectionUtil;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcservice.model.conf.BuildType;
@@ -208,18 +207,18 @@ public class DsChainUi {
webToHist = buildWebLink(tcIgnited, ctx);
webToBuild = buildWebLinkToBuild(tcIgnited, ctx);
- Stream<T2<MultBuildRunCtx, IMultTestOccurrence>> allLongRunning = ctx.suites().flatMap(
+ Stream<T2<MultBuildRunCtx, TestCompactedMult>> allLongRunning = ctx.suites().flatMap(
suite -> suite.getTopLongRunning().map(t -> new T2<>(suite, t))
);
- Comparator<T2<MultBuildRunCtx, IMultTestOccurrence>> durationComp
+ Comparator<T2<MultBuildRunCtx, TestCompactedMult>> durationComp
= Comparator.comparing((pair) -> pair.get2().getAvgDurationMs());
CollectionUtil.top(allLongRunning, 3, durationComp).forEach(
pairCtxAndOccur -> {
MultBuildRunCtx suite = pairCtxAndOccur.get1();
- IMultTestOccurrence longRunningOccur = pairCtxAndOccur.get2();
+ TestCompactedMult longRunningOccur = pairCtxAndOccur.get2();
- DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, longRunningOccur, baseBranchTc);
+ DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, longRunningOccur, baseBranchTc);
failure.testName = "[" + suite.suiteName() + "] " + failure.testName; //may be separate field
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 9f2b021..8a1b003 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -29,20 +29,18 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.tcbot.common.util.UrlUtil;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
import org.apache.ignite.tcbot.engine.ui.BotUrls.GetBuildLog;
-import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
-import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
+import org.apache.ignite.tcignited.history.IRunHistory;
-import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable;
+import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
@@ -165,9 +163,11 @@ public class DsSuiteUi extends DsHistoryStatUi {
name = suite.suiteName();
String failRateNormalizedBranch = normalizeBranch(baseBranch);
- String curBranchNormalized = normalizeBranch(suite.branchName());
Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch);
+ String curBranchNormalized = normalizeBranch(suite.branchName());
+ Integer curBranchId = compactor.getStringIdIfPresent(curBranchNormalized);
+
IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId());
Set<String> collect = suite.lastChangeUsers().collect(Collectors.toSet());
@@ -188,26 +188,28 @@ public class DsSuiteUi extends DsHistoryStatUi {
webToHistBaseBranch = buildWebLink(tcIgnited, suite, baseBranch);
webToBuild = buildWebLinkToBuild(tcIgnited, suite);
+ Integer buildTypeIdId = suite.buildTypeIdId();
if (includeTests) {
List<TestCompactedMult> tests = suite.getFailedTests();
- Function<TestCompactedMult, Float> function = foccur -> {
- IRunHistory apply = tcIgnited.getTestRunHist(foccur.testName(), suite.buildTypeIdId(), baseBranchId);
+ Function<TestCompactedMult, Float> function = testCompactedMult -> {
+ IRunHistory res = testCompactedMult.history(tcIgnited, buildTypeIdId, baseBranchId);
- return apply == null ? 0f : apply.getFailRate();
+ return res == null ? 0f : res.getFailRate();
};
tests.sort(Comparator.comparing(function).reversed());
tests.forEach(occurrence -> {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), baseBranch);
- failure.initStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized);
+ failure.initFromOccurrence(occurrence, buildTypeIdId, tcIgnited, suite.projectId(),
+ suite.branchName(), baseBranch, baseBranchId);
+ failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, curBranchId);
testFailures.add(failure);
});
suite.getTopLongRunning().forEach(occurrence -> {
- final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, occurrence, baseBranch);
+ final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, occurrence, baseBranch);
topLongRunning.add(failure);
});
@@ -317,16 +319,18 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
@Nonnull public static DsTestFailureUi createOrrucForLongRun(ITeamcityIgnited tcIgnited,
- @Nonnull MultBuildRunCtx suite,
- final IMultTestOccurrence occurrence,
- @Nullable final String failRateBranch) {
+ IStringCompactor compactor, @Nonnull MultBuildRunCtx suite,
+ final TestCompactedMult occurrence,
+ @Nullable final String failRateBranch) {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), failRateBranch);
+ Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(failRateBranch));
+ Integer buildTypeIdId = suite.buildTypeIdId();
+ failure.initFromOccurrence(occurrence, buildTypeIdId, tcIgnited, suite.projectId(), suite.branchName(),
+ failRateBranch, baseBranchId);
- failure.initStat(tcIgnited,
- normalizeBranch(failRateBranch),
- normalizeBranch(suite.branchName()));
+ failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId,
+ compactor.getStringIdIfPresent(normalizeBranch(suite.branchName())));
return failure;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
index b0cb5d9..9c146f9 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
@@ -25,17 +25,15 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.tcbot.common.util.UrlUtil;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildlog.LogMsgToWarn;
import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcignited.ITeamcityIgnited;
-import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable;
+import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
@@ -92,16 +90,20 @@ public class DsTestFailureUi {
/**
* @param failure test ocurrence (probably multiple)
+ * @param buildTypeIdId
* @param tcIgn Teamcity.
* @param projectId project ID.
- * @param branchName
- * @param baseBranchName base branch name (e.g. master).
+ * @param branchName current branch name.
+ * @param baseBranchName base branch name (e.g. master), without normalization.
+ * @param baseBranchId Normalized base branch ID (from compactor).
*/
- public void initFromOccurrence(@Nonnull final IMultTestOccurrence failure,
+ public void initFromOccurrence(@Nonnull final TestCompactedMult failure,
+ Integer buildTypeIdId,
@Nonnull final ITeamcityIgnited tcIgn,
@Nullable final String projectId,
@Nullable final String branchName,
- @Nullable final String baseBranchName) {
+ @Nullable final String baseBranchName,
+ Integer baseBranchId) {
name = failure.getName();
investigated = failure.isInvestigated();
curFailures = failure.failuresCount();
@@ -144,8 +146,7 @@ public class DsTestFailureUi {
webUrlBaseBranch = buildWebLink(tcIgn, full.test.id, projectId, baseBranchName);
});
- final IRunHistory stat = tcIgn.getTestRunHist(name, normalizeBranch(baseBranchName));
-
+ final IRunHistory stat = failure.history(tcIgn, buildTypeIdId, baseBranchId);
blockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
}
@@ -191,22 +192,22 @@ public class DsTestFailureUi {
}
/**
+ * @param occurrence
+ * @param buildTypeIdId
* @param tcIgnited TC service as Run stat supplier.
- * @param failRateNormalizedBranch Base branch: Fail rate and flakyness detection normalized branch.
+ * @param baseBranchId Base branch: Fail rate and flakyness detection normalized branch.
* @param curBranchNormalized Cur branch normalized.
*/
- public void initStat(ITeamcityIgnited tcIgnited,
- String failRateNormalizedBranch,
- String curBranchNormalized) {
-
- final IRunHistory stat = tcIgnited.getTestRunHist(name, failRateNormalizedBranch);
-
+ public void initStat(TestCompactedMult occurrence, Integer buildTypeIdId, ITeamcityIgnited tcIgnited,
+ @Nullable Integer baseBranchId,
+ @Nullable Integer curBranchNormalized) {
+ final IRunHistory stat = occurrence.history(tcIgnited, buildTypeIdId, baseBranchId);
histBaseBranch.init(stat);
- IRunHistory statForProblemsDetection = null;
+ IRunHistory statForProblemsDetection;
- if (!curBranchNormalized.equals(failRateNormalizedBranch)) {
- statForProblemsDetection = tcIgnited.getTestRunHist(name, curBranchNormalized);
+ if (!Objects.equals(curBranchNormalized, baseBranchId)) {
+ statForProblemsDetection = occurrence.history(tcIgnited, buildTypeIdId, curBranchNormalized);
if (statForProblemsDetection != null) {
histCurBranch = new DsTestHistoryUi();
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
index 8b8597f..e0fe975 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
@@ -27,7 +27,7 @@ import org.apache.ignite.tcignited.history.IRunHistory;
*/
public class DsTestHistoryUi {
/** 'All the time' runs history statistic. */
- public DsHistoryStatUi allTime = new DsHistoryStatUi();
+ @Deprecated public DsHistoryStatUi allTime = new DsHistoryStatUi();
/** Latest runs history statistic. */
public DsHistoryStatUi recent = new DsHistoryStatUi();
[ignite-teamcity-bot] 05/07: Trusted tests & suite history
performance fixes: skipping known builds
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 487bced945579380248ceb33e6eeb30f1d426f68
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Jun 19 21:50:40 2019 +0300
Trusted tests & suite history performance fixes: skipping known builds
---
.../apache/ignite/tcignited/TeamcityIgnitedImpl.java | 20 +++++++++++---------
.../apache/ignite/tcignited/build/FatBuildDao.java | 15 +++++++++------
2 files changed, 20 insertions(+), 15 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 707285f..dfdf970 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -34,7 +34,7 @@ import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Supplier;
+import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -441,21 +441,23 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0)
return null;
- Supplier<Set<Integer>> supplier = () -> {
+ Function<Set<Integer>, Set<Integer>> supplier = (knownBuilds) -> {
String btId = compactor.getStringFromId(buildTypeId);
String branchId = compactor.getStringFromId(normalizedBaseBranch);
List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId);
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = compacted.stream()
+ .filter(b -> !knownBuilds.contains(b.id()))
+ //todo filter queued, cancelled and so on
.filter(
- bRef -> {
- Long startTime = getBuildStartTime(bRef.id());
- if (startTime == null)
- return false;
+ bRef -> {
+ Long startTime = getBuildStartTime(bRef.id());
+ if (startTime == null)
+ return false;
- return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
- }
- ).map(BuildRefCompacted::id).collect(Collectors.toSet());
+ return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
+ }
+ ).map(BuildRefCompacted::id).collect(Collectors.toSet());
System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index af59fd9..e0dce95 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -28,7 +28,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -232,7 +232,7 @@ public class FatBuildDao {
*/
@AutoProfiling
public IRunHistory getTestRunHist(int srvIdMaskHigh,
- Supplier<Set<Integer>> buildIdsSupplier, int testName, int buildTypeId, int normalizedBaseBranch) {
+ Function<Set<Integer>, Set<Integer>> buildIdsSupplier, int testName, int buildTypeId, int normalizedBaseBranch) {
RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
@@ -248,14 +248,16 @@ public class FatBuildDao {
return hist.testsHistory.get(testName);
}
+
+ //todo create standalone history collector class
@AutoProfiling
public SuiteHistory loadSuiteHistory(int srvId,
- Supplier<Set<Integer>> buildIdsSupplier,
+ Function<Set<Integer>, Set<Integer>> buildIdsSupplier,
int buildTypeId,
int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
- Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
+ Set<Integer> buildIds = determineLatestBuildsFunction(buildIdsSupplier, suiteRunHist.keySet());
HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
@@ -361,8 +363,9 @@ public class FatBuildDao {
}
@AutoProfiling
- protected Set<Integer> determineLatestBuilds(Supplier<Set<Integer>> buildIdsSupplier) {
- return buildIdsSupplier.get();
+ protected Set<Integer> determineLatestBuildsFunction(Function<Set<Integer>, Set<Integer>> buildIdsSupplier,
+ Set<Integer> known) {
+ return buildIdsSupplier.apply(known);
}
public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
[ignite-teamcity-bot] 07/07: Trusted tests & suite history
performance fixes: Build start time implementation using entry processor
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 5c777bebbe2277cd212716b6fecf5ea321c8d95d
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jun 20 17:08:15 2019 +0300
Trusted tests & suite history performance fixes: Build start time implementation using entry processor
---
.../apache/ignite/tcignited/build/FatBuildDao.java | 36 ++++++++-
.../ignite/tcignited/history/HistoryCollector.java | 93 ++++++++++------------
2 files changed, 73 insertions(+), 56 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 4424158..98ef8fd 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -19,6 +19,7 @@ package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -29,14 +30,19 @@ import java.util.stream.StreamSupport;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.cache.Cache;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcservice.model.changes.ChangesList;
import org.apache.ignite.tcservice.model.result.Build;
@@ -191,12 +197,36 @@ public class FatBuildDao {
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
+ private static Set<Long> buildsIdsToCacheKeys(int srvId, Collection<Integer> stream) {
+ return stream.stream()
+ .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet());
+ }
+ public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) {
+ Map<Long, EntryProcessorResult<Long>> map = buildsCache.invokeAll(buildsIdsToCacheKeys(srvId, ids), new GetStartTimeProc());
- private static Set<Long> buildsIdsToCacheKeys(int srvIdMaskHigh, Collection<Integer> stream) {
- return stream.stream()
- .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
+ HashMap<Integer, Long> res = new HashMap<>();
+
+ map.forEach((k, r)->{
+ Long aLong = r.get();
+ if(aLong!=null)
+ res.put(BuildRefDao.cacheKeyToBuildId(k), aLong);
+ });
+
+ return res;
}
+ private static class GetStartTimeProc implements CacheEntryProcessor<Long, FatBuildCompacted, Long> {
+ public GetStartTimeProc() {
+ }
+
+ @Override public Long process(MutableEntry<Long, FatBuildCompacted> entry,
+ Object... arguments) throws EntryProcessorException {
+ if (entry.getValue() == null)
+ return null;
+ FatBuildCompacted fatBuildCompacted = entry.getValue();
+ return fatBuildCompacted.getStartDateTs();
+ }
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index 68743b7..f5b720e 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -27,14 +27,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.MutableEntry;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
-import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
@@ -115,10 +111,10 @@ public class HistoryCollector {
* @param srvId
* @param buildTypeId
* @param normalizedBaseBranch
- * @param knownBuilds Known builds.
+ * @param knownBuilds Known builds, which already present in run history.
*/
@AutoProfiling
- protected Set<Integer> determineLatestBuildsFunction(
+ protected Set<Integer> determineLatestBuilds(
int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
String btId = compactor.getStringFromId(buildTypeId);
String branchId = compactor.getStringFromId(normalizedBaseBranch);
@@ -127,31 +123,50 @@ public class HistoryCollector {
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = compacted.stream()
.filter(b -> !knownBuilds.contains(b.id()))
- //todo filter queued, cancelled and so on
- .filter(
- bRef -> {
- //todo getAll
- //todo getStartTime From FatBuild
- Long startTime = runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
- if (startTime == null)
- return false;
-
- return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
- }
- ).map(BuildRefCompacted::id).collect(Collectors.toSet());
+ .filter(this::applicableForHistory)
+ .map(BuildRefCompacted::id).collect(Collectors.toSet());
+
+ Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, buildIds);
+
+ //todo filter queued, cancelled and so on
+ Set<Integer> buildInScope = buildIds.stream().filter(
+ bId -> {
+ //todo getAll
+ //todo getStartTime From FatBuild
+ Long startTime = buildStartTimeFromFatBuild.get(bId);
+ //todo cache in runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
+ if (startTime == null)
+ return false;
+
+ return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
+ }
+ ).collect(Collectors.toSet());
- System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+ System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " +
+ buildInScope+ " from " + buildIds.size());
return buildIds;
}
@AutoProfiling
+ protected Map<Integer, Long> getStartTimeFromFatBuild(int srvId, Set<Integer> buildIds) {
+ return fatBuildDao.getBuildStartTime(srvId, buildIds);
+ }
+
+ /**
+ * @param ref Build Reference or fat build.
+ */
+ private boolean applicableForHistory(BuildRefCompacted ref) {
+ return !ref.isFakeStub() && !ref.isCancelled(compactor) && ref.isFinished(compactor);
+ }
+
+ @AutoProfiling
public SuiteHistory loadSuiteHistory(int srvId,
int buildTypeId,
int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
- Set<Integer> buildIds = determineLatestBuildsFunction(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
+ Set<Integer> buildIds = determineLatestBuilds(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
@@ -162,8 +177,8 @@ public class HistoryCollector {
System.err.println("***** + Adding to persisted history for suite "
+ compactor.getStringFromId(buildTypeId)
- + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- addl.size() + " invocations");
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " +
+ addl.size() + " invocations from " + missedBuildsIds + " builds checked");
histDao.putAll(srvId, addl);
suiteRunHist.putAll(addl);
@@ -191,8 +206,6 @@ public class HistoryCollector {
}
-
-
@AutoProfiling
public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
@@ -200,6 +213,9 @@ public class HistoryCollector {
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
fatBuildDao.getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
+ if(!applicableForHistory(fatBuildCompacted))
+ return;
+
SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
@@ -258,33 +274,4 @@ public class HistoryCollector {
return hist;
}
*/
-
-
- private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
- private final int successStatusStrId;
-
- public HistoryCollectProcessor(int successStatusStrId) {
- this.successStatusStrId = successStatusStrId;
- }
-
- @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
- Object... arguments) throws EntryProcessorException {
- if (entry.getValue() == null)
- return null;
-
- Map<Integer, Invocation> hist = new HashMap<>();
- FatBuildCompacted fatBuildCompacted = entry.getValue();
- Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
- tests.forEach(
- testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
-
- hist.put(testCompacted.testName(), invocation);
- }
- );
-
- return hist;
- }
- }
-
}
[ignite-teamcity-bot] 06/07: Trusted tests & suite history
performance fixes: history collection was moved to standalone class
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 685ad52962f4b3c23590433bda7ea52f4e2a7726
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jun 20 15:22:11 2019 +0300
Trusted tests & suite history performance fixes: history collection was moved to standalone class
---
.../ignite/tcignited/TeamcityIgnitedImpl.java | 58 ++---
.../ignite/tcignited/TeamcityIgnitedModule.java | 2 +
.../apache/ignite/tcignited/build/FatBuildDao.java | 213 +--------------
.../ignite/tcignited/build/SuiteHistory.java | 14 +
.../BranchEquivalence.java} | 35 ++-
.../ignite/tcignited/buildref/BuildRefDao.java | 2 +-
.../ignite/tcignited/history/HistoryCollector.java | 290 +++++++++++++++++++++
7 files changed, 346 insertions(+), 268 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index dfdf970..18ea52b 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -61,12 +61,15 @@ import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
+import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.buildref.BuildRefSync;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.tcignited.history.RunHistSync;
+import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.tcservice.ITeamcity;
@@ -94,11 +97,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Max build id diff to enforce reload during incremental refresh. */
public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000;
- /** Default synonyms. */
- private static final List<String> DEFAULT_SYNONYMS
- = Collections.unmodifiableList(
- Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER));
-
/** Server (service) code. */
private String srvCode;
@@ -150,11 +148,20 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Run history sync. */
@Inject private RunHistSync runHistSync;
- @Inject private BuildLogCheckResultDao logCheckResultDao;
+ /** Logger check result DAO. */
+ @Inject private BuildLogCheckResultDao logCheckResDao;
+
+ /** History DAO. */
+ @Inject private SuiteInvocationHistoryDao histDao;
+
+ /** History collector. */
+ @Inject private HistoryCollector histCollector;
/** Strings compactor. */
@Inject private IStringCompactor compactor;
+ @Inject private BranchEquivalence branchEquivalence;
+
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
@@ -171,7 +178,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
changesDao.init();
runHistCompactedDao.init();
muteDao.init();
- logCheckResultDao.init();
+ logCheckResDao.init();
+ histDao.init();
}
/**
@@ -351,7 +359,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Nullable String branchName) {
ensureActualizeRequested();
- return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchForQuery(branchName));
+ return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchEquivalence.branchForQuery(branchName));
}
/** {@inheritDoc} */
@@ -364,7 +372,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (stateQueuedId == null)
return Collections.emptyList();
- Set<Integer> branchNameIds = branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str))
+ Set<Integer> branchNameIds = branchEquivalence.branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str))
.filter(Objects::nonNull).collect(Collectors.toSet());
List<BuildRefCompacted> res = new ArrayList<>();
@@ -441,30 +449,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0)
return null;
- Function<Set<Integer>, Set<Integer>> supplier = (knownBuilds) -> {
- String btId = compactor.getStringFromId(buildTypeId);
- String branchId = compactor.getStringFromId(normalizedBaseBranch);
- List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId);
- long curTs = System.currentTimeMillis();
- Set<Integer> buildIds = compacted.stream()
- .filter(b -> !knownBuilds.contains(b.id()))
- //todo filter queued, cancelled and so on
- .filter(
- bRef -> {
- Long startTime = getBuildStartTime(bRef.id());
- if (startTime == null)
- return false;
-
- return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
- }
- ).map(BuildRefCompacted::id).collect(Collectors.toSet());
-
- System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
-
- return buildIds;
- };
-
- return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, buildTypeId, normalizedBaseBranch);
+ return histCollector.getTestRunHist(srvIdMaskHigh, testName, buildTypeId, normalizedBaseBranch);
}
/** {@inheritDoc} */
@@ -509,13 +494,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return buildTypeDao.getFatBuildType(srvIdMaskHigh, buildTypeId);
}
- public List<String> branchForQuery(@Nullable String branchName) {
- if (ITeamcity.DEFAULT.equals(branchName))
- return DEFAULT_SYNONYMS;
- else
- return Collections.singletonList(branchName);
- }
-
/**
* Enables scheduling for build refs/builds/history sync
*/
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
index 86cb573..bc0b203 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
@@ -67,6 +68,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(MuteSync.class).in(new SingletonScope());
bind(BuildLogCheckResultDao.class).in(new SingletonScope());
bind(SuiteInvocationHistoryDao.class).in(new SingletonScope());
+ bind(HistoryCollector.class).in(new SingletonScope());
TcRealConnectionModule module = new TcRealConnectionModule();
if (conn != null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index e0dce95..4424158 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -18,49 +18,30 @@
package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.cache.Cache;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
-import javax.cache.processor.MutableEntry;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheEntryProcessor;
-import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
-import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcignited.history.SuiteInvocation;
-import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcservice.model.changes.ChangesList;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
-import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,19 +65,8 @@ public class FatBuildDao {
/** Compactor. */
@Inject private IStringCompactor compactor;
- @Inject SuiteInvocationHistoryDao historyDao;
-
- /**
- * Non persistence cache for all suite RunHistory for particular branch.
- * RunHistKey(ServerId||BranchId||suiteId)-> Build reference
- */
- private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache
- = CacheBuilder.newBuilder()
- .maximumSize(8000)
- .expireAfterAccess(16, TimeUnit.MINUTES)
- .softValues()
- .build();
-
+ /** History collector. */
+ @Inject private HistoryCollector histCollector;
/**
*
@@ -104,8 +74,6 @@ public class FatBuildDao {
public FatBuildDao init() {
buildsCache = igniteProvider.get().getOrCreateCache(CacheConfigs.getCacheV2Config(TEAMCITY_FAT_BUILD_CACHE_NAME));
- historyDao.init();
-
return this;
}
@@ -158,7 +126,7 @@ public class FatBuildDao {
public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
- invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
+ histCollector.invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
}
public static int[] extractChangeIds(@Nonnull ChangesList changesList) {
@@ -223,185 +191,12 @@ public class FatBuildDao {
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
- /**
- * @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
- * @param buildIdsSupplier Latest actual Build ids supplier. This supplier should handle all equivalent branches in it.
- * @param testName Test name.
- * @param buildTypeId Suite (Build type) id.
- * @param normalizedBaseBranch Branch name.
- */
- @AutoProfiling
- public IRunHistory getTestRunHist(int srvIdMaskHigh,
- Function<Set<Integer>, Set<Integer>> buildIdsSupplier, int testName, int buildTypeId, int normalizedBaseBranch) {
-
- RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
-
- SuiteHistory hist;
- try {
- hist = runHistInMemCache.get(runHistKey,
- () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, buildTypeId, normalizedBaseBranch));
- }
- catch (ExecutionException e) {
- throw ExceptionUtil.propagateException(e);
- }
-
- return hist.testsHistory.get(testName);
- }
-
-
- //todo create standalone history collector class
- @AutoProfiling
- public SuiteHistory loadSuiteHistory(int srvId,
- Function<Set<Integer>, Set<Integer>> buildIdsSupplier,
- int buildTypeId,
- int normalizedBaseBranch) {
- Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
-
- Set<Integer> buildIds = determineLatestBuildsFunction(buildIdsSupplier, suiteRunHist.keySet());
-
- HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
-
- missedBuildsIds.removeAll(suiteRunHist.keySet());
-
- if (!missedBuildsIds.isEmpty()) {
- Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
-
- System.err.println("***** + Adding to persisted history for suite "
- + compactor.getStringFromId(buildTypeId)
- + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- addl.size() + " invocations");
-
- historyDao.putAll(srvId, addl);
- suiteRunHist.putAll(addl);
- }
-
- SuiteHistory sumary = new SuiteHistory();
-
- suiteRunHist.forEach((buildId, suiteInv) -> {
- suiteInv.tests().forEach((tName, test) -> {
- sumary.testsHistory.computeIfAbsent(tName,
- k_ -> new RunHistCompacted()).innerAddInvocation(test);
- });
-
- });
-
- System.err.println("***** History for suite "
- + compactor.getStringFromId(buildTypeId)
- + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- sumary.size(igniteProvider.get()) + " bytes");
-
- return sumary;
- }
-
- @AutoProfiling
- public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
- HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
- Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
- int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
-
- getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
- SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
-
- Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
- tests.forEach(
- testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
-
- sinv.addTest(testCompacted.testName(), invocation);
- }
- );
-
- suiteRunHist.put(fatBuildCompacted.id(), sinv);
- });
-
- return suiteRunHist;
- }
-
- @AutoProfiling
- protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
- Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
-
- int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
-
- CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new HistoryCollectProcessor(successStatusStrId);
-
- Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
-
- SuiteHistory hist = new SuiteHistory();
-
- map.values().forEach(
- res -> {
- if (res == null)
- return;
-
- Map<Integer, Invocation> invocationMap = res.get();
-
- if (invocationMap == null)
- return;
-
- invocationMap.forEach((k, v) -> {
- RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
- k_ -> new RunHistCompacted());
-
- compacted.innerAddInvocation(v);
- });
-
- }
- );
- System.err.println("Suite history: tests in scope "
- + hist.testsHistory.size()
- + " for " +buildIds.size() + " builds checked"
- + " size " + hist.size(igniteProvider.get()));
-
- return hist;
- }
private static Set<Long> buildsIdsToCacheKeys(int srvIdMaskHigh, Collection<Integer> stream) {
return stream.stream()
.filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
}
- @AutoProfiling
- protected Set<Integer> determineLatestBuildsFunction(Function<Set<Integer>, Set<Integer>> buildIdsSupplier,
- Set<Integer> known) {
- return buildIdsSupplier.apply(known);
- }
-
- public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
- Iterable<RunHistKey> objects =
- stream
- .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
- .collect(Collectors.toSet());
-
- runHistInMemCache.invalidateAll(objects);
- }
-
- private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
- private final int successStatusStrId;
-
- public HistoryCollectProcessor(int successStatusStrId) {
- this.successStatusStrId = successStatusStrId;
- }
-
- @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
- Object... arguments) throws EntryProcessorException {
- if (entry.getValue() == null)
- return null;
-
- Map<Integer, Invocation> hist = new HashMap<>();
- FatBuildCompacted fatBuildCompacted = entry.getValue();
- Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
- tests.forEach(
- testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
-
- hist.put(testCompacted.testName(), invocation);
- }
- );
-
- return hist;
- }
- }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index a7af3f8..394b75d 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -22,8 +22,10 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.ignite.Ignite;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.internal.binary.BinaryObjectExImpl;
+import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.SuiteInvocation;
/**
@@ -37,4 +39,16 @@ public class SuiteHistory {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
return binary.length();
}
+
+ public IRunHistory getTestRunHist(int name) {
+ return testsHistory.get(name);
+ }
+
+ public RunHistCompacted getOrAddTestsHistory(Integer tName) {
+ return testsHistory.computeIfAbsent(tName, k_ -> new RunHistCompacted());
+ }
+
+ public void addTestInvocation(Integer tName, Invocation invocation) {
+ getOrAddTestsHistory(tName).innerAddInvocation(invocation);
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
similarity index 53%
copy from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
index a7af3f8..ab5f0b9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
@@ -14,27 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ignite.tcignited.buildref;
-package org.apache.ignite.tcignited.build;
+import com.google.common.collect.Lists;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.ignite.tcservice.ITeamcity;
-import java.util.HashMap;
-import java.util.Map;
+public class BranchEquivalence {
+ /** Default synonyms. */
+ private static final List<String> DEFAULT_SYNONYMS
+ = Collections.unmodifiableList(
+ Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER));
-import java.util.TreeMap;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
-import org.apache.ignite.internal.binary.BinaryObjectExImpl;
-import org.apache.ignite.tcignited.history.SuiteInvocation;
-/**
- * Suite run history summary.
- */
-public class SuiteHistory {
- /** Tests history: Test name ID->RunHistory */
- Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
-
- public int size(Ignite ignite) {
- BinaryObjectExImpl binary = ignite.binary().toBinary(this);
- return binary.length();
+ public List<String> branchForQuery(@Nullable String branchName) {
+ if (ITeamcity.DEFAULT.equals(branchName))
+ return DEFAULT_SYNONYMS;
+ else
+ return Collections.singletonList(branchName);
}
+
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index f3a3e89..e8c8ae0 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -175,7 +175,7 @@ public class BuildRefDao {
*/
@AutoProfiling
@Nonnull public List<BuildRefCompacted> getAllBuildsCompacted(int srvId,
- String buildTypeId,
+ String buildTypeId,
List<String> bracnhNameQry) {
Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId);
if (buildTypeIdId == null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
new file mode 100644
index 0000000..68743b7
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+import com.google.common.cache.CacheBuilder;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
+import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.build.SuiteHistory;
+import org.apache.ignite.tcignited.buildref.BranchEquivalence;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
+
+/**
+ *
+ */
+public class HistoryCollector {
+ /** History DAO. */
+ @Inject private SuiteInvocationHistoryDao histDao;
+
+ /** Fat build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
+ /** Build reference DAO. */
+ @Inject private BuildRefDao buildRefDao;
+
+ /** Compactor. */
+ @Inject private IStringCompactor compactor;
+
+ /** Ignite provider. */
+ @Inject private Provider<Ignite> igniteProvider;
+
+ /** Branch equivalence. */
+ @Inject private BranchEquivalence branchEquivalence;
+
+ /** Run history DAO. */
+ @Inject private RunHistCompactedDao runHistCompactedDao;
+
+ /**
+ * Non persistence cache for all suite RunHistory for particular branch. RunHistKey(ServerId||BranchId||suiteId)->
+ * Build reference
+ */
+ private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache
+ = CacheBuilder.newBuilder()
+ .maximumSize(8000)
+ .expireAfterAccess(16, TimeUnit.MINUTES)
+ .softValues()
+ .build();
+
+ /**
+ * @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
+ * @param testName Test name.
+ * @param buildTypeId Suite (Build type) id.
+ * @param normalizedBaseBranch Branch name.
+ */
+ @AutoProfiling
+ public IRunHistory getTestRunHist(int srvIdMaskHigh, int testName, int buildTypeId,
+ int normalizedBaseBranch) {
+
+ RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+
+ SuiteHistory hist;
+ try {
+ hist = runHistInMemCache.get(runHistKey,
+ () -> loadSuiteHistory(srvIdMaskHigh, buildTypeId, normalizedBaseBranch));
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+
+ return hist.getTestRunHist(testName);
+ }
+
+
+ /**
+ * Latest actual Build ids supplier. This supplier should handle all equivalent branches in
+ * it.
+ * @param srvId
+ * @param buildTypeId
+ * @param normalizedBaseBranch
+ * @param knownBuilds Known builds.
+ */
+ @AutoProfiling
+ protected Set<Integer> determineLatestBuildsFunction(
+ int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
+ String btId = compactor.getStringFromId(buildTypeId);
+ String branchId = compactor.getStringFromId(normalizedBaseBranch);
+ List<BuildRefCompacted> compacted = buildRefDao.getAllBuildsCompacted(srvId, btId,
+ branchEquivalence.branchForQuery(branchId));
+ long curTs = System.currentTimeMillis();
+ Set<Integer> buildIds = compacted.stream()
+ .filter(b -> !knownBuilds.contains(b.id()))
+ //todo filter queued, cancelled and so on
+ .filter(
+ bRef -> {
+ //todo getAll
+ //todo getStartTime From FatBuild
+ Long startTime = runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
+ if (startTime == null)
+ return false;
+
+ return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
+ }
+ ).map(BuildRefCompacted::id).collect(Collectors.toSet());
+
+ System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+
+ return buildIds;
+ }
+
+ @AutoProfiling
+ public SuiteHistory loadSuiteHistory(int srvId,
+ int buildTypeId,
+ int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
+
+ Set<Integer> buildIds = determineLatestBuildsFunction(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
+
+ HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
+
+ missedBuildsIds.removeAll(suiteRunHist.keySet());
+
+ if (!missedBuildsIds.isEmpty()) {
+ Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
+
+ System.err.println("***** + Adding to persisted history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ addl.size() + " invocations");
+
+ histDao.putAll(srvId, addl);
+ suiteRunHist.putAll(addl);
+ }
+
+ SuiteHistory sumary = new SuiteHistory();
+
+ suiteRunHist.forEach((buildId, suiteInv) -> suiteInv.tests().forEach(sumary::addTestInvocation));
+
+ System.err.println("***** History for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ sumary.size(igniteProvider.get()) + " bytes");
+
+ return sumary;
+ }
+
+ public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
+ Iterable<RunHistKey> objects =
+ stream
+ .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
+ .collect(Collectors.toSet());
+
+ runHistInMemCache.invalidateAll(objects);
+ }
+
+
+
+
+ @AutoProfiling
+ public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
+ HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ fatBuildDao.getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
+ SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
+
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+
+ sinv.addTest(testCompacted.testName(), invocation);
+ }
+ );
+
+ suiteRunHist.put(fatBuildCompacted.id(), sinv);
+ });
+
+ return suiteRunHist;
+ }
+
+ /*
+ @AutoProfiling
+ protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
+ Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
+
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new FatBuildDao.HistoryCollectProcessor(successStatusStrId);
+
+ Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
+
+ SuiteHistory hist = new SuiteHistory();
+
+ map.values().forEach(
+ res -> {
+ if (res == null)
+ return;
+
+ Map<Integer, Invocation> invocationMap = res.get();
+
+ if (invocationMap == null)
+ return;
+
+ invocationMap.forEach((k, v) -> {
+ RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
+ k_ -> new RunHistCompacted());
+
+ compacted.innerAddInvocation(v);
+ });
+
+ }
+ );
+
+ System.err.println("Suite history: tests in scope "
+ + hist.testsHistory.size()
+ + " for " +buildIds.size() + " builds checked"
+ + " size " + hist.size(igniteProvider.get()));
+
+ return hist;
+ }
+ */
+
+
+ private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
+ private final int successStatusStrId;
+
+ public HistoryCollectProcessor(int successStatusStrId) {
+ this.successStatusStrId = successStatusStrId;
+ }
+
+ @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
+ Object... arguments) throws EntryProcessorException {
+ if (entry.getValue() == null)
+ return null;
+
+ Map<Integer, Invocation> hist = new HashMap<>();
+ FatBuildCompacted fatBuildCompacted = entry.getValue();
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+
+ hist.put(testCompacted.testName(), invocation);
+ }
+ );
+
+ return hist;
+ }
+ }
+
+}
[ignite-teamcity-bot] 03/07: Trusted tests & suite history
performance fixes: Storage into TTL persisted cache
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 9a87e9cb54df4c4bf8ddb890cd1297a621ac1208
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Jun 19 20:48:54 2019 +0300
Trusted tests & suite history performance fixes: Storage into TTL persisted cache
---
.../apache/ignite/tcignited/ITeamcityIgnited.java | 6 +--
.../ignite/tcignited/TeamcityIgnitedImpl.java | 19 ++++----
.../ignite/tcignited/TeamcityIgnitedModule.java | 2 +
.../apache/ignite/tcignited/build/FatBuildDao.java | 56 +++++++++++++---------
.../ignite/tcignited/build/SuiteHistory.java | 2 -
.../ignite/tcignited/history/SuiteInvocation.java | 11 +++--
.../history/SuiteInvocationHistoryDao.java | 14 ++++--
7 files changed, 67 insertions(+), 43 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 38120db..f19a4ad 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -210,10 +210,10 @@ public interface ITeamcityIgnited {
* V.3.0 run history implementation based on scan of fat builds.
*
* @param testName Test name.
- * @param suiteName Suite name. Null suite name means suite not found
- * @param branchName Branch name.
+ * @param buildTypeId Suite (Build Type) ID, ID for compactor. Null suite name means suite not found.
+ * @param normalizedBaseBranch Branch name. This branch name does not support branches equivalence, only exact query will work.
*/
- @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName);
+ @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
/**
* @param suiteBuildTypeId Suite id.
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index def3851..707285f 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -433,20 +433,21 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
- @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName,
- @Nullable Integer branchName) {
- if (suiteName == null || branchName == null)
+ @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId,
+ @Nullable Integer normalizedBaseBranch) {
+ if (buildTypeId == null || normalizedBaseBranch == null)
return null;
- if (testName < 0 || suiteName < 0 || branchName < 0)
+ if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0)
return null;
Supplier<Set<Integer>> supplier = () -> {
- String btId = compactor.getStringFromId(suiteName);
- String branchId = compactor.getStringFromId(branchName);
+ String btId = compactor.getStringFromId(buildTypeId);
+ String branchId = compactor.getStringFromId(normalizedBaseBranch);
List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId);
long curTs = System.currentTimeMillis();
- Set<Integer> buildIds = compacted.stream().filter(
+ Set<Integer> buildIds = compacted.stream()
+ .filter(
bRef -> {
Long startTime = getBuildStartTime(bRef.id());
if (startTime == null)
@@ -456,12 +457,12 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
).map(BuildRefCompacted::id).collect(Collectors.toSet());
- System.err.println("Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+ System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
return buildIds;
};
- return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, suiteName, branchName);
+ return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, buildTypeId, normalizedBaseBranch);
}
/** {@inheritDoc} */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
index 707cab8..86cb573 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.tcignited.buildlog.BuildLogProcessorModule;
@@ -65,6 +66,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(MuteDao.class).in(new SingletonScope());
bind(MuteSync.class).in(new SingletonScope());
bind(BuildLogCheckResultDao.class).in(new SingletonScope());
+ bind(SuiteInvocationHistoryDao.class).in(new SingletonScope());
TcRealConnectionModule module = new TcRealConnectionModule();
if (conn != null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 341c3b9..af59fd9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -225,22 +225,21 @@ public class FatBuildDao {
/**
* @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
- * @param buildIdsSupplier Latest actual Build ids supplier.
+ * @param buildIdsSupplier Latest actual Build ids supplier. This supplier should handle all equivalent branches in it.
* @param testName Test name.
- * @param suiteName Suite name.
- * @param branchName Branch name.
+ * @param buildTypeId Suite (Build type) id.
+ * @param normalizedBaseBranch Branch name.
*/
@AutoProfiling
public IRunHistory getTestRunHist(int srvIdMaskHigh,
- Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) {
+ Supplier<Set<Integer>> buildIdsSupplier, int testName, int buildTypeId, int normalizedBaseBranch) {
- RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, suiteName, branchName);
+ RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
SuiteHistory hist;
-
try {
hist = runHistInMemCache.get(runHistKey,
- () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, suiteName, branchName));
+ () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, buildTypeId, normalizedBaseBranch));
}
catch (ExecutionException e) {
throw ExceptionUtil.propagateException(e);
@@ -252,9 +251,9 @@ public class FatBuildDao {
@AutoProfiling
public SuiteHistory loadSuiteHistory(int srvId,
Supplier<Set<Integer>> buildIdsSupplier,
- int suiteName,
- int branchName) {
- Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, suiteName, branchName);// todo RunHistSync.normalizeBranch();
+ int buildTypeId,
+ int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
@@ -262,33 +261,44 @@ public class FatBuildDao {
missedBuildsIds.removeAll(suiteRunHist.keySet());
- if (!missedBuildsIds.isEmpty())
- addSuiteInvocationsToHistory(srvId, suiteRunHist, missedBuildsIds);
+ if (!missedBuildsIds.isEmpty()) {
+ Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
+
+ System.err.println("***** + Adding to persisted history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ addl.size() + " invocations");
+
+ historyDao.putAll(srvId, addl);
+ suiteRunHist.putAll(addl);
+ }
SuiteHistory sumary = new SuiteHistory();
suiteRunHist.forEach((buildId, suiteInv) -> {
suiteInv.tests().forEach((tName, test) -> {
- RunHistCompacted compacted = sumary.testsHistory.computeIfAbsent(tName,
- k_ -> new RunHistCompacted());
-
- compacted.innerAddInvocation(test);
+ sumary.testsHistory.computeIfAbsent(tName,
+ k_ -> new RunHistCompacted()).innerAddInvocation(test);
});
});
+ System.err.println("***** History for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ sumary.size(igniteProvider.get()) + " bytes");
+
return sumary;
}
@AutoProfiling
- public void addSuiteInvocationsToHistory(int srvId, Map<Integer, SuiteInvocation> suiteRunHist,
- HashSet<Integer> missedBuildsIds) {
+ public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
+ HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
- Map<Long, FatBuildCompacted> buildsCacheAll = getAllFatBuilds(srvId, missedBuildsIds);
-
- buildsCacheAll.forEach((buildCacheKey, fatBuildCompacted) -> {
- SuiteInvocation sinv = new SuiteInvocation(fatBuildCompacted, compactor, (k, v) -> false);
+ getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
+ SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
tests.forEach(
@@ -301,6 +311,8 @@ public class FatBuildDao {
suiteRunHist.put(fatBuildCompacted.id(), sinv);
});
+
+ return suiteRunHist;
}
@AutoProfiling
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index e9e9b22..a7af3f8 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -33,8 +33,6 @@ public class SuiteHistory {
/** Tests history: Test name ID->RunHistory */
Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
- Map<Integer, SuiteInvocation> invocationMap = new TreeMap<>();
-
public int size(Ignite ignite) {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
return binary.length();
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
index e2cc0cd..0687a13 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -38,7 +38,7 @@ public class SuiteInvocation {
/** Suite name for queries */
@QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 1)})
- private int suiteName;
+ private int buildTypeId;
/** Teamcity branch name for queries */
@QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)})
@@ -52,10 +52,13 @@ public class SuiteInvocation {
public SuiteInvocation() {}
- public SuiteInvocation(FatBuildCompacted buildCompacted, IStringCompactor comp,
+ public SuiteInvocation(int srvId, int normalizedBaseBranch, FatBuildCompacted buildCompacted, IStringCompactor comp,
BiPredicate<Integer, Integer> filter) {
- buildStartTime = buildCompacted.getStartDateTs();
- suite = buildCompacted.toInvocation(comp, filter);
+ this.srvId = srvId;
+ this.normalizedBranchName = normalizedBaseBranch;
+ this.buildStartTime = buildCompacted.getStartDateTs();
+ this.suite = buildCompacted.toInvocation(comp, filter);
+ this.buildTypeId = buildCompacted.buildTypeId();
}
public void addTest(int testName, Invocation invocation) {
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
index 1cd066a..3b5b450 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
@@ -58,11 +58,11 @@ public class SuiteInvocationHistoryDao {
suiteHistory = ignite.getOrCreateCache(ccfg);
}
- public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int suiteName, int normalizedBranchName) {
+ public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int buildTypeId, int normalizedBranchName) {
java.util.Map<Integer, SuiteInvocation> map = new HashMap<>();
try (QueryCursor<Cache.Entry<Long, SuiteInvocation>> qryCursor = suiteHistory.query(
- new SqlQuery<Long, SuiteInvocation>(SuiteInvocation.class, "srvId = ? and suiteName = ? and normalizedBranchName = ?")
- .setArgs(srvId, suiteName, normalizedBranchName))) {
+ new SqlQuery<Long, SuiteInvocation>(SuiteInvocation.class, "srvId = ? and buildTypeId = ? and normalizedBranchName = ?")
+ .setArgs(srvId, buildTypeId, normalizedBranchName))) {
for (Cache.Entry<Long, SuiteInvocation> next : qryCursor) {
Long key = next.getKey();
@@ -73,4 +73,12 @@ public class SuiteInvocationHistoryDao {
return map;
}
+
+ public void putAll(int srvId, Map<Integer, SuiteInvocation> addl) {
+ Map<Long, SuiteInvocation> data = new HashMap<>();
+
+ addl.forEach((k, v) -> data.put(BuildRefDao.buildIdToCacheKey(srvId, k), v));
+
+ suiteHistory.putAll(data);
+ }
}
[ignite-teamcity-bot] 01/07: Trusted tests & suite history
performance fixes
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit e78c82ebd213425538c66bba6a25ae2ce9286257
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Jun 18 21:45:05 2019 +0300
Trusted tests & suite history performance fixes
---
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 2 +-
.../tcbot/engine/chain/TestCompactedMult.java | 4 ++
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 4 +-
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 3 +-
.../apache/ignite/tcignited/ITeamcityIgnited.java | 2 +-
.../ignite/tcignited/TeamcityIgnitedImpl.java | 13 ++--
.../apache/ignite/tcignited/build/FatBuildDao.java | 57 +++++++++++-----
.../ignite/tcignited/build/SuiteHistory.java | 7 ++
.../ignite/tcignited/history/SuiteInvocation.java | 47 +++++++++++++
.../history/SuiteInvocationHistoryDao.java | 76 ++++++++++++++++++++++
10 files changed, 189 insertions(+), 26 deletions(-)
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index a794110..2207e2a 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -315,7 +315,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(res.values().stream(), 3, comparing).stream();
}
- public List<IMultTestOccurrence> getFailedTests() {
+ public List<TestCompactedMult> getFailedTests() {
Map<Integer, TestCompactedMult> res = new HashMap<>();
builds.forEach(singleBuildRunCtx -> {
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
index 908bc5f..792d5c6 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.history.IRunHistSummary;
@@ -39,6 +40,9 @@ public class TestCompactedMult implements IMultTestOccurrence {
this.compactor = compactor;
}
+ @Nullable public Integer testName() {
+ return occurrences.isEmpty() ? null : occurrences.iterator().next().testName();
+ }
/** {@inheritDoc} */
@Override public String getName() {
return occurrences.isEmpty() ? "" : occurrences.iterator().next().testName(compactor);
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index 7d4b4d3..b3fb4bd 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -258,12 +258,14 @@ public class PrChainsProcessor {
.failedChildSuites()
.map((ctx) -> {
String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch);
+ Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch);
IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(ctx.suiteId(), normalizedBaseBranch);
+ Integer suiteId = compactor.getStringIdIfPresent(ctx.suiteId()); // can be inlined
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
List<DsTestFailureUi> failures = ctx.getFailedTests().stream().map(occurrence -> {
- IRunHistory stat = tcIgnited.getTestRunHist(occurrence.getName(), normalizedBaseBranch);
+ IRunHistory stat = tcIgnited.getTestRunHist(occurrence.testName(), suiteId, baseBranchId);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 8d25cf2..4a5ca06 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -33,6 +33,7 @@ import javax.annotation.Nullable;
import org.apache.ignite.tcbot.common.util.UrlUtil;
import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
+import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
import org.apache.ignite.tcbot.engine.ui.BotUrls.GetBuildLog;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
@@ -187,7 +188,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
webToBuild = buildWebLinkToBuild(tcIgnited, suite);
if (includeTests) {
- List<IMultTestOccurrence> tests = suite.getFailedTests();
+ List<TestCompactedMult> tests = suite.getFailedTests();
Function<IMultTestOccurrence, Float> function = foccur -> {
IRunHistory apply = tcIgnited.getTestRunHist(foccur.getName(), failRateNormalizedBranch);
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 91c4850..51429b9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -209,7 +209,7 @@ public interface ITeamcityIgnited {
* V.3.0 run history implementation based on scan of fat builds.
*
* @param testName Test name.
- * @param suiteName Suite name.
+ * @param suiteName Suite name. Null suite name means suite not found
* @param branchName Branch name.
*/
@Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName);
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index a6e37e5..eb14115 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -32,7 +32,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.OptionalInt;
import java.util.Set;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
@@ -52,8 +51,6 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.tcignited.mute.MuteDao;
-import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
@@ -70,6 +67,8 @@ import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.tcignited.history.RunHistSync;
+import org.apache.ignite.tcignited.mute.MuteDao;
+import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.model.agent.Agent;
@@ -434,11 +433,15 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return runHistCompactedDao.getSuiteRunHist(srvIdMaskHigh, suiteId, branch);
}
- @Nullable @Override
- public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName) {
+ /** {@inheritDoc} */
+ @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName,
+ @Nullable Integer branchName) {
if (suiteName == null || branchName == null)
return null;
+ if (testName < 0 || suiteName < 0 || branchName < 0)
+ return null;
+
Supplier<Set<Integer>> supplier = () -> {
String btId = compactor.getStringFromId(suiteName);
String branchId = compactor.getStringFromId(branchName);
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index ad08fe3..f83521b 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -21,6 +21,7 @@ import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -53,6 +54,8 @@ import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.history.IRunHistory;
+import org.apache.ignite.tcignited.history.SuiteInvocation;
+import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcservice.model.changes.ChangesList;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
@@ -78,13 +81,11 @@ public class FatBuildDao {
/** Builds cache. */
private IgniteCache<Long, FatBuildCompacted> buildsCache;
-
- /** Suite history cache. */
- private IgniteCache<RunHistKey, SuiteHistory> suiteHistory;
-
/** Compactor. */
@Inject private IStringCompactor compactor;
+ @Inject SuiteInvocationHistoryDao historyDao;
+
/**
* Non persistence cache for all suite RunHistory for particular branch.
* RunHistKey(ServerId||BranchId||suiteId)-> Build reference
@@ -103,6 +104,8 @@ public class FatBuildDao {
public FatBuildDao init() {
buildsCache = igniteProvider.get().getOrCreateCache(CacheConfigs.getCacheV2Config(TEAMCITY_FAT_BUILD_CACHE_NAME));
+ historyDao.init();
+
return this;
}
@@ -172,7 +175,7 @@ public class FatBuildDao {
}
/**
- * @param srvIdMaskHigh Server id mask high.
+ * @param srvIdMaskHigh Server id mask to be placed at high bits of the key.
* @param buildId Build id.
*/
public static long buildIdToCacheKey(int srvIdMaskHigh, int buildId) {
@@ -197,10 +200,7 @@ public class FatBuildDao {
public Map<Long, FatBuildCompacted> getAllFatBuilds(int srvIdMaskHigh, Collection<Integer> buildsIds) {
Preconditions.checkNotNull(buildsCache, "init() was not called");
- Set<Long> ids = buildsIds.stream()
- .filter(Objects::nonNull)
- .map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId))
- .collect(Collectors.toSet());
+ Set<Long> ids = buildsIdsToCacheKeys(srvIdMaskHigh, buildsIds);
return buildsCache.getAll(ids);
}
@@ -223,19 +223,37 @@ public class FatBuildDao {
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
+ /**
+ * @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
+ * @param buildIdsSupplier Latest actual Build ids supplier.
+ * @param testName Test name.
+ * @param suiteName Suite name.
+ * @param branchName Branch name.
+ */
public IRunHistory getTestRunHist(int srvIdMaskHigh,
Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) {
-
RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, suiteName, branchName);
- SuiteHistory history;
+ SuiteHistory hist;
try {
- history = runHistInMemCache.get(runHistKey,
+ hist = runHistInMemCache.get(runHistKey,
() -> {
+ Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvIdMaskHigh, suiteName, branchName);// todo RunHistSync.normalizeBranch();
+
Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
+ HashSet<Integer> missedBuilds = new HashSet<>(buildIds);
+
+ missedBuilds.removeAll(suiteRunHist.keySet());
+
+ if (!missedBuilds.isEmpty()) {
+
+ }
+
+ Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
+
return calcSuiteHistory(srvIdMaskHigh, buildIds);
});
}
@@ -243,12 +261,12 @@ public class FatBuildDao {
throw ExceptionUtil.propagateException(e);
}
- return history.testsHistory.get(testName);
+ return hist.testsHistory.get(testName);
}
@AutoProfiling
protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
- Set<Long> cacheKeys = buildIds.stream().map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
+ Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
@@ -259,13 +277,13 @@ public class FatBuildDao {
SuiteHistory hist = new SuiteHistory();
map.values().forEach(
- res-> {
- if(res==null)
+ res -> {
+ if (res == null)
return;
Map<Integer, Invocation> invocationMap = res.get();
- if(invocationMap == null)
+ if (invocationMap == null)
return;
invocationMap.forEach((k, v) -> {
@@ -286,6 +304,11 @@ public class FatBuildDao {
return hist;
}
+ private static Set<Long> buildsIdsToCacheKeys(int srvIdMaskHigh, Collection<Integer> stream) {
+ return stream.stream()
+ .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
+ }
+
@AutoProfiling
protected Set<Integer> determineLatestBuilds(Supplier<Set<Integer>> buildIdsSupplier) {
return buildIdsSupplier.get();
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index 8969736..e9e9b22 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -20,14 +20,21 @@ package org.apache.ignite.tcignited.build;
import java.util.HashMap;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.ignite.Ignite;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.internal.binary.BinaryObjectExImpl;
+import org.apache.ignite.tcignited.history.SuiteInvocation;
+/**
+ * Suite run history summary.
+ */
public class SuiteHistory {
/** Tests history: Test name ID->RunHistory */
Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
+ Map<Integer, SuiteInvocation> invocationMap = new TreeMap<>();
+
public int size(Ignite ignite) {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
return binary.length();
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
new file mode 100644
index 0000000..bd3aa80
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+import java.util.List;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.tcbot.persistence.Persisted;
+
+/**
+ * Shorter verison of FatBuild with less data: created only if run history was required,
+ * has time limitation of MAX_DAYS, may have TTL.
+ */
+@Persisted
+public class SuiteInvocation {
+ /** Server ID for queries */
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 0)})
+ private int srvId;
+
+ /** Suite name for queries */
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 1)})
+ private int suiteName;
+
+ /** Teamcity branch name for queries */
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)})
+ private int normalizedBranchName;
+
+ Invocation suite;
+
+ List<Invocation> tests;
+
+ Long buildStartTime;
+}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
new file mode 100644
index 0000000..1cd066a
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import javax.cache.Cache;
+import javax.cache.expiry.AccessedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.tcbot.persistence.CacheConfigs;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+
+import static java.util.concurrent.TimeUnit.HOURS;
+
+/**
+ * Suite invocation history access object.
+ */
+public class SuiteInvocationHistoryDao {
+ /** Ignite provider. */
+ @Inject
+ private Provider<Ignite> igniteProvider;
+
+ /** Suite history cache. */
+ private IgniteCache<Long, SuiteInvocation> suiteHistory;
+
+ public void init() {
+ CacheConfiguration<Long , SuiteInvocation> ccfg = CacheConfigs.getCacheV2Config("suiteHistory");
+ ccfg.setExpiryPolicyFactory(AccessedExpiryPolicy.factoryOf(new Duration(HOURS, 12)));
+ ccfg.setEagerTtl(true);
+
+ ccfg.setQueryEntities(Collections.singletonList(new QueryEntity(Long.class, SuiteInvocation.class)));
+
+ Ignite ignite = igniteProvider.get();
+
+ suiteHistory = ignite.getOrCreateCache(ccfg);
+ }
+
+ public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int suiteName, int normalizedBranchName) {
+ java.util.Map<Integer, SuiteInvocation> map = new HashMap<>();
+ try (QueryCursor<Cache.Entry<Long, SuiteInvocation>> qryCursor = suiteHistory.query(
+ new SqlQuery<Long, SuiteInvocation>(SuiteInvocation.class, "srvId = ? and suiteName = ? and normalizedBranchName = ?")
+ .setArgs(srvId, suiteName, normalizedBranchName))) {
+
+ for (Cache.Entry<Long, SuiteInvocation> next : qryCursor) {
+ Long key = next.getKey();
+ int buildId = BuildRefDao.cacheKeyToBuildId(key);
+ map.put(buildId, next.getValue());
+ }
+ }
+
+ return map;
+ }
+}