You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/25 14:49:12 UTC
[ignite-teamcity-bot] branch master updated: Trusted tests & suite
history performance fixes: Storage into persisted cache + TTL (#129)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 47f23e7 Trusted tests & suite history performance fixes: Storage into persisted cache + TTL (#129)
47f23e7 is described below
commit 47f23e711abf43de86afcf8a9e4a7150988cb94a
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Jun 25 17:49:07 2019 +0300
Trusted tests & suite history performance fixes: Storage into persisted cache + TTL (#129)
---
build.gradle | 5 +-
ignite-tc-helper-web/build.gradle | 4 -
.../ignite/ci/tcbot/issue/IssueDetector.java | 17 +-
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 46 ++-
jetty-launcher/build.gradle | 4 -
.../tcbot/engine/chain/BuildChainProcessor.java | 50 ++--
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 83 ++++--
.../tcbot/engine/chain/SingleBuildRunCtx.java | 4 +
.../tcbot/engine/chain/TestCompactedMult.java | 25 +-
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 8 +-
.../tracked/TrackedBranchChainsProcessor.java | 7 +-
.../apache/ignite/tcbot/engine/ui/DsChainUi.java | 15 +-
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 62 ++--
.../ignite/tcbot/engine/ui/DsTestFailureUi.java | 39 ++-
.../ignite/tcbot/engine/ui/DsTestHistoryUi.java | 2 +-
tcbot-server-node/build.gradle | 3 -
.../apache/ignite/tcignited/ITeamcityIgnited.java | 10 +-
.../ignite/tcignited/TeamcityIgnitedImpl.java | 83 +++---
.../ignite/tcignited/TeamcityIgnitedModule.java | 4 +
.../apache/ignite/tcignited/build/FatBuildDao.java | 155 +++-------
.../tcignited/build/ProactiveFatBuildSync.java | 3 +-
.../ignite/tcignited/build/SuiteHistory.java | 39 ++-
.../BranchEquivalence.java} | 30 +-
.../ignite/tcignited/buildref/BuildRefDao.java | 2 +-
.../ignite/tcignited/history/HistoryCollector.java | 326 +++++++++++++++++++++
.../ISuiteRunHistory.java} | 21 +-
.../tcignited/history/RunHistCompactedDao.java | 36 +++
.../ignite/tcignited/history/RunHistSync.java | 2 +
.../ignite/tcignited/history/SuiteInvocation.java | 77 +++++
.../history/SuiteInvocationHistoryDao.java | 87 ++++++
31 files changed, 919 insertions(+), 332 deletions(-)
diff --git a/build.gradle b/build.gradle
index edf7b84..43f7f5a 100644
--- a/build.gradle
+++ b/build.gradle
@@ -38,10 +38,11 @@ subprojects {
}
allprojects {
+ apply plugin: 'java'
+ sourceCompatibility = '1.8'
+ targetCompatibility = '1.8'
ext {
- sourceCompatibility = '1.8'
- targetCompatibility = '1.8'
jettyVer = '9.4.12.v20180830'
diff --git a/ignite-tc-helper-web/build.gradle b/ignite-tc-helper-web/build.gradle
index a9cb41f..e4fa011 100644
--- a/ignite-tc-helper-web/build.gradle
+++ b/ignite-tc-helper-web/build.gradle
@@ -18,10 +18,6 @@
apply plugin: 'java'
apply plugin: 'war'
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
-
// https://www.apache.org/legal/resolved.html#category-a
dependencies {
compile (project(":tcbot-common"));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index e3800a7..62842f0 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -261,13 +261,14 @@ public class IssueDetector {
final String trackedBranch = res.getTrackedBranch();
+ String suiteId = suiteCurrentStatus.suiteId;
for (DsTestFailureUi testFailure : suiteCurrentStatus.testFailures) {
- if (registerTestFailIssues(tcIgnited, srvCode, normalizeBranch, testFailure, trackedBranch,
+ if (registerTestFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, testFailure, trackedBranch,
suiteCurrentStatus.tags))
newIssues++;
}
- if (registerSuiteFailIssues(tcIgnited, srvCode, normalizeBranch, suiteCurrentStatus, trackedBranch))
+ if (registerSuiteFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, suiteCurrentStatus, trackedBranch))
newIssues++;
}
}
@@ -286,13 +287,15 @@ public class IssueDetector {
*/
private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
+ String suiteId,
String normalizeBranch,
DsSuiteUi suiteFailure,
String trackedBranch) {
- String suiteId = suiteFailure.suiteId;
+ Integer btId = compactor.getStringIdIfPresent(suiteId);
+ Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch);
- IRunHistory runStat = tcIgnited.getSuiteRunHist(suiteId, normalizeBranch);
+ IRunHistory runStat = tcIgnited.getSuiteRunHist(btId, brNormId).self();
if (runStat == null)
return false;
@@ -361,13 +364,17 @@ public class IssueDetector {
private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
+ String suiteId,
String normalizeBranch,
DsTestFailureUi testFailure,
String trackedBranch,
@Nonnull Set<String> suiteTags) {
String name = testFailure.name;
+ int tname = compactor.getStringId(name);
+ Integer btId = compactor.getStringIdIfPresent(suiteId);
+ Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch);
- IRunHistory runStat = tcIgnited.getTestRunHist(name, normalizeBranch);
+ IRunHistory runStat = tcIgnited.getTestRunHist(tname, btId, brNormId);
if (runStat == null)
return false;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index dcc646e..c1bd22d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
/** TC Bot Version. */
- public static final String VERSION = "20190613";
+ public static final String VERSION = "20190621";
/** Java version, where Web App is running. */
public String javaVer;
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
index 4f9eacd..9a0d174 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
@@ -31,6 +31,7 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.SyncMode;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.jetbrains.annotations.NotNull;
import org.mockito.Mockito;
@@ -90,10 +91,10 @@ public class TeamcityIgnitedMock {
.collect(Collectors.toList());
});
- when(tcIgnited.getTestRunHist(anyString(), anyString()))
+ when(tcIgnited.getTestRunHist(anyInt(), anyInt(), anyInt()))
.thenAnswer((inv) -> {
- final String name = inv.getArgument(0);
- final String branch = inv.getArgument(1);
+ final Integer tstName = inv.getArgument(0);
+ final Integer branchId = inv.getArgument(2);
// System.out.println("Search history " + name + " in " + branch + ": " );
if (histCache.isEmpty()) {
@@ -103,11 +104,9 @@ public class TeamcityIgnitedMock {
}
}
- final Integer tstName = c.getStringIdIfPresent(name);
if (tstName == null)
return null;
- final Integer branchId = c.getStringIdIfPresent(branch);
if (branchId == null)
return null;
@@ -115,11 +114,46 @@ public class TeamcityIgnitedMock {
final RunHistCompacted runHistCompacted = histCache.get(key);
- System.out.println("Test history " + name + " in " + branch + " => " + runHistCompacted);
+ System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted);
return runHistCompacted;
});
+ when(tcIgnited.getSuiteRunHist(anyInt(), anyInt()))
+ .thenAnswer((inv) -> {
+ final Integer suiteName = inv.getArgument(0);
+ final Integer branchId = inv.getArgument(1);
+ // System.out.println("Search history " + name + " in " + branch + ": " );
+ if (histCache.isEmpty()) {
+ synchronized (histCache) {
+ if (histCache.isEmpty())
+ initHistory(c, histCache, builds, srvId);
+ }
+ }
+
+ ISuiteRunHistory mock = Mockito.mock(ISuiteRunHistory.class);
+
+ when(mock.getTestRunHist(anyInt())).thenAnswer((inv2)-> {
+ final Integer tstName = inv2.getArgument(0);
+
+ if (tstName == null)
+ return null;
+
+ if (branchId == null)
+ return null;
+
+ final RunHistKey key = new RunHistKey(srvId, tstName, branchId);
+
+ final RunHistCompacted runHistCompacted = histCache.get(key);
+
+ System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted);
+
+ return runHistCompacted;
+ });
+
+ return mock;
+ });
+
// when(tcIgnited.gitBranchPrefix()).thenReturn("ignite-");
ITcServerConfig mock = mock(ITcServerConfig.class);
diff --git a/jetty-launcher/build.gradle b/jetty-launcher/build.gradle
index edfb84d..5df3d7b 100644
--- a/jetty-launcher/build.gradle
+++ b/jetty-launcher/build.gradle
@@ -18,10 +18,6 @@
apply plugin: 'java'
apply plugin: 'application'
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
-
mainClassName = 'org.apache.ignite.ci.TcHelperJettyLauncher'
applicationDefaultJvmArgs = ["-Dteamcity.helper.home=../work",
"-Dteamcity.bot.regionsize=16", // 16g Durable Memory region
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index 62bb209..579dbb0 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -19,12 +19,32 @@ package org.apache.ignite.tcbot.engine.chain;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.Futures;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.util.FutureUtil;
import org.apache.ignite.tcbot.engine.pool.TcUpdatePool;
-import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.engine.ui.LrTestUi;
import org.apache.ignite.tcbot.engine.ui.LrTestsSuiteSummaryUi;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
@@ -38,18 +58,6 @@ import org.apache.ignite.tcservice.model.result.Build;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-
/**
* Process whole Build Chain, E.g. runAll at particular server, including all builds involved
*/
@@ -144,6 +152,8 @@ public class BuildChainProcessor {
if (entryPoints.isEmpty())
return new FullChainRunCtx(Build.createFakeStub());
+ Integer failRateBranchId = compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch));
+
Map<Integer, Future<FatBuildCompacted>> builds = loadAllBuildsInChains(entryPoints, mode, tcIgn);
Map<String, List<Future<FatBuildCompacted>>> freshRebuilds = new ConcurrentHashMap<>();
@@ -177,6 +187,11 @@ public class BuildChainProcessor {
buildsForSuite.forEach(buildCompacted -> ctx.addBuild(loadChanges(buildCompacted, tcIgn)));
+ //ask for history for the suite in parallel
+ tcUpdatePool.getService().submit(() -> {
+ ctx.history(tcIgn, failRateBranchId);
+ });
+
analyzeTests(ctx, tcIgn, procLog);
fillBuildCounts(ctx, tcIgn, includeScheduledInfo);
@@ -185,9 +200,7 @@ public class BuildChainProcessor {
});
Function<MultBuildRunCtx, Float> function = ctx -> {
-
- //todo cache RunStat instance into suite context to compare
- IRunHistory runStat = tcIgn.getSuiteRunHist(ctx.suiteId(), RunHistSync.normalizeBranch(failRateBranch));
+ IRunHistory runStat = ctx.history(tcIgn, failRateBranchId);
if (runStat == null)
return 0f;
@@ -230,7 +243,8 @@ public class BuildChainProcessor {
.peek(val -> Preconditions.checkNotNull(val, "Build future should be in context"))
.flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()).collect(Collectors.toSet());
- logger.info("Level [" + level + "] dependencies:" + depsNextLevel);
+ if(logger.isDebugEnabled())
+ logger.debug("Level [" + level + "] dependencies:" + depsNextLevel);
remainedUnloadedDeps = depsNextLevel;
}
@@ -330,7 +344,7 @@ public class BuildChainProcessor {
protected void fillBuildCounts(MultBuildRunCtx outCtx,
ITeamcityIgnited teamcityIgnited, boolean includeScheduledInfo) {
if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) {
- final List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.buildTypeId(), outCtx.branchName());
+ List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.suiteId(), outCtx.branchName());
long cntRunning = runAllBuilds
.stream()
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index a794110..f965db3 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -17,7 +17,9 @@
package org.apache.ignite.tcbot.engine.chain;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.cache.CacheBuilder;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -28,23 +30,25 @@ import java.util.Optional;
import java.util.OptionalDouble;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
+import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.util.CollectionUtil;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcignited.history.IRunHistory;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
@@ -67,6 +71,9 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();
+ private final com.google.common.cache.Cache<Integer, Optional<ISuiteRunHistory>> historyCacheMap
+ = CacheBuilder.newBuilder().build();
+
public void addBuild(SingleBuildRunCtx ctx) {
builds.add(ctx);
}
@@ -98,8 +105,9 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().map(SingleBuildRunCtx::getTestLogCheckResult).filter(Objects::nonNull);
}
- public String suiteId() {
- return firstBuildInfo.suiteId();
+ /** {@inheritDoc} */
+ @Override public String suiteId() {
+ return firstBuild().map(SingleBuildRunCtx::suiteId).orElse(null);
}
/** {@inheritDoc} */
@@ -107,9 +115,6 @@ public class MultBuildRunCtx implements ISuiteResults {
return getBuildMessageProblemCount() > 0;
}
- public String buildTypeId() {
- return firstBuildInfo.buildTypeId;
- }
public boolean hasAnyBuildProblemExceptTestOrSnapshot() {
return allProblemsInAllBuilds()
@@ -170,6 +175,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().filter(ISuiteResults::hasCompilationProblem).count();
}
+ /** {@inheritDoc} */
public boolean hasTimeoutProblem() {
return getExecutionTimeoutCount() > 0;
}
@@ -178,6 +184,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().filter(SingleBuildRunCtx::hasTimeoutProblem).count();
}
+ /** {@inheritDoc} */
public boolean hasJvmCrashProblem() {
return getJvmCrashProblemCount() > 0;
}
@@ -186,10 +193,12 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsCntHavingBuildProblem(ProblemOccurrence.TC_JVM_CRASH);
}
+ /** {@inheritDoc} */
public boolean hasOomeProblem() {
return getOomeProblemCount() > 0;
}
+ /** {@inheritDoc} */
@Override public boolean hasExitCodeProblem() {
return getExitCodeProblemsCount() > 0;
}
@@ -303,8 +312,8 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(logSizeBytes.entrySet().stream(), 3, comparing).stream();
}
- public Stream<? extends IMultTestOccurrence> getTopLongRunning() {
- Comparator<IMultTestOccurrence> comparing = Comparator.comparing(IMultTestOccurrence::getAvgDurationMs);
+ public Stream<TestCompactedMult> getTopLongRunning() {
+ Comparator<TestCompactedMult> comparing = Comparator.comparing(TestCompactedMult::getAvgDurationMs);
Map<Integer, TestCompactedMult> res = new HashMap<>();
@@ -315,7 +324,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(res.values().stream(), 3, comparing).stream();
}
- public List<IMultTestOccurrence> getFailedTests() {
+ public List<TestCompactedMult> getFailedTests() {
Map<Integer, TestCompactedMult> res = new HashMap<>();
builds.forEach(singleBuildRunCtx -> {
@@ -327,7 +336,7 @@ public class MultBuildRunCtx implements ISuiteResults {
public void saveToMap(Map<Integer, TestCompactedMult> res, Stream<TestCompacted> tests) {
tests.forEach(testCompacted -> {
- res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor))
+ res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor, this))
.add(testCompacted);
});
}
@@ -607,17 +616,13 @@ public class MultBuildRunCtx implements ISuiteResults {
//todo can cache mult occurrences in ctx
builds.forEach(singleBuildRunCtx -> {
- saveToMap(res, singleBuildRunCtx.getAllTests()
- .filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
+ saveToMap(res,
+ singleBuildRunCtx.getAllTests().filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
- Integer suiteName = compactor.getStringIdIfPresent( buildTypeId());
-
- // res.clear(); //todo enable feature back
- //todo can cache fail rate in mult occur
- res.keySet().forEach((testNameId) -> {
- IRunHistory stat = tcIgnited.getTestRunHist(testNameId, suiteName, branchName);
+ res.forEach((testNameId, compactedMult) -> {
+ IRunHistory stat = compactedMult.history(tcIgnited, branchName);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
boolean b = testBlockerComment != null;
if (b) // this test will be considered as blocker if will fail
@@ -627,4 +632,46 @@ public class MultBuildRunCtx implements ISuiteResults {
return trustedCnt.get();
}
+ /**
+ * Returns suite name non compacted.
+ */
+ public Integer buildTypeIdId() {
+ return firstBuild().map(SingleBuildRunCtx::buildTypeIdId).orElse(null);
+ }
+
+ public Optional<SingleBuildRunCtx> firstBuild() {
+ return builds.stream().findFirst();
+ }
+
+ /**
+ * @param tcIgn Tc ign.
+ * @param baseBranchId Base branch id.
+ */
+ public IRunHistory history(ITeamcityIgnited tcIgn, Integer baseBranchId) {
+ if (baseBranchId == null)
+ return null;
+
+ ISuiteRunHistory suiteHist = suiteHist(tcIgn, baseBranchId);
+ if (suiteHist == null)
+ return null;
+
+ return suiteHist.self();
+ }
+
+ @Nullable
+ ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, Integer baseBranchId) {
+ Integer buildTypeIdId = buildTypeIdId();
+ Preconditions.checkNotNull(buildTypeIdId, "Build type ID should be filled");
+
+ try {
+ return historyCacheMap.get(baseBranchId,
+ () -> {
+ return Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId, baseBranchId));
+ })
+ .orElse(null);
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
index dab646e..e41d65f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java
@@ -346,4 +346,8 @@ public class SingleBuildRunCtx implements ISuiteResults {
public int totalNotMutedTests() {
return buildCompacted.totalNotMutedTests();
}
+
+ public int buildTypeIdId() {
+ return buildCompacted.buildTypeId();
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
index 908bc5f..7d5d3c3 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
@@ -21,10 +21,14 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.history.IRunHistSummary;
+import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
/**
@@ -33,12 +37,17 @@ import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
public class TestCompactedMult implements IMultTestOccurrence {
private final List<TestCompacted> occurrences = new ArrayList<>();
private IStringCompactor compactor;
+ private MultBuildRunCtx ctx;
private long avgDuration = -1;
- public TestCompactedMult(IStringCompactor compactor) {
+ public TestCompactedMult(IStringCompactor compactor, MultBuildRunCtx ctx) {
this.compactor = compactor;
+ this.ctx = ctx;
}
+ @Nullable public Integer testName() {
+ return occurrences.isEmpty() ? null : occurrences.iterator().next().testName();
+ }
/** {@inheritDoc} */
@Override public String getName() {
return occurrences.isEmpty() ? "" : occurrences.iterator().next().testName(compactor);
@@ -109,4 +118,18 @@ public class TestCompactedMult implements IMultTestOccurrence {
public void add(TestCompacted next) {
occurrences.add(next);
}
+
+
+ public IRunHistory history(ITeamcityIgnited ignited, Integer baseBranchId) {
+ Integer name = testName();
+ if (name == null)
+ return null;
+
+ ISuiteRunHistory suiteRunHist = ctx.suiteHist(ignited, baseBranchId);
+
+ if (suiteRunHist == null)
+ return null;
+
+ return suiteRunHist.getTestRunHist(name);
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index 7d4b4d3..5a9246c 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -258,19 +258,19 @@ public class PrChainsProcessor {
.failedChildSuites()
.map((ctx) -> {
String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch);
- IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(ctx.suiteId(), normalizedBaseBranch);
+ Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch);
+ IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId);
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
List<DsTestFailureUi> failures = ctx.getFailedTests().stream().map(occurrence -> {
- IRunHistory stat = tcIgnited.getTestRunHist(occurrence.getName(), normalizedBaseBranch);
-
+ IRunHistory stat = occurrence.history(tcIgnited, baseBranchId);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
if (!Strings.isNullOrEmpty(testBlockerComment)) {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch);
+ failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch, baseBranchId);
return failure;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
index bda0b02..a0b40af 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
@@ -22,18 +22,17 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;
-
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx;
import org.apache.ignite.tcbot.engine.chain.LatestRebuildMode;
import org.apache.ignite.tcbot.engine.chain.ProcessLogsMode;
-import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
-import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
+import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
@@ -107,7 +106,7 @@ public class TrackedBranchChainsProcessor {
boolean includeScheduled = buildResMergeCnt == 1;
final FullChainRunCtx ctx = chainProc.loadFullChainContext(
- tcIgnited,
+ tcIgnited,
chains,
rebuild,
logs,
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
index 913c08a..d8c8746 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
@@ -24,13 +24,12 @@ import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;
import javax.annotation.Nullable;
-
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.tcbot.common.util.CollectionUtil;
import org.apache.ignite.tcbot.common.util.UrlUtil;
import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
-import org.apache.ignite.tcbot.common.util.CollectionUtil;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcservice.model.conf.BuildType;
@@ -208,18 +207,18 @@ public class DsChainUi {
webToHist = buildWebLink(tcIgnited, ctx);
webToBuild = buildWebLinkToBuild(tcIgnited, ctx);
- Stream<T2<MultBuildRunCtx, IMultTestOccurrence>> allLongRunning = ctx.suites().flatMap(
+ Stream<T2<MultBuildRunCtx, TestCompactedMult>> allLongRunning = ctx.suites().flatMap(
suite -> suite.getTopLongRunning().map(t -> new T2<>(suite, t))
);
- Comparator<T2<MultBuildRunCtx, IMultTestOccurrence>> durationComp
+ Comparator<T2<MultBuildRunCtx, TestCompactedMult>> durationComp
= Comparator.comparing((pair) -> pair.get2().getAvgDurationMs());
CollectionUtil.top(allLongRunning, 3, durationComp).forEach(
pairCtxAndOccur -> {
MultBuildRunCtx suite = pairCtxAndOccur.get1();
- IMultTestOccurrence longRunningOccur = pairCtxAndOccur.get2();
+ TestCompactedMult longRunningOccur = pairCtxAndOccur.get2();
- DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, longRunningOccur, baseBranchTc);
+ DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, longRunningOccur, baseBranchTc);
failure.testName = "[" + suite.suiteName() + "] " + failure.testName; //may be separate field
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 8d25cf2..eea9212 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -29,19 +29,18 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.tcbot.common.util.UrlUtil;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
+import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
import org.apache.ignite.tcbot.engine.ui.BotUrls.GetBuildLog;
-import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
-import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
+import org.apache.ignite.tcignited.history.IRunHistory;
-import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable;
+import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
@@ -164,9 +163,12 @@ public class DsSuiteUi extends DsHistoryStatUi {
name = suite.suiteName();
String failRateNormalizedBranch = normalizeBranch(baseBranch);
+ Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch);
+
String curBranchNormalized = normalizeBranch(suite.branchName());
+ Integer curBranchId = compactor.getStringIdIfPresent(curBranchNormalized);
- IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId());
+ IRunHistory baseBranchHist = initSuiteStat(tcIgnited, baseBranchId, curBranchId, suite);
Set<String> collect = suite.lastChangeUsers().collect(Collectors.toSet());
@@ -186,26 +188,28 @@ public class DsSuiteUi extends DsHistoryStatUi {
webToHistBaseBranch = buildWebLink(tcIgnited, suite, baseBranch);
webToBuild = buildWebLinkToBuild(tcIgnited, suite);
+ Integer buildTypeIdId = suite.buildTypeIdId();
if (includeTests) {
- List<IMultTestOccurrence> tests = suite.getFailedTests();
- Function<IMultTestOccurrence, Float> function = foccur -> {
- IRunHistory apply = tcIgnited.getTestRunHist(foccur.getName(), failRateNormalizedBranch);
+ List<TestCompactedMult> tests = suite.getFailedTests();
+ Function<TestCompactedMult, Float> function = testCompactedMult -> {
+ IRunHistory res = testCompactedMult.history(tcIgnited, baseBranchId);
- return apply == null ? 0f : apply.getFailRate();
+ return res == null ? 0f : res.getFailRate();
};
tests.sort(Comparator.comparing(function).reversed());
tests.forEach(occurrence -> {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), baseBranch);
- failure.initStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized);
+ failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(),
+ suite.branchName(), baseBranch, baseBranchId);
+ failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, curBranchId);
testFailures.add(failure);
});
suite.getTopLongRunning().forEach(occurrence -> {
- final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, occurrence, baseBranch);
+ final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, occurrence, baseBranch);
topLongRunning.add(failure);
});
@@ -260,13 +264,10 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
private IRunHistory initSuiteStat(ITeamcityIgnited tcIgnited,
- String failRateNormalizedBranch,
- String curBranchNormalized,
- String suiteId) {
- if (Strings.isNullOrEmpty(suiteId))
- return null;
-
- final IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(suiteId, failRateNormalizedBranch);
+ Integer failRateNormalizedBranch,
+ Integer curBranchNormalized,
+ MultBuildRunCtx suite) {
+ IRunHistory statInBaseBranch = suite.history(tcIgnited, failRateNormalizedBranch);
if (statInBaseBranch != null) {
failures = statInBaseBranch.getFailuresCount();
@@ -285,9 +286,8 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
IRunHistory latestRunsSrc = null;
- if (!failRateNormalizedBranch.equals(curBranchNormalized)) {
-
- final IRunHistory statForStripe = tcIgnited.getSuiteRunHist(suiteId, curBranchNormalized);
+ if (!Objects.equals(failRateNormalizedBranch, curBranchNormalized)) {
+ IRunHistory statForStripe = suite.history(tcIgnited, curBranchNormalized);
latestRunsSrc = statForStripe;
latestRuns = statForStripe != null ? statForStripe.getLatestRunResults() : null;
@@ -315,16 +315,18 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
@Nonnull public static DsTestFailureUi createOrrucForLongRun(ITeamcityIgnited tcIgnited,
- @Nonnull MultBuildRunCtx suite,
- final IMultTestOccurrence occurrence,
- @Nullable final String failRateBranch) {
+ IStringCompactor compactor, @Nonnull MultBuildRunCtx suite,
+ final TestCompactedMult occurrence,
+ @Nullable final String failRateBranch) {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), failRateBranch);
+ Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(failRateBranch));
+ Integer buildTypeIdId = suite.buildTypeIdId();
+ failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(),
+ failRateBranch, baseBranchId);
- failure.initStat(tcIgnited,
- normalizeBranch(failRateBranch),
- normalizeBranch(suite.branchName()));
+ failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId,
+ compactor.getStringIdIfPresent(normalizeBranch(suite.branchName())));
return failure;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
index b0cb5d9..6f9f127 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
@@ -25,17 +25,15 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.tcbot.common.util.UrlUtil;
-import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildlog.LogMsgToWarn;
import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcignited.ITeamcityIgnited;
-import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable;
+import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
@@ -94,14 +92,16 @@ public class DsTestFailureUi {
* @param failure test ocurrence (probably multiple)
* @param tcIgn Teamcity.
* @param projectId project ID.
- * @param branchName
- * @param baseBranchName base branch name (e.g. master).
+ * @param branchName current branch name.
+ * @param baseBranchName base branch name (e.g. master), without normalization.
+ * @param baseBranchId Normalized base branch ID (from compactor).
*/
- public void initFromOccurrence(@Nonnull final IMultTestOccurrence failure,
+ public void initFromOccurrence(@Nonnull final TestCompactedMult failure,
@Nonnull final ITeamcityIgnited tcIgn,
@Nullable final String projectId,
@Nullable final String branchName,
- @Nullable final String baseBranchName) {
+ @Nullable final String baseBranchName,
+ Integer baseBranchId) {
name = failure.getName();
investigated = failure.isInvestigated();
curFailures = failure.failuresCount();
@@ -144,8 +144,7 @@ public class DsTestFailureUi {
webUrlBaseBranch = buildWebLink(tcIgn, full.test.id, projectId, baseBranchName);
});
- final IRunHistory stat = tcIgn.getTestRunHist(name, normalizeBranch(baseBranchName));
-
+ final IRunHistory stat = failure.history(tcIgn, baseBranchId);
blockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
}
@@ -191,22 +190,22 @@ public class DsTestFailureUi {
}
/**
+ * @param occurrence
+ * @param buildTypeIdId
* @param tcIgnited TC service as Run stat supplier.
- * @param failRateNormalizedBranch Base branch: Fail rate and flakyness detection normalized branch.
+ * @param baseBranchId Base branch: Fail rate and flakyness detection normalized branch.
* @param curBranchNormalized Cur branch normalized.
*/
- public void initStat(ITeamcityIgnited tcIgnited,
- String failRateNormalizedBranch,
- String curBranchNormalized) {
-
- final IRunHistory stat = tcIgnited.getTestRunHist(name, failRateNormalizedBranch);
-
+ public void initStat(TestCompactedMult occurrence, Integer buildTypeIdId, ITeamcityIgnited tcIgnited,
+ @Nullable Integer baseBranchId,
+ @Nullable Integer curBranchNormalized) {
+ final IRunHistory stat = occurrence.history(tcIgnited, baseBranchId);
histBaseBranch.init(stat);
- IRunHistory statForProblemsDetection = null;
+ IRunHistory statForProblemsDetection;
- if (!curBranchNormalized.equals(failRateNormalizedBranch)) {
- statForProblemsDetection = tcIgnited.getTestRunHist(name, curBranchNormalized);
+ if (!Objects.equals(curBranchNormalized, baseBranchId)) {
+ statForProblemsDetection = occurrence.history(tcIgnited, curBranchNormalized);
if (statForProblemsDetection != null) {
histCurBranch = new DsTestHistoryUi();
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
index 8b8597f..e0fe975 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java
@@ -27,7 +27,7 @@ import org.apache.ignite.tcignited.history.IRunHistory;
*/
public class DsTestHistoryUi {
/** 'All the time' runs history statistic. */
- public DsHistoryStatUi allTime = new DsHistoryStatUi();
+ @Deprecated public DsHistoryStatUi allTime = new DsHistoryStatUi();
/** Latest runs history statistic. */
public DsHistoryStatUi recent = new DsHistoryStatUi();
diff --git a/tcbot-server-node/build.gradle b/tcbot-server-node/build.gradle
index 3ffcf74..0bc43f8 100644
--- a/tcbot-server-node/build.gradle
+++ b/tcbot-server-node/build.gradle
@@ -18,9 +18,6 @@
apply plugin: 'java'
apply plugin: 'application'
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
repositories {
mavenCentral()
mavenLocal()
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 91c4850..5ffd315 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -33,6 +33,7 @@ import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.agent.Agent;
import org.apache.ignite.tcservice.model.mute.MuteInfo;
import org.apache.ignite.tcservice.model.result.Build;
@@ -200,19 +201,22 @@ public interface ITeamcityIgnited {
*/
public BuildTypeCompacted getBuildType(String buildTypeId);
+ @Deprecated
@Nullable public IRunHistory getTestRunHist(String testName, @Nullable String branch);
+ @Deprecated
@Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch);
+ @Nullable public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
/**
* V.3.0 run history implementation based on scan of fat builds.
*
* @param testName Test name.
- * @param suiteName Suite name.
- * @param branchName Branch name.
+ * @param buildTypeId Suite (Build Type) ID, ID for compactor. Null suite name means suite not found.
+ * @param normalizedBaseBranch Branch name. This branch name does not support branches equivalence, only exact query will work.
*/
- @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName);
+ @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
/**
* @param suiteBuildTypeId Suite id.
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index a6e37e5..ea68f59 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -16,10 +16,8 @@
*/
package org.apache.ignite.tcignited;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.io.File;
-import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -32,10 +30,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.OptionalInt;
import java.util.Set;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -52,9 +48,6 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.tcignited.mute.MuteDao;
-import org.apache.ignite.tcignited.mute.MuteSync;
-import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
@@ -64,13 +57,18 @@ import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
+import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.buildref.BuildRefSync;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.tcignited.history.RunHistSync;
-import org.apache.ignite.tcservice.ITeamcity;
+import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
+import org.apache.ignite.tcignited.mute.MuteDao;
+import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.model.agent.Agent;
import org.apache.ignite.tcservice.model.conf.Project;
@@ -95,11 +93,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Max build id diff to enforce reload during incremental refresh. */
public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000;
- /** Default synonyms. */
- private static final List<String> DEFAULT_SYNONYMS
- = Collections.unmodifiableList(
- Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER));
-
/** Server (service) code. */
private String srvCode;
@@ -151,11 +144,20 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Run history sync. */
@Inject private RunHistSync runHistSync;
- @Inject private BuildLogCheckResultDao logCheckResultDao;
+ /** Logger check result DAO. */
+ @Inject private BuildLogCheckResultDao logCheckResDao;
+
+ /** History DAO. */
+ @Inject private SuiteInvocationHistoryDao histDao;
+
+ /** History collector. */
+ @Inject private HistoryCollector histCollector;
/** Strings compactor. */
@Inject private IStringCompactor compactor;
+ @Inject private BranchEquivalence branchEquivalence;
+
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
@@ -172,7 +174,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
changesDao.init();
runHistCompactedDao.init();
muteDao.init();
- logCheckResultDao.init();
+ logCheckResDao.init();
+ histDao.init();
}
/**
@@ -352,7 +355,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Nullable String branchName) {
ensureActualizeRequested();
- return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchForQuery(branchName));
+ return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchEquivalence.branchForQuery(branchName));
}
/** {@inheritDoc} */
@@ -365,7 +368,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (stateQueuedId == null)
return Collections.emptyList();
- Set<Integer> branchNameIds = branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str))
+ Set<Integer> branchNameIds = branchEquivalence.branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str))
.filter(Objects::nonNull).collect(Collectors.toSet());
List<BuildRefCompacted> res = new ArrayList<>();
@@ -422,7 +425,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** {@inheritDoc} */
@Nullable
- @AutoProfiling
@Override public IRunHistory getTestRunHist(String testName, @Nullable String branch) {
return runHistCompactedDao.getTestRunHist(srvIdMaskHigh, testName, branch);
}
@@ -434,32 +436,27 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return runHistCompactedDao.getSuiteRunHist(srvIdMaskHigh, suiteId, branch);
}
- @Nullable @Override
- public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName) {
- if (suiteName == null || branchName == null)
+ /** {@inheritDoc} */
+ @Nullable @Override public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch) {
+ if (buildTypeId == null || normalizedBaseBranch == null)
return null;
- Supplier<Set<Integer>> supplier = () -> {
- String btId = compactor.getStringFromId(suiteName);
- String branchId = compactor.getStringFromId(branchName);
- List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId);
- long curTs = System.currentTimeMillis();
- Set<Integer> buildIds = compacted.stream().filter(
- bRef -> {
- Long startTime = getBuildStartTime(bRef.id());
- if (startTime == null)
- return false;
+ if (buildTypeId < 0 || normalizedBaseBranch < 0)
+ return null;
- return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
- }
- ).map(BuildRefCompacted::id).collect(Collectors.toSet());
+ return histCollector.getSuiteRunHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+ }
- System.err.println("Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+ /** {@inheritDoc} */
+ @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId,
+ @Nullable Integer normalizedBaseBranch) {
+ if (buildTypeId == null || normalizedBaseBranch == null)
+ return null;
- return buildIds;
- };
+ if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0)
+ return null;
- return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, suiteName, branchName);
+ return histCollector.getTestRunHist(srvIdMaskHigh, testName, buildTypeId, normalizedBaseBranch);
}
/** {@inheritDoc} */
@@ -504,13 +501,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return buildTypeDao.getFatBuildType(srvIdMaskHigh, buildTypeId);
}
- public List<String> branchForQuery(@Nullable String branchName) {
- if (ITeamcity.DEFAULT.equals(branchName))
- return DEFAULT_SYNONYMS;
- else
- return Collections.singletonList(branchName);
- }
-
/**
* Enables scheduling for build refs/builds/history sync
*/
@@ -522,7 +512,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
// schedule find missing later
fatBuildSync.ensureActualizationRequested(srvCode, conn);
- runHistSync.invokeLaterFindMissingHistory(srvCode);
+ //todo remove unused code
+ // runHistSync.invokeLaterFindMissingHistory(srvCode);
}
/** {@inheritDoc} */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
index 707cab8..bc0b203 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
@@ -29,6 +29,8 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.history.HistoryCollector;
+import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.tcignited.buildlog.BuildLogProcessorModule;
@@ -65,6 +67,8 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(MuteDao.class).in(new SingletonScope());
bind(MuteSync.class).in(new SingletonScope());
bind(BuildLogCheckResultDao.class).in(new SingletonScope());
+ bind(SuiteInvocationHistoryDao.class).in(new SingletonScope());
+ bind(HistoryCollector.class).in(new SingletonScope());
TcRealConnectionModule module = new TcRealConnectionModule();
if (conn != null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index ad08fe3..b40e204 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -18,16 +18,13 @@
package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Iterables;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -41,23 +38,18 @@ import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.CacheEntryProcessor;
-import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
-import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.tcignited.history.IRunHistory;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcservice.model.changes.ChangesList;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
-import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -78,24 +70,11 @@ public class FatBuildDao {
/** Builds cache. */
private IgniteCache<Long, FatBuildCompacted> buildsCache;
-
- /** Suite history cache. */
- private IgniteCache<RunHistKey, SuiteHistory> suiteHistory;
-
/** Compactor. */
@Inject private IStringCompactor compactor;
- /**
- * Non persistence cache for all suite RunHistory for particular branch.
- * RunHistKey(ServerId||BranchId||suiteId)-> Build reference
- */
- private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache
- = CacheBuilder.newBuilder()
- .maximumSize(8000)
- .expireAfterAccess(16, TimeUnit.MINUTES)
- .softValues()
- .build();
-
+ /** History collector. */
+ @Inject private HistoryCollector histCollector;
/**
*
@@ -155,7 +134,7 @@ public class FatBuildDao {
public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
- invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
+ histCollector.invalidateHistoryInMem(srvIdMaskHigh, newBuild);
}
public static int[] extractChangeIds(@Nonnull ChangesList changesList) {
@@ -172,7 +151,7 @@ public class FatBuildDao {
}
/**
- * @param srvIdMaskHigh Server id mask high.
+ * @param srvIdMaskHigh Server id mask to be placed at high bits of the key.
* @param buildId Build id.
*/
public static long buildIdToCacheKey(int srvIdMaskHigh, int buildId) {
@@ -197,10 +176,7 @@ public class FatBuildDao {
public Map<Long, FatBuildCompacted> getAllFatBuilds(int srvIdMaskHigh, Collection<Integer> buildsIds) {
Preconditions.checkNotNull(buildsCache, "init() was not called");
- Set<Long> ids = buildsIds.stream()
- .filter(Objects::nonNull)
- .map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId))
- .collect(Collectors.toSet());
+ Set<Long> ids = buildsIdsToCacheKeys(srvIdMaskHigh, buildsIds);
return buildsCache.getAll(ids);
}
@@ -223,108 +199,47 @@ public class FatBuildDao {
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
- public IRunHistory getTestRunHist(int srvIdMaskHigh,
- Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) {
-
-
- RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, suiteName, branchName);
-
- SuiteHistory history;
-
- try {
- history = runHistInMemCache.get(runHistKey,
- () -> {
- Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
-
- return calcSuiteHistory(srvIdMaskHigh, buildIds);
- });
- }
- catch (ExecutionException e) {
- throw ExceptionUtil.propagateException(e);
- }
-
- return history.testsHistory.get(testName);
+ private static Set<Long> buildsIdsToCacheKeys(int srvId, Collection<Integer> stream) {
+ return stream.stream()
+ .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet());
}
- @AutoProfiling
- protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
- Set<Long> cacheKeys = buildIds.stream().map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
-
- int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
-
- CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new HistoryCollectProcessor(successStatusStrId);
-
- Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
-
- SuiteHistory hist = new SuiteHistory();
-
- map.values().forEach(
- res-> {
- if(res==null)
- return;
-
- Map<Integer, Invocation> invocationMap = res.get();
-
- if(invocationMap == null)
- return;
-
- invocationMap.forEach((k, v) -> {
- RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
- k_ -> new RunHistCompacted());
-
- compacted.innerAddInvocation(v);
+ /**
+ * @param srvId Server id.
+ * @param ids Ids.
+ */
+ public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) {
+ IgniteCache<Long, BinaryObject> cacheBin = buildsCache.withKeepBinary();
+ Set<Long> keys = buildsIdsToCacheKeys(srvId, ids);
+ HashMap<Integer, Long> res = new HashMap<>();
+
+ Iterables.partition(keys, 32 * 10).forEach(
+ chunk -> {
+ Map<Long, EntryProcessorResult<Long>> map = cacheBin.invokeAll(keys, new GetStartTimeProc());
+ map.forEach((k, r) -> {
+ Long ts = r.get();
+ if (ts != null)
+ res.put(BuildRefDao.cacheKeyToBuildId(k), ts);
});
-
}
);
- System.err.println("Suite history: tests in scope "
- + hist.testsHistory.size()
- + " for " +buildIds.size() + " builds checked"
- + " size " + hist.size(igniteProvider.get()));
-
- return hist;
+ return res;
}
- @AutoProfiling
- protected Set<Integer> determineLatestBuilds(Supplier<Set<Integer>> buildIdsSupplier) {
- return buildIdsSupplier.get();
- }
-
- public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
- Iterable<RunHistKey> objects =
- stream
- .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
- .collect(Collectors.toSet());
-
- runHistInMemCache.invalidateAll(objects);
- }
-
-
- private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
- private final int successStatusStrId;
-
- public HistoryCollectProcessor(int successStatusStrId) {
- this.successStatusStrId = successStatusStrId;
+ private static class GetStartTimeProc implements CacheEntryProcessor<Long, BinaryObject, Long> {
+ public GetStartTimeProc() {
}
- @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
+ /** {@inheritDoc} */
+ @Override public Long process(MutableEntry<Long, BinaryObject> entry,
Object... arguments) throws EntryProcessorException {
if (entry.getValue() == null)
return null;
- Map<Integer, Invocation> hist = new HashMap<>();
- FatBuildCompacted fatBuildCompacted = entry.getValue();
- Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
- tests.forEach(
- testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
-
- hist.put(testCompacted.testName(), invocation);
- }
- );
+ BinaryObject buildBinary = entry.getValue();
- return hist;
+ return buildBinary.field("startDate");
}
}
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
index 3e01c05..8ad06a7 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
@@ -315,7 +315,8 @@ public class ProactiveFatBuildSync {
buildRefDao.save(srvIdMask, refCompacted);
- runHistSync.saveToHistoryLater(srvCode, savedVer);
+ //todo remove unused code
+ // runHistSync.saveToHistoryLater(srvCode, savedVer);
return savedVer;
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index 8969736..0e2d010 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -21,15 +21,50 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.Ignite;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.internal.binary.BinaryObjectExImpl;
+import org.apache.ignite.tcignited.history.IRunHistory;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
+import org.apache.ignite.tcignited.history.SuiteInvocation;
-public class SuiteHistory {
+/**
+ * Suite run history summary.
+ */
+public class SuiteHistory implements ISuiteRunHistory {
/** Tests history: Test name ID->RunHistory */
- Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
+ private Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
+
+ private RunHistCompacted suiteHist = new RunHistCompacted();
public int size(Ignite ignite) {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
return binary.length();
}
+
+ public IRunHistory getTestRunHist(int testName) {
+ return testsHistory.get(testName);
+ }
+
+ public RunHistCompacted getOrAddTestsHistory(Integer tName) {
+ return testsHistory.computeIfAbsent(tName, k_ -> new RunHistCompacted());
+ }
+
+ public void addTestInvocation(Integer tName, Invocation invocation) {
+ getOrAddTestsHistory(tName).innerAddInvocation(invocation);
+ }
+
+ public void addSuiteInvocation(SuiteInvocation suiteInv) {
+ suiteInv.tests().forEach(this::addTestInvocation);
+
+ suiteHist.innerAddInvocation(suiteInv.suiteInvocation());
+ }
+
+ public RunHistCompacted getSuiteHist() {
+ return suiteHist;
+ }
+
+ @Override public IRunHistory self() {
+ return suiteHist;
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
similarity index 53%
copy from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
index 8969736..ab5f0b9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
@@ -14,22 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ignite.tcignited.buildref;
-package org.apache.ignite.tcignited.build;
+import com.google.common.collect.Lists;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.ignite.tcservice.ITeamcity;
-import java.util.HashMap;
-import java.util.Map;
+public class BranchEquivalence {
+ /** Default synonyms. */
+ private static final List<String> DEFAULT_SYNONYMS
+ = Collections.unmodifiableList(
+ Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER));
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
-import org.apache.ignite.internal.binary.BinaryObjectExImpl;
-public class SuiteHistory {
- /** Tests history: Test name ID->RunHistory */
- Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
-
- public int size(Ignite ignite) {
- BinaryObjectExImpl binary = ignite.binary().toBinary(this);
- return binary.length();
+ public List<String> branchForQuery(@Nullable String branchName) {
+ if (ITeamcity.DEFAULT.equals(branchName))
+ return DEFAULT_SYNONYMS;
+ else
+ return Collections.singletonList(branchName);
}
+
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index f3a3e89..e8c8ae0 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -175,7 +175,7 @@ public class BuildRefDao {
*/
@AutoProfiling
@Nonnull public List<BuildRefCompacted> getAllBuildsCompacted(int srvId,
- String buildTypeId,
+ String buildTypeId,
List<String> bracnhNameQry) {
Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId);
if (buildTypeIdId == null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
new file mode 100644
index 0000000..7254774
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Iterables;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
+import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.build.SuiteHistory;
+import org.apache.ignite.tcignited.buildref.BranchEquivalence;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class HistoryCollector {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(HistoryCollector.class);
+
+ /** History DAO. */
+ @Inject private SuiteInvocationHistoryDao histDao;
+
+ /** Fat build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
+ /** Build reference DAO. */
+ @Inject private BuildRefDao buildRefDao;
+
+ /** Compactor. */
+ @Inject private IStringCompactor compactor;
+
+ /** Ignite provider. */
+ @Inject private Provider<Ignite> igniteProvider;
+
+ /** Branch equivalence. */
+ @Inject private BranchEquivalence branchEquivalence;
+
+ /** Run history DAO. */
+ @Inject private RunHistCompactedDao runHistCompactedDao;
+
+ /**
+ * Non persistence cache for all suite RunHistory for particular branch. RunHistKey(ServerId||BranchId||suiteId)->
+ * Build reference
+ */
+ private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache
+ = CacheBuilder.newBuilder()
+ .maximumSize(8000)
+ .expireAfterAccess(16, TimeUnit.MINUTES)
+ .softValues()
+ .build();
+
+ /** Biggest build ID, which out of history scope (MAX days + 2). */
+ private final ConcurrentMap<Integer, AtomicInteger> biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>();
+
+ /**
+ * @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
+ * @param testName Test name.
+ * @param buildTypeId Suite (Build type) id.
+ * @param normalizedBaseBranch Branch name.
+ */
+ public IRunHistory getTestRunHist(int srvIdMaskHigh, int testName, int buildTypeId,
+ int normalizedBaseBranch) {
+
+ SuiteHistory hist = getSuiteHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+
+ return hist.getTestRunHist(testName);
+ }
+
+ @AutoProfiling
+ protected SuiteHistory getSuiteHist(int srvIdMaskHigh, int buildTypeId, int normalizedBaseBranch) {
+ RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+
+ SuiteHistory hist;
+ try {
+ hist = runHistInMemCache.get(runHistKey,
+ () -> loadSuiteHistory(srvIdMaskHigh, buildTypeId, normalizedBaseBranch));
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+
+ return hist;
+ }
+
+ /**
+ * Latest actual Build ids supplier. This supplier should handle all equivalent branches in
+ * it.
+ * @param srvId
+ * @param buildTypeId
+ * @param normalizedBaseBranch
+ * @param knownBuilds Known builds, which already present in run history.
+ */
+ @AutoProfiling
+ protected Set<Integer> determineLatestBuilds(
+ int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
+ String btId = compactor.getStringFromId(buildTypeId);
+ String branchId = compactor.getStringFromId(normalizedBaseBranch);
+ List<BuildRefCompacted> bRefsList = buildRefDao.getAllBuildsCompacted(srvId, btId,
+ branchEquivalence.branchForQuery(branchId));
+
+ long curTs = System.currentTimeMillis();
+ Set<Integer> buildIds = bRefsList.stream()
+ .filter(b -> {
+ AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
+ int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
+ return b.id() > outOfScopeBuildId;
+ })
+ .filter(this::applicableForHistory)
+ .map(BuildRefCompacted::id)
+ .filter(bId -> !knownBuilds.contains(bId)).collect(Collectors.toSet());
+
+ System.out.println("***** Loading build start time history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" );
+
+ Map<Integer, Long> buildStartTimes = getStartTimeFromSpecialCache(srvId, buildIds);
+
+ Set<Integer> notFoundKeys = new HashSet<>(buildIds);
+ notFoundKeys.removeAll(buildStartTimes.keySet());
+
+ if (!notFoundKeys.isEmpty()) {
+ Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, notFoundKeys);
+
+ buildStartTimes.putAll(buildStartTimeFromFatBuild);
+
+ runHistCompactedDao.setBuildsStartTime(srvId, buildStartTimeFromFatBuild);
+ }
+
+ Set<Integer> buildInScope = buildIds.stream().filter(
+ bId -> {
+ Long startTime = buildStartTimes.get(bId);
+ if (startTime == null)
+ return false;
+
+ long ageInDays = Duration.ofMillis(curTs - startTime).toDays();
+
+ if (ageInDays > InvocationData.MAX_DAYS + 2) {
+ AtomicInteger integer = biggestBuildIdOutOfHistoryScope.computeIfAbsent(srvId,
+ s -> {
+ AtomicInteger atomicInteger = new AtomicInteger();
+ atomicInteger.set(-1);
+ return atomicInteger;
+ });
+
+ int newBorder = integer.accumulateAndGet(bId, Math::max);
+
+ if (newBorder == bId)
+ logger.info("History Collector: New border for server was set " + bId);
+ }
+
+ return ageInDays < InvocationData.MAX_DAYS;
+ }
+ ).collect(Collectors.toSet());
+
+ System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " +
+ buildInScope.size() + " from " + bRefsList.size());
+
+ return buildInScope;
+ }
+
+ @AutoProfiling
+ protected Map<Integer, Long> getStartTimeFromSpecialCache(int srvId, Set<Integer> buildIds) {
+ return runHistCompactedDao.getBuildsStartTime(srvId, buildIds);
+ }
+
+ @AutoProfiling
+ protected Map<Integer, Long> getStartTimeFromFatBuild(int srvId, Set<Integer> buildIds) {
+ return fatBuildDao.getBuildStartTime(srvId, buildIds);
+ }
+
+ /**
+ * @param ref Build Reference or fat build.
+ */
+ private boolean applicableForHistory(BuildRefCompacted ref) {
+ return !ref.isFakeStub() && !ref.isCancelled(compactor) && ref.isFinished(compactor);
+ }
+
+ @AutoProfiling
+ protected SuiteHistory loadSuiteHistory(int srvId,
+ int buildTypeId,
+ int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
+
+ System.out.println("***** Found history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + suiteRunHist.size() );
+
+ Set<Integer> buildIds = determineLatestBuilds(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
+
+ HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
+
+ missedBuildsIds.removeAll(suiteRunHist.keySet());
+
+ if (!missedBuildsIds.isEmpty()) {
+ Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
+
+ suiteRunHist.putAll(addl);
+
+ /*
+ Map<Integer, SuiteInvocation> reloaded = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
+
+ addl.keySet().forEach((k) -> {
+ Preconditions.checkState( reloaded.containsKey(k));
+ });
+ */
+ }
+
+ SuiteHistory sumary = new SuiteHistory();
+
+ suiteRunHist.forEach((buildId, suiteInv) -> sumary.addSuiteInvocation(suiteInv));
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("***** History for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ sumary.size(igniteProvider.get()) + " bytes");
+ }
+
+ return sumary;
+ }
+
+ /**
+ * @param srvId Server id.
+ * @param b Build ref to invalidate.
+ */
+ public void invalidateHistoryInMem(int srvId, BuildRefCompacted b) {
+ RunHistKey inv = new RunHistKey(srvId, b.buildTypeId(), b.branchName());
+
+ runHistInMemCache.invalidate(inv);
+ }
+
+
+ @AutoProfiling
+ protected Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
+ HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ System.err.println(Thread.currentThread().getName() + ": GET ALL: " + missedBuildsIds.size());
+
+ Iterables.partition(missedBuildsIds, 32 * 10).forEach(
+ chunk -> {
+ fatBuildDao.getAllFatBuilds(srvId, chunk).forEach((buildCacheKey, fatBuildCompacted) -> {
+ if (!applicableForHistory(fatBuildCompacted))
+ return;
+
+ BiPredicate<Integer, Integer> paramsFilter = (k, v) -> false;
+
+ SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, paramsFilter);
+
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, paramsFilter, successStatusStrId);
+
+ sinv.addTest(testCompacted.testName(), invocation);
+ }
+ );
+
+ suiteRunHist.put(fatBuildCompacted.id(), sinv);
+ });
+ }
+ );
+
+
+ System.err.println("***** + Adding to persisted history "
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " +
+ suiteRunHist.size() + " invocations from " + missedBuildsIds.size() + " builds checked");
+
+ histDao.putAll(srvId, suiteRunHist);
+
+ return suiteRunHist;
+ }
+
+ /**
+ * @param srvId Server id.
+ * @param buildTypeId Build type id.
+ * @param normalizedBaseBranch Normalized base branch.
+ */
+ public ISuiteRunHistory getSuiteRunHist(int srvId, int buildTypeId, int normalizedBaseBranch) {
+ return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch);
+ }
+
+}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java
similarity index 59%
copy from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java
index 8969736..38dff1c 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java
@@ -14,22 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ignite.tcignited.history;
-package org.apache.ignite.tcignited.build;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
-import org.apache.ignite.internal.binary.BinaryObjectExImpl;
-
-public class SuiteHistory {
- /** Tests history: Test name ID->RunHistory */
- Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
-
- public int size(Ignite ignite) {
- BinaryObjectExImpl binary = ignite.binary().toBinary(this);
- return binary.length();
- }
+public interface ISuiteRunHistory {
+ IRunHistory self();
+ IRunHistory getTestRunHist(int testName);
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index 8fed725..ffc0c47 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -17,9 +17,15 @@
package org.apache.ignite.tcignited.history;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.cache.Cache;
@@ -40,6 +46,7 @@ import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
@@ -61,6 +68,7 @@ public class RunHistCompactedDao {
private Provider<Ignite> igniteProvider;
/** Test history cache. */
+ @Deprecated
private IgniteCache<RunHistKey, RunHistCompacted> testHistCache;
/** Suite history cache. */
@@ -234,4 +242,32 @@ public class RunHistCompactedDao {
cluster.disableWal(testHistCache.getName());
cluster.disableWal(suiteHistCache.getName());
}
+
+ private static Set<Long> buildsIdsToCacheKeys(int srvId, Collection<Integer> ids) {
+ return ids.stream()
+ .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet());
+ }
+
+ public Map<Integer, Long> getBuildsStartTime(int srvId, Set<Integer> ids) {
+ Set<Long> cacheKeys = buildsIdsToCacheKeys(srvId, ids);
+
+ Map<Integer, Long> res = new HashMap<>();
+
+ buildStartTime.getAll(cacheKeys).forEach((k, r) -> {
+ res.put(BuildRefDao.cacheKeyToBuildId(k), r);
+ });
+
+ return res;
+ }
+
+ public void setBuildsStartTime(int srvId, Map<Integer, Long> builds) {
+ Map<Long, Long> res = new HashMap<>();
+
+ builds.forEach((buildId, ts) -> {
+ if (ts != null)
+ res.put(buildIdToCacheKey(srvId, buildId), ts);
+ });
+
+ buildStartTime.putAll(res);
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
index a0a90f4..9a621ee 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
@@ -205,6 +205,8 @@ public class RunHistSync {
@Nonnull protected String saveInvocationsMap(
Map<RunHistKey, List<Invocation>> buildsSaveThisRun,
Map<RunHistKey, List<Invocation>> testsSaveThisRun) {
+ if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE)))
+ return "Skipped";
if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE)))
if (testsSaveThisRun.size() > 100)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
new file mode 100644
index 0000000..7342af7
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.BiPredicate;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcbot.persistence.Persisted;
+
+/**
+ * Shorter verison of FatBuild with less data: created only if run history was required,
+ * has time limitation of MAX_DAYS, may have TTL.
+ */
+@Persisted
+public class SuiteInvocation {
+ /** Server ID for queries */
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 0)})
+ private int srvId;
+
+ /** Suite name for queries */
+ @AffinityKeyMapped
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 1)})
+ private int buildTypeId;
+
+ /** Teamcity branch name for queries */
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)})
+ private int normalizedBranchName;
+
+ private Invocation suite;
+
+ private Map<Integer, Invocation> tests = new HashMap<>();
+
+ Long buildStartTime;
+
+ public SuiteInvocation() {}
+
+ public SuiteInvocation(int srvId, int normalizedBaseBranch, FatBuildCompacted buildCompacted, IStringCompactor comp,
+ BiPredicate<Integer, Integer> filter) {
+ this.srvId = srvId;
+ this.normalizedBranchName = normalizedBaseBranch;
+ this.buildStartTime = buildCompacted.getStartDateTs();
+ this.suite = buildCompacted.toInvocation(comp, filter);
+ this.buildTypeId = buildCompacted.buildTypeId();
+ }
+
+ public void addTest(int testName, Invocation invocation) {
+ tests.put(testName, invocation);
+ }
+
+ public Map<Integer, Invocation> tests() {
+ return Collections.unmodifiableMap(tests);
+ }
+
+ public Invocation suiteInvocation() {
+ return suite;
+ }
+}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
new file mode 100644
index 0000000..dd93c5e
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import javax.cache.Cache;
+import javax.cache.expiry.AccessedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
+import org.apache.ignite.tcbot.persistence.CacheConfigs;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+
+import static java.util.concurrent.TimeUnit.HOURS;
+
+/**
+ * Suite invocation history access object.
+ */
+public class SuiteInvocationHistoryDao {
+ /** Ignite provider. */
+ @Inject
+ private Provider<Ignite> igniteProvider;
+
+ /** Suite history cache. */
+ private IgniteCache<Long, SuiteInvocation> suiteHistory;
+
+ public void init() {
+ CacheConfiguration<Long , SuiteInvocation> ccfg = CacheConfigs.getCacheV2Config("teamcitySuiteHistory");
+ ccfg.setExpiryPolicyFactory(AccessedExpiryPolicy.factoryOf(new Duration(HOURS, 12)));
+ ccfg.setEagerTtl(true);
+
+ ccfg.setQueryEntities(Collections.singletonList(new QueryEntity(Long.class, SuiteInvocation.class)));
+
+ Ignite ignite = igniteProvider.get();
+
+ suiteHistory = ignite.getOrCreateCache(ccfg);
+ }
+
+ @AutoProfiling
+ public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int buildTypeId, int normalizedBranchName) {
+ java.util.Map<Integer, SuiteInvocation> map = new HashMap<>();
+ try (QueryCursor<Cache.Entry<Long, SuiteInvocation>> qryCursor = suiteHistory.query(
+ new SqlQuery<Long, SuiteInvocation>(SuiteInvocation.class, "srvId = ? and buildTypeId = ? and normalizedBranchName = ?")
+ .setArgs(srvId, buildTypeId, normalizedBranchName))) {
+
+ for (Cache.Entry<Long, SuiteInvocation> next : qryCursor) {
+ Long key = next.getKey();
+ int buildId = BuildRefDao.cacheKeyToBuildId(key);
+ map.put(buildId, next.getValue());
+ }
+ }
+
+ return map;
+ }
+
+ @AutoProfiling
+ public void putAll(int srvId, Map<Integer, SuiteInvocation> addl) {
+ Map<Long, SuiteInvocation> data = new HashMap<>();
+
+ addl.forEach((k, v) -> data.put(BuildRefDao.buildIdToCacheKey(srvId, k), v));
+
+ suiteHistory.putAll(data);
+ }
+}