You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/30 12:29:29 UTC
[ignite-teamcity-bot] branch master updated: IGNITE-9848: Loading
builds chain in accordance with fat build approach,
calculate statistics moved to later execute. - Fixes #54.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 9592a86 IGNITE-9848: Loading builds chain in accordance with fat build approach, calculate statistics moved to later execute. - Fixes #54.
9592a86 is described below
commit 9592a867a0fd84ecff1e33e440ff35911749d6eb
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Oct 30 15:29:22 2018 +0300
IGNITE-9848: Loading builds chain in accordance with fat build approach, calculate statistics moved to later execute. - Fixes #54.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
---
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 9 +-
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 40 ++--
.../ignite/ci/analysis/SingleBuildRunCtx.java | 12 +-
...lureOccurrences.java => TestCompactedMult.java} | 48 ++---
.../java/org/apache/ignite/ci/db/DbMigrations.java | 8 +-
.../ci/tcbot/builds/CompareBuildsService.java | 8 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 201 ++++++++++-----------
.../ci/teamcity/ignited/BuildRefCompacted.java | 39 +++-
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 27 ++-
.../ci/teamcity/ignited/ITeamcityIgnited.java | 17 ++
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 43 ++++-
.../ci/teamcity/ignited/change/ChangeDao.java | 18 +-
.../ci/teamcity/ignited/change/ChangeSync.java | 5 +-
.../ignited/fatbuild/FatBuildCompacted.java | 29 ++-
.../ignited/fatbuild/ProactiveFatBuildSync.java | 3 -
.../teamcity/ignited/fatbuild/TestCompacted.java | 25 ++-
.../org/apache/ignite/ci/util/CollectionUtil.java | 2 +-
.../ci/web/model/current/SuiteCurrentStatus.java | 23 +--
18 files changed, 348 insertions(+), 209 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 4f13083..42a1508 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -61,6 +61,7 @@ import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.db.DbMigrations;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.di.cache.GuavaCached;
import org.apache.ignite.ci.tcmodel.agent.Agent;
import org.apache.ignite.ci.tcmodel.changes.Change;
import org.apache.ignite.ci.tcmodel.changes.ChangesList;
@@ -951,7 +952,9 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return key -> key == null ? null : getRunStatForTest(key);
}
+ @SuppressWarnings("WeakerAccess")
@AutoProfiling
+ @GuavaCached(maximumSize = 200, expireAfterAccessSecs = 30, softValues = true)
protected RunStat getRunStatForTest(TestInBranch key) {
return testRunStatCache().get(key);
}
@@ -971,12 +974,14 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@Override public Function<SuiteInBranch, RunStat> getBuildFailureRunStatProvider() {
- return key -> key == null ? null : getRunStatForTest(key);
+ return key -> key == null ? null : getRunStatForSuite(key);
}
+ @SuppressWarnings("WeakerAccess")
@AutoProfiling
- protected RunStat getRunStatForTest(SuiteInBranch key) {
+ @GuavaCached(maximumSize = 500, expireAfterAccessSecs = 90, softValues = true)
+ protected RunStat getRunStatForSuite(SuiteInBranch key) {
return buildsFailureRunStatCache().get(key);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index 11e0db5..56613c2 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -24,6 +24,7 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.util.CollectionUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -49,10 +50,6 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();
- /** Tests: Map from full test name to multiple test occurrence. */
- @Deprecated
- private final Map<String, MultTestFailureOccurrences> tests = new ConcurrentSkipListMap<>();
-
public void addBuild(SingleBuildRunCtx ctx) {
builds.add(ctx);
}
@@ -226,22 +223,33 @@ public class MultBuildRunCtx implements ISuiteResults {
}
public Stream<? extends ITestFailures> getTopLongRunning() {
- Stream<MultTestFailureOccurrences> stream = tests.values().stream();
- Comparator<MultTestFailureOccurrences> comparing = Comparator.comparing(MultTestFailureOccurrences::getAvgDurationMs);
- return CollectionUtil.top(stream, 3, comparing).stream();
+ Comparator<ITestFailures> comparing = Comparator.comparing(ITestFailures::getAvgDurationMs);
+
+ Map<Integer, TestCompactedMult> res = new HashMap<>();
+
+ builds.forEach(singleBuildRunCtx -> {
+ saveToMap(res, singleBuildRunCtx.getAllTests());
+ });
+
+ return CollectionUtil.top(res.values().stream(), 3, comparing).stream();
}
- public Stream<? extends ITestFailures> getFailedTests() {
- return tests.values().stream().filter(MultTestFailureOccurrences::hasFailedButNotMuted);
+ public List<ITestFailures> getFailedTests() {
+ Map<Integer, TestCompactedMult> res = new HashMap<>();
+
+ builds.forEach(singleBuildRunCtx -> {
+ saveToMap(res, singleBuildRunCtx.getFailedNotMutedTests());
+ });
+
+
+ return new ArrayList<>(res.values());
}
- @Deprecated
- public void addTests(Iterable<TestOccurrenceFull> tests) {
- for (TestOccurrenceFull next : tests) {
- this.tests.computeIfAbsent(next.name,
- k -> new MultTestFailureOccurrences())
- .add(next);
- }
+ public void saveToMap(Map<Integer, TestCompactedMult> res, Stream<TestCompacted> tests) {
+ tests.forEach(testCompacted -> {
+ res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor))
+ .add(testCompacted);
+ });
}
public int getBuildId() {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
index 685c916..5560929 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
@@ -30,6 +30,7 @@ import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.jetbrains.annotations.Nullable;
@@ -169,16 +170,21 @@ public class SingleBuildRunCtx implements ISuiteResults {
* @return Names of not muted or ignored test failed for non composite build
*/
public Stream<String> getFailedNotMutedTestNames() {
- if(isComposite())
- return Stream.empty();
+ return isComposite() ? Stream.empty() : buildCompacted.getFailedNotMutedTestNames(compactor);
+ }
- return buildCompacted.getFailedNotMutedTestNames(compactor);
+ public Stream<TestCompacted> getFailedNotMutedTests() {
+ return isComposite() ? Stream.empty() : buildCompacted.getFailedNotMutedTests(compactor);
}
public Stream<String> getAllTestNames() {
return buildCompacted.getAllTestNames(compactor);
}
+ public Stream<TestCompacted> getAllTests() {
+ return isComposite() ? Stream.empty() : buildCompacted.getAllTests();
+ }
+
public String suiteName() {
return buildCompacted.buildTypeName(compactor);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultTestFailureOccurrences.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
similarity index 56%
rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultTestFailureOccurrences.java
rename to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
index fa177f0..e76dcf7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultTestFailureOccurrences.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
@@ -17,36 +17,35 @@
package org.apache.ignite.ci.analysis;
+import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.stream.Stream;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
+import java.util.stream.Collectors;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
-public class MultTestFailureOccurrences implements ITestFailures {
- private final List<TestOccurrenceFull> occurrences = new CopyOnWriteArrayList<>();
-
- public MultTestFailureOccurrences() {
+public class TestCompactedMult implements ITestFailures {
+ private final List<TestCompacted> occurrences = new ArrayList<>();
+ private IStringCompactor compactor;
+ private long avgDuration = -1;
+ public TestCompactedMult(IStringCompactor compactor) {
+ this.compactor = compactor;
}
@Override public String getName() {
- return occurrences.isEmpty() ? "" : occurrences.iterator().next().name;
+ return occurrences.isEmpty() ? "" : occurrences.iterator().next().testName(compactor);
}
@Override public boolean isInvestigated() {
- return occurrences.stream().anyMatch(TestOccurrence::isInvestigated);
- }
-
- public boolean hasFailedButNotMuted() {
- return getFailedButNotMutedCount() > 0;
+ return occurrences.stream().anyMatch(TestCompacted::isInvestigated);
}
private int getFailedButNotMutedCount() {
return (int)occurrences.stream()
.filter(Objects::nonNull)
- .filter(TestOccurrence::isFailedButNotMuted).count();
+ .filter(t -> t.isFailedButNotMuted(compactor)).count();
}
@Override public int failuresCount() {
@@ -54,19 +53,24 @@ public class MultTestFailureOccurrences implements ITestFailures {
}
public long getAvgDurationMs() {
- //todo avg
- Stream<Long> stream = occurrences.stream().map(TestOccurrence::getDuration);
- return stream.findAny().orElse(0L);
+ if (avgDuration < 0) {
+ avgDuration = (long)occurrences.stream()
+ .map(TestCompacted::getDuration)
+ .filter(Objects::nonNull)
+ .mapToInt(i -> i)
+ .average()
+ .orElse(0);
+ }
+ return avgDuration;
}
@Override public Iterable<TestOccurrenceFull> getOccurrences() {
- return occurrences;
+ return occurrences.stream()
+ .map(testCompacted -> testCompacted.toTestOccurrence(compactor, 0))
+ .collect(Collectors.toList());
}
- public void add(TestOccurrenceFull next) {
- if (next.getId() == null)
- return;
-
+ public void add(TestCompacted next) {
occurrences.add(next);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 6441ab1..824dc4a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -404,10 +404,10 @@ public class DbMigrations {
applyDestroyIgnCacheMigration(FINISHED_BUILDS_INCLUDE_FAILED);
applyDestroyIgnCacheMigration(TEST_OCCURRENCE_FULL);
- applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD, TEAMCITY_BUILD_CACHE_NAME_OLD);
applyDestroyIgnCacheMigration(TESTS);
applyDestroyIgnCacheMigration(STAT);
applyDestroyIgnCacheMigration(BUILD_STATISTICS);
+ applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD, TEAMCITY_BUILD_CACHE_NAME_OLD);
}
private void applyDestroyIgnCacheMigration(String cacheName) {
@@ -415,12 +415,12 @@ public class DbMigrations {
applyDestroyCacheMigration(cacheName, ignCacheNme);
}
- private void applyDestroyCacheMigration(String dispCacheName, String ignCacheNme) {
+ private void applyDestroyCacheMigration(String dispCacheName, String cacheNme) {
applyMigration("destroy-" + dispCacheName, () -> {
- IgniteCache<Object, Object> cache = ignite.cache(ignCacheNme);
+ IgniteCache<Object, Object> cache = ignite.cache(cacheNme);
if (cache == null) {
- System.err.println("cache not found");
+ System.err.println("cache [" + cacheNme + "] not found");
return;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
index da17398..71715c8 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider;
import org.apache.ignite.ci.user.ICredentialsProv;
import org.slf4j.Logger;
@@ -59,7 +60,7 @@ public class CompareBuildsService {
}
/** */
- private List<String> tests0(ITeamcity tc, ITeamcityIgnited server,
+ private List<String> tests0(ITeamcity tc, ITeamcityIgnited tcIgnited,
BuildRef ref, BuildChainProcessor bcp) {
List<String> tests = new ArrayList<>();
@@ -71,14 +72,15 @@ public class CompareBuildsService {
logger.info("Build {} is composite ({}).", build.getId(), deps.size());
for (BuildRef ref0 : deps)
- tests.addAll(tests0(tc, server, ref0, bcp));
+ tests.addAll(tests0(tc, tcIgnited, ref0, bcp));
}
else {
logger.info("Loading tests for build {}.", build.getId());
MultBuildRunCtx buildCtx = new MultBuildRunCtx(build, compactor);
- buildCtx.addBuild(bcp.loadTestsAndProblems(tc, build, buildCtx, server));
+ final FatBuildCompacted fatBuild = tcIgnited.getFatBuild(build.getId());
+ buildCtx.addBuild(bcp.loadTestsAndProblems(fatBuild, tcIgnited));
for (String testName : buildCtx.tests())
tests.add(extractTestName(testName));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index fe26954..1295510 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.inject.Inject;
@@ -43,9 +44,9 @@ import org.apache.ignite.ci.analysis.mode.ProcessLogsMode;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
-import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
@@ -89,36 +90,38 @@ public class BuildChainProcessor {
if (entryPoints.isEmpty())
return new FullChainRunCtx(Build.createFakeStub());
- BuildRef next = entryPoints.iterator().next();
- Build results = teamcity.getBuild(next.href);
- FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(results);
+ Map<Integer, FatBuildCompacted> builds = new ConcurrentHashMap<>();
- Map<Integer, BuildRef> unique = new ConcurrentHashMap<>();
- Map<String, MultBuildRunCtx> buildsCtxMap = new ConcurrentHashMap<>();
-
- Stream<? extends BuildRef> uniqueBuldsInvolved = entryPoints.stream()
- .parallel()
- .unordered()
- .flatMap(ref -> dependencies(teamcity, ref)).filter(Objects::nonNull)
- .flatMap(ref -> dependencies(teamcity, ref)).filter(Objects::nonNull)
- .filter(ref -> ensureUnique(unique, ref))
- ;
+ final Stream<FatBuildCompacted> entryPointsFatBuilds = entryPoints.stream().map(BuildRef::getId)
+ .filter(Objects::nonNull)
+ .filter(id -> !builds.containsKey(id)) //load and propagate only new entry points
+ .map(id -> builds.computeIfAbsent(id, teamcityIgnited::getFatBuild));
final ExecutorService svc = tcUpdatePool.getService();
- final List<Future<Stream<BuildRef>>> phase1Submitted = uniqueBuldsInvolved
- .map((buildRef) -> svc.submit(
- () -> replaceWithRecent(teamcityIgnited, includeLatestRebuild, unique, buildRef, entryPoints.size())))
+ final Stream<FatBuildCompacted> depsFirstLevel = entryPointsFatBuilds
+ .map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, ref)))
+ .collect(Collectors.toList())
+ .stream()
+ .flatMap(fut -> FutureUtil.getResult(fut));
+
+ Stream<FatBuildCompacted> secondLevelDeps = depsFirstLevel
+ .map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, ref)))
+ .collect(Collectors.toList())
+ .stream()
+ .flatMap(fut -> FutureUtil.getResult(fut));
+
+ // builds may became non unique because of rece in filtering and acquiring deps
+ final List<Future<Stream<FatBuildCompacted>>> phase3Submitted = secondLevelDeps
+ .map((fatBuild) -> svc.submit(
+ () -> replaceWithRecent(teamcityIgnited, includeLatestRebuild, builds, fatBuild, entryPoints.size())))
.collect(Collectors.toList());
+ Map<String, MultBuildRunCtx> buildsCtxMap = new ConcurrentHashMap<>();
- final List<Future<? extends Stream<? extends BuildRef>>> phase2Submitted = phase1Submitted.stream()
- .map(FutureUtil::getResult)
- .map((s) -> svc.submit(
- () -> processBuildList(teamcity, teamcityIgnited, buildsCtxMap, s)))
- .collect(Collectors.toList());
-
- phase2Submitted.forEach(FutureUtil::getResult);
+ phase3Submitted.stream()
+ .flatMap(fut -> FutureUtil.getResult(fut))
+ .forEach((fatBuild) -> createCxt(teamcityIgnited, buildsCtxMap, fatBuild));
ArrayList<MultBuildRunCtx> contexts = new ArrayList<>(buildsCtxMap.values());
@@ -131,6 +134,7 @@ public class BuildChainProcessor {
Function<MultBuildRunCtx, Float> function = ctx -> {
SuiteInBranch key = new SuiteInBranch(ctx.suiteId(), normalizeBranch(failRateBranch));
+ //todo place RunStat into suite context to compare
RunStat runStat = teamcity.getBuildFailureRunStatProvider().apply(key);
if (runStat == null)
@@ -140,6 +144,10 @@ public class BuildChainProcessor {
return runStat.getCriticalFailRate() * 3.14f + runStat.getFailRate();
};
+ BuildRef someEntryPoint = entryPoints.iterator().next();
+ FatBuildCompacted build = builds.computeIfAbsent(someEntryPoint.getId(), teamcityIgnited::getFatBuild);
+ FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(build.toBuild(compactor));
+
contexts.sort(Comparator.comparing(function).reversed());
fullChainRunCtx.addAllSuites(contexts);
@@ -147,43 +155,31 @@ public class BuildChainProcessor {
return fullChainRunCtx;
}
+
@SuppressWarnings("WeakerAccess")
- @NotNull
@AutoProfiling
- protected Stream<? extends BuildRef> processBuildList(ITeamcity teamcity,
- ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap,
- Stream<? extends BuildRef> list) {
- list.forEach((BuildRef ref) -> {
- MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId,
- k -> new MultBuildRunCtx(ref, compactor));
-
- ctx.addBuild(loadTestsAndProblems(teamcity, ref, ctx, teamcityIgnited));
- });
+ protected void createCxt(ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap,
+ FatBuildCompacted buildCompacted) {
+ final BuildRef ref = buildCompacted.toBuildRef(compactor);
+
+ final MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId,
+ k -> new MultBuildRunCtx(ref, compactor));
- return list;
+ ctx.addBuild(loadTestsAndProblems(buildCompacted, teamcityIgnited));
}
/**
* Runs deep collection of all related statistics for particular build.
*
+ * @param buildCompacted Build ref from history with references to tests.
* @param tcIgnited
- * @param buildRef Build ref from history with references to tests.
* @return Full context.
*/
- public SingleBuildRunCtx loadTestsAndProblems(ITeamcity teamcity, @Nonnull BuildRef buildRef,
- @Deprecated MultBuildRunCtx mCtx, ITeamcityIgnited tcIgnited) {
-
- FatBuildCompacted buildCompacted = tcIgnited.getFatBuild(buildRef.getId());
-
+ public SingleBuildRunCtx loadTestsAndProblems(@Nonnull FatBuildCompacted buildCompacted,
+ ITeamcityIgnited tcIgnited) {
SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildCompacted, compactor);
- if (!buildCompacted.isComposite())
- mCtx.addTests(buildCompacted.getTestOcurrences(compactor).getTests());
-
- final int[] changeIds = buildCompacted.changes();
-
- Collection<ChangeCompacted> changes = tcIgnited.getAllChanges(changeIds);
- ctx.setChanges(changes);
+ ctx.setChanges(tcIgnited.getAllChanges(buildCompacted.changes()));
//todo support storing build.lastChanges.changes) ?
@@ -193,34 +189,35 @@ public class BuildChainProcessor {
@SuppressWarnings("WeakerAccess")
@NotNull
@AutoProfiling
- protected static Stream< BuildRef> replaceWithRecent(ITeamcityIgnited teamcityIgnited,
- LatestRebuildMode includeLatestRebuild,
- Map<Integer, BuildRef> unique,
- BuildRef buildRef,
- int cntLimit) {
+ protected Stream<FatBuildCompacted> replaceWithRecent(ITeamcityIgnited teamcityIgnited,
+ LatestRebuildMode includeLatestRebuild,
+ Map<Integer, FatBuildCompacted> builds,
+ FatBuildCompacted buildCompacted,
+ int cntLimit) {
if (includeLatestRebuild == LatestRebuildMode.NONE)
- return Stream.of(buildRef);
+ return Stream.of(buildCompacted);
- final String branch = getBranchOrDefault(buildRef.branchName);
+ final String branch = getBranchOrDefault(buildCompacted.branchName(compactor));
- //todo now it is a full scan without caching, probably it is better to make branch based index query & caching results
- Stream<BuildRef> history = teamcityIgnited.getBuildHistory(buildRef.buildTypeId, branch)
+ final String buildTypeId = buildCompacted.buildTypeId(compactor);
+ Stream<BuildRefCompacted> history = teamcityIgnited.getBuildHistoryCompacted(buildTypeId, branch)
.stream()
- .filter(BuildRef::isNotCancelled)
- .filter(BuildRef::isFinished);
+ .filter(t -> t.isNotCancelled(compactor))
+ .filter(t -> t.isFinished(compactor));
if (includeLatestRebuild == LatestRebuildMode.LATEST) {
- BuildRef recentRef = history.max(Comparator.comparing(BuildRef::getId)).orElse(buildRef);
+ BuildRefCompacted recentRef = history.max(Comparator.comparing(BuildRefCompacted::id))
+ .orElse(buildCompacted);
- return Stream.of(recentRef.isFakeStub() ? buildRef : recentRef);
+ return Stream.of(recentRef)
+ .map(b -> builds.computeIfAbsent(b.id(), teamcityIgnited::getFatBuild));
}
if (includeLatestRebuild == LatestRebuildMode.ALL) {
return history
- .filter(ref -> !ref.isFakeStub())
- .filter(ref -> ensureUnique(unique, ref))
- .sorted(Comparator.comparing(BuildRef::getId).reversed())
- .limit(cntLimit); // applying same limit
+ .sorted(Comparator.comparing(BuildRefCompacted::id).reversed())
+ .limit(cntLimit)
+ .map(b -> builds.computeIfAbsent(b.id(), teamcityIgnited::getFatBuild));
}
throw new UnsupportedOperationException("invalid mode " + includeLatestRebuild);
@@ -230,35 +227,37 @@ public class BuildChainProcessor {
return branchName == null ? ITeamcity.DEFAULT : branchName;
}
- private static boolean ensureUnique(Map<Integer, BuildRef> unique, BuildRef ref) {
- if (ref.isFakeStub())
- return false;
-
- BuildRef prevVal = unique.putIfAbsent(ref.getId(), ref);
-
- return prevVal == null;
- }
-
- private static void fillBuildCounts(MultBuildRunCtx outCtx,
+ @SuppressWarnings("WeakerAccess")
+ @AutoProfiling
+ protected void fillBuildCounts(MultBuildRunCtx outCtx,
ITeamcityIgnited teamcityIgnited, boolean includeScheduledInfo) {
if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) {
- //todo queue compacted instead of full
- long cntRunning = teamcityIgnited.getBuildHistory(outCtx.buildTypeId(), outCtx.branchName())
- .stream().filter(BuildRef::isNotCancelled).filter(BuildRef::isRunning).count();
+ final List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getBuildHistoryCompacted(outCtx.buildTypeId(), outCtx.branchName());
+
+ long cntRunning = runAllBuilds
+ .stream()
+ .filter(r -> r.isNotCancelled(compactor))
+ .filter(r -> r.isRunning(compactor)).count();
outCtx.setRunningBuildCount((int) cntRunning);
- long cntQueued = teamcityIgnited.getBuildHistory(outCtx.buildTypeId(), outCtx.branchName())
- .stream().filter(BuildRef::isNotCancelled).filter(BuildRef::isQueued).count();
+ long cntQueued = runAllBuilds
+ .stream()
+ .filter(r -> r.isNotCancelled(compactor))
+ .filter(r -> r.isQueued(compactor)).count();
outCtx.setQueuedBuildCount((int) cntQueued);
}
}
- private static void analyzeTests(MultBuildRunCtx outCtx, IAnalyticsEnabledTeamcity teamcity,
+ @SuppressWarnings("WeakerAccess")
+ @AutoProfiling
+ protected void analyzeTests(MultBuildRunCtx outCtx, IAnalyticsEnabledTeamcity teamcity,
ProcessLogsMode procLog) {
for (SingleBuildRunCtx ctx : outCtx.getBuilds()) {
- teamcity.calculateBuildStatistic(ctx);
+ tcUpdatePool.getService().submit(() -> {
+ teamcity.calculateBuildStatistic(ctx);
+ });
if ((procLog == ProcessLogsMode.SUITE_NOT_COMPLETE && ctx.hasSuiteIncompleteFailure())
|| procLog == ProcessLogsMode.ALL)
@@ -279,24 +278,24 @@ public class BuildChainProcessor {
return branch;
}
- @Nullable private static Stream<? extends BuildRef> dependencies(ITeamcity teamcity, BuildRef ref) {
- Build results = teamcity.getBuild(ref.href);
- if (results == null)
- return Stream.of(ref);
-
- List<BuildRef> deps = results.getSnapshotDependenciesNonNull();
-
- if(deps.isEmpty())
- return Stream.of(ref);
-
- if(logger.isDebugEnabled())
- logger.debug("Snapshot deps found: " +
- ref.suiteId() + "->" + deps.stream().map(BuildRef::suiteId).collect(Collectors.toList()));
-
- Collection<BuildRef> buildAndDeps = new ArrayList<>(deps);
-
- buildAndDeps.add(ref);
-
- return buildAndDeps.stream();
+ @NotNull
+ private Stream<FatBuildCompacted> dependencies(
+ ITeamcityIgnited teamcityIgnited,
+ Map<Integer, FatBuildCompacted> builds,
+ FatBuildCompacted build) {
+ return Stream.concat(
+ Stream.of(build),
+ IntStream.of(build.snapshotDependencies())
+ .mapToObj(id -> {
+ if (builds.containsKey(id))
+ return null; //load and propagate only new dependencies
+
+ FatBuildCompacted buildLoaded = teamcityIgnited.getFatBuild(id);
+
+ FatBuildCompacted prevVal = builds.putIfAbsent(id, buildLoaded);
+
+ return prevVal == null ? buildLoaded : null;
+ }))
+ .filter(Objects::nonNull);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index 589b13b..73a704a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -22,6 +22,8 @@ import org.apache.ignite.ci.db.Persisted;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.jetbrains.annotations.NotNull;
+import static org.apache.ignite.ci.tcmodel.hist.BuildRef.*;
+
@Persisted
public class BuildRefCompacted {
/** Build Id without modifications, -1 if it is null. */
@@ -83,13 +85,21 @@ public class BuildRefCompacted {
protected void fillBuildRefFields(IStringCompactor compactor, BuildRef res) {
res.setId(id < 0 ? null : id);
- res.buildTypeId = compactor.getStringFromId(buildTypeId);
- res.branchName = compactor.getStringFromId(branchName);
+ res.buildTypeId = buildTypeId(compactor);
+ res.branchName = branchName(compactor);
res.status = compactor.getStringFromId(status);
res.state = compactor.getStringFromId(state);
res.href = getHrefForId(id());
}
+ public String buildTypeId(IStringCompactor compactor) {
+ return compactor.getStringFromId(buildTypeId);
+ }
+
+ public String branchName(IStringCompactor compactor) {
+ return compactor.getStringFromId(branchName);
+ }
+
@NotNull protected static String getHrefForId(int id) {
return "/app/rest/latest/builds/id:" + id;
}
@@ -137,4 +147,29 @@ public class BuildRefCompacted {
public int state() {
return state;
}
+
+ /** */
+ public boolean isFakeStub() {
+ return id() < 0;
+ }
+
+ public boolean isNotCancelled(IStringCompactor compactor) {
+ return !hasUnknownStatus(compactor);
+ }
+
+ private boolean hasUnknownStatus(IStringCompactor compactor) {
+ return compactor.getStringId(STATUS_UNKNOWN) == state();
+ }
+
+ public boolean isRunning(IStringCompactor compactor) {
+ return compactor.getStringId(STATE_RUNNING) == state();
+ }
+
+ public boolean isFinished(IStringCompactor compactor) {
+ return compactor.getStringId(STATE_FINISHED) == state();
+ }
+
+ public boolean isQueued(IStringCompactor compactor) {
+ return compactor.getStringId(STATE_QUEUED) == state();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index 92b1684..d871f84 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -130,9 +130,9 @@ public class BuildRefDao {
* @param bracnhNameQry Bracnh name query.
*/
@AutoProfiling
- @NotNull public List<BuildRef> findBuildsInHistory(int srvId,
- @Nullable String buildTypeId,
- String bracnhNameQry) {
+ @NotNull public List<BuildRefCompacted> findBuildsInHistoryCompacted(int srvId,
+ @Nullable String buildTypeId,
+ String bracnhNameQry) {
Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId);
if (buildTypeIdId == null)
@@ -143,15 +143,22 @@ public class BuildRefDao {
return Collections.emptyList();
return getBuildsForBranch(srvId, bracnhNameQry).stream()
- .filter(e -> e.buildTypeId() == buildTypeIdId)
+ .filter(e -> e.buildTypeId() == buildTypeIdId)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * @param srvId Server id mask high.
+ * @param buildTypeId Build type id.
+ * @param bracnhNameQry Bracnh name query.
+ */
+ @AutoProfiling
+ @NotNull public List<BuildRef> findBuildsInHistory(int srvId,
+ @Nullable String buildTypeId,
+ String bracnhNameQry) {
+ return findBuildsInHistoryCompacted(srvId, buildTypeId, bracnhNameQry).stream()
.map(compacted -> compacted.toBuildRef(compactor))
.collect(Collectors.toList());
-/*
- return compactedBuildsForServer(srvId)
- .filter(e -> e.buildTypeId() == buildTypeIdId)
- .filter(e -> e.branchName() == bracnhNameQryId)
- .map(compacted -> compacted.toBuildRef(compactor))
- .collect(Collectors.toList());*/
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
index 9903920..5fea687 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
@@ -30,6 +30,11 @@ import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
*/
public interface ITeamcityIgnited {
/**
+ * @return Internal server ID as string
+ */
+ String serverId();
+
+ /**
* @return Normalized Host address, ends with '/'.
*/
public String host();
@@ -41,6 +46,18 @@ public interface ITeamcityIgnited {
* @param branchName Branch name.
* @return list of builds in history, includes all statuses: queued, running, etc
*/
+ public List<BuildRefCompacted> getBuildHistoryCompacted(
+ @Nullable String buildTypeId,
+ @Nullable String branchName);
+
+
+ /**
+ * Retun all builds for branch and suite, without relation to its status.
+ *
+ * @param buildTypeId Build type identifier.
+ * @param branchName Branch name.
+ * @return list of builds in history, includes all statuses: queued, running, etc
+ */
public List<BuildRef> getBuildHistory(
@Nullable String buildTypeId,
@Nullable String branchName);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 310ec4e..67ea0c2 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -104,6 +104,11 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
+ @Override public String serverId() {
+ return srvNme;
+ }
+
+ /** {@inheritDoc} */
@Override public String host() {
return conn.host();
}
@@ -111,17 +116,38 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** {@inheritDoc} */
@AutoProfiling
@Override public List<BuildRef> getBuildHistory(
- @Nullable String buildTypeId,
- @Nullable String branchName) {
- scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
+ @Nullable String buildTypeId,
+ @Nullable String branchName) {
+ ensureActualizeRequested();
+
+ String bracnhNameQry = branchForQuery(branchName);
+
+ return buildRefDao.findBuildsInHistory(srvIdMaskHigh, buildTypeId, bracnhNameQry);
+ }
+
+ /** {@inheritDoc} */
+ @AutoProfiling
+ @Override public List<BuildRefCompacted> getBuildHistoryCompacted(
+ @Nullable String buildTypeId,
+ @Nullable String branchName) {
+ ensureActualizeRequested();
+
+ String bracnhNameQry = branchForQuery(branchName);
+
+ return buildRefDao.findBuildsInHistoryCompacted(srvIdMaskHigh, buildTypeId, bracnhNameQry);
+ }
- String bracnhNameQry ;
+ public String branchForQuery(@Nullable String branchName) {
+ String bracnhNameQry;
if (ITeamcity.DEFAULT.equals(branchName))
bracnhNameQry = "refs/heads/master";
else
bracnhNameQry = branchName;
+ return bracnhNameQry;
+ }
- return buildRefDao.findBuildsInHistory(srvIdMaskHigh, buildTypeId, bracnhNameQry);
+ public void ensureActualizeRequested() {
+ scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
}
/** {@inheritDoc} */
@@ -147,6 +173,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
@Override public FatBuildCompacted getFatBuild(int buildId) {
+ ensureActualizeRequested();
FatBuildCompacted existingBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, buildId);
//todo additionally check queued and running builds, refesh builds if they are queued.
@@ -162,8 +189,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return savedVer == null ? existingBuild : savedVer;
}
- @Override
- public Collection<ChangeCompacted> getAllChanges(int[] changeIds) {
+ @AutoProfiling
+ @Override public Collection<ChangeCompacted> getAllChanges(int[] changeIds) {
final Map<Long, ChangeCompacted> all = changesDao.getAll(srvIdMaskHigh, changeIds);
final Map<Integer, ChangeCompacted> changes = new HashMap<>();
@@ -177,7 +204,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
for (int changeId : changeIds) {
if (!changes.containsKey(changeId)) {
- final ChangeCompacted change = changeSync.change(srvIdMaskHigh, changeId, conn);
+ final ChangeCompacted change = changeSync.reloadChange(srvIdMaskHigh, changeId, conn);
changes.put(changeId, change);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java
index f4a7d9a..14040ca 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeDao.java
@@ -17,6 +17,11 @@
package org.apache.ignite.ci.teamcity.ignited.change;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
+import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
@@ -24,10 +29,6 @@ import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.configuration.CacheConfiguration;
-import javax.inject.Inject;
-import javax.inject.Provider;
-import java.util.*;
-
public class ChangeDao {
/** Cache name */
public static final String TEAMCITY_CHANGE_CACHE_NAME = "teamcityChange";
@@ -92,14 +93,13 @@ public class ChangeDao {
return changesCache.get(changeIdToCacheKey(srvId, changeId));
}
+ @AutoProfiling
public Map<Long, ChangeCompacted> getAll(int srvIdMaskHigh, int[] changeIds) {
final Set<Long> collect = new HashSet<>();
- for (int changeId : changeIds) {
- collect.add(changeIdToCacheKey(srvIdMaskHigh, changeId));
- }
- final Map<Long, ChangeCompacted> all = changesCache.getAll(collect);
+ for (int changeId : changeIds)
+ collect.add(changeIdToCacheKey(srvIdMaskHigh, changeId));
- return all;
+ return changesCache.getAll(collect);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java
index 739f119..7e00f56 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/change/ChangeSync.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.teamcity.ignited.change;
import com.google.common.base.Throwables;
+import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.changes.Change;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
@@ -48,6 +49,7 @@ public class ChangeSync {
}
@NotNull
+ @AutoProfiling
public ChangeCompacted reloadChange(int srvId, int changeId, ITeamcityConn conn) {
Change change;
try {
@@ -63,9 +65,8 @@ public class ChangeSync {
//can re-queue this change without details
change = new Change();
change.id = Integer.toString(changeId);
- } else {
+ } else
throw ExceptionUtil.propagateException(e);
- }
}
final ChangeCompacted changeCompacted = new ChangeCompacted(compactor, change);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 48eb899..aeeb644 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -55,6 +55,7 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
/** flag offset. */
public static final int FAKE_BUILD_F = 4;
+ public static final int[] EMPTY = new int[0];
/** Entity fields version. */
private short _ver = LATEST_VERSION;
@@ -288,20 +289,27 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
return flag != null && flag;
}
- public Stream<String> getFailedNotMutedTestNames(IStringCompactor compactor) {
+ public Stream<TestCompacted> getFailedNotMutedTests(IStringCompactor compactor) {
if (tests == null)
return Stream.of();
return tests.stream()
- .filter(t -> t.isFailedButNotMuted(compactor))
- .map(t -> t.getTestName(compactor));
+ .filter(t -> t.isFailedButNotMuted(compactor));
}
- public Stream<String> getAllTestNames(IStringCompactor compactor) {
+ public Stream<String> getFailedNotMutedTestNames(IStringCompactor compactor) {
+ return getFailedNotMutedTests(compactor).map(t -> t.testName(compactor));
+ }
+
+ public Stream<TestCompacted> getAllTests() {
if (tests == null)
return Stream.of();
- return tests.stream().map(t -> t.getTestName(compactor));
+ return tests.stream();
+ }
+
+ public Stream<String> getAllTestNames(IStringCompactor compactor) {
+ return getAllTests().map(t -> t.testName(compactor));
}
public String buildTypeName(IStringCompactor compactor) {
@@ -354,9 +362,16 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
}
public int[] changes() {
- if(changesIds==null)
- return new int[0];
+ if (changesIds == null)
+ return EMPTY;
return changesIds;
}
+
+ public int[] snapshotDependencies() {
+ if (snapshotDeps == null)
+ return EMPTY;
+
+ return snapshotDeps.clone();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 4a5743a..47d7480 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -97,7 +97,6 @@ public class ProactiveFatBuildSync {
}
-
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0})
@AutoProfiling
@@ -106,8 +105,6 @@ public class ProactiveFatBuildSync {
final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
- Arrays.parallelSort(buildRefKeys);
-
List<Integer> buildsIdsToLoad = new ArrayList<>();
int totalAskedToLoad = 0;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index 573b87c..dfd0c1e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -145,14 +145,14 @@ public class TestCompacted {
buildId +
")";
occurrence.id(fullStrId);
- occurrence.duration = duration < 0 ? null : duration;
+ occurrence.duration = getDuration();
occurrence.name = compactor.getStringFromId(name);
occurrence.status = compactor.getStringFromId(status);
occurrence.href = "/app/rest/latest/testOccurrences/" + fullStrId;
occurrence.muted = getMutedFlag();
occurrence.currentlyMuted = getFlag(CUR_MUTED_F);
- occurrence.currentlyInvestigated = getFlag(CUR_INV_F);
+ occurrence.currentlyInvestigated = getCurrInvestigatedFlag();
occurrence.ignored = getIgnoredFlag();
if (actualBuildId > 0) {
@@ -176,6 +176,10 @@ public class TestCompacted {
return occurrence;
}
+ public Boolean getCurrInvestigatedFlag() {
+ return getFlag(CUR_INV_F);
+ }
+
/**
*
*/
@@ -331,7 +335,22 @@ public class TestCompacted {
return compactor.getStringId(TestOccurrence.STATUS_SUCCESS) != status;
}
- public String getTestName(IStringCompactor compactor) {
+ public String testName(IStringCompactor compactor) {
return compactor.getStringFromId(name);
}
+
+ public int testName() {
+ return name;
+ }
+
+ public boolean isInvestigated() {
+ final Boolean investigatedFlag = getCurrInvestigatedFlag();
+
+ return investigatedFlag != null && investigatedFlag;
+ }
+
+ @Nullable
+ public Integer getDuration() {
+ return duration < 0 ? null : duration;
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CollectionUtil.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CollectionUtil.java
index ba4b973..95b9f4f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CollectionUtil.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CollectionUtil.java
@@ -26,7 +26,7 @@ import java.util.stream.Stream;
* Collection util.
*/
public class CollectionUtil {
- public static <T> List<T> top(Stream<T> data, int cnt, Comparator<T> comp) {
+ public static <T> List<T> top(Stream<? extends T> data, int cnt, Comparator<T> comp) {
Comparator<T> reversedComp = comp.reversed();
return data.sorted(reversedComp).limit(cnt).collect(Collectors.toList());
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
index b693f73..0784e96 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
@@ -39,7 +39,6 @@ import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.analysis.TestLogCheckResult;
import org.apache.ignite.ci.issue.EventTemplates;
import org.apache.ignite.ci.issue.ProblemRef;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.web.model.hist.FailureSummary;
import org.apache.ignite.ci.web.rest.GetBuildLog;
import org.jetbrains.annotations.NotNull;
@@ -116,7 +115,7 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
public void initFromContext(@Nonnull final ITeamcity teamcity,
@Nonnull final MultBuildRunCtx suite,
- @Nullable final ITcAnalytics tcAnalytics,
+ @NotNull final ITcAnalytics tcAnalytics,
@Nullable final String baseBranch) {
name = suite.suiteName();
@@ -140,23 +139,21 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
webToHistBaseBranch = buildWebLink(teamcity, suite, baseBranch);
webToBuild = buildWebLinkToBuild(teamcity, suite);
- Stream<? extends ITestFailures> tests = suite.getFailedTests();
- if (tcAnalytics != null) {
- Function<ITestFailures, Float> function = foccur -> {
- TestInBranch branch = new TestInBranch(foccur.getName(), failRateNormalizedBranch);
+ List<ITestFailures> tests = suite.getFailedTests();
+ Function<ITestFailures, Float> function = foccur -> {
+ TestInBranch testInBranch = new TestInBranch(foccur.getName(), failRateNormalizedBranch);
- RunStat apply = tcAnalytics.getTestRunStatProvider().apply(branch);
+ RunStat apply = tcAnalytics.getTestRunStatProvider().apply(testInBranch);
- return apply == null ? 0f : apply.getFailRate();
- };
- tests = tests.sorted(Comparator.comparing(function).reversed());
- }
+ return apply == null ? 0f : apply.getFailRate();
+ };
+
+ tests.sort(Comparator.comparing(function).reversed());
tests.forEach(occurrence -> {
final TestFailure failure = new TestFailure();
failure.initFromOccurrence(occurrence, teamcity, suite.projectId(), suite.branchName(), baseBranch);
- if (tcAnalytics != null)
- failure.initStat(tcAnalytics.getTestRunStatProvider(), failRateNormalizedBranch, curBranchNormalized);
+ failure.initStat(tcAnalytics.getTestRunStatProvider(), failRateNormalizedBranch, curBranchNormalized);
testFailures.add(failure);
});