You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/26 13:48:16 UTC
[ignite-teamcity-bot] branch ignite-9848-load-all-builds updated:
Queuing build history migrated to new Ignite store
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-load-all-builds by this push:
new 8f62f2e Queuing build history migrated to new Ignite store
8f62f2e is described below
commit 8f62f2e9b48b36b0105653deb885b2eb019128a2
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Oct 26 16:47:26 2018 +0300
Queuing build history migrated to new Ignite store
---
.../apache/ignite/ci/analysis/FullChainRunCtx.java | 12 ------
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 43 +++++--------------
.../ignite/ci/analysis/SingleBuildRunCtx.java | 8 ++++
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 49 ++++++++++++----------
.../ci/teamcity/ignited/ITeamcityIgnited.java | 2 +
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 2 +-
.../ignited/fatbuild/FatBuildCompacted.java | 13 +++++-
7 files changed, 59 insertions(+), 70 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java
index 6a800eb..b8593fa 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java
@@ -72,18 +72,6 @@ public class FullChainRunCtx {
return chainResults.branchName;
}
- public int failedTests() {
- return buildCfgsResults.stream().mapToInt(MultBuildRunCtx::failedTests).sum();
- }
-
- public int mutedTests() {
- return buildCfgsResults.stream().mapToInt(MultBuildRunCtx::mutedTests).sum();
- }
-
- public int totalTests() {
- return buildCfgsResults.stream().mapToInt(MultBuildRunCtx::totalTests).sum();
- }
-
public Stream<MultBuildRunCtx> failedChildSuites() {
return suites().stream().filter(MultBuildRunCtx::isFailed);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index fe0a0d6..34da5c7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -32,6 +32,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
+import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
@@ -51,7 +52,8 @@ import static java.util.stream.Stream.concat;
* Includes tests and problem occurrences; if logs processing is done also contains last started test
*/
public class MultBuildRunCtx implements ISuiteResults {
- @Nonnull private final Build firstBuildInfo;
+ /** First build info. */
+ @Nonnull private final BuildRef firstBuildInfo;
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();
@@ -75,7 +77,7 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Currently scheduled builds */
@Nullable private CompletableFuture<Long> queuedBuildCount;
- public MultBuildRunCtx(@Nonnull final Build buildInfo) {
+ public MultBuildRunCtx(@Nonnull final BuildRef buildInfo) {
this.firstBuildInfo = buildInfo;
}
@@ -100,10 +102,6 @@ public class MultBuildRunCtx implements ISuiteResults {
return firstBuildInfo.suiteId();
}
- public String suiteName() {
- return firstBuildInfo.suiteName();
- }
-
public String buildTypeId() {
return firstBuildInfo.buildTypeId;
}
@@ -172,26 +170,6 @@ public class MultBuildRunCtx implements ISuiteResults {
return buildsStream().flatMap(SingleBuildRunCtx::getFailedNotMutedTestNames).distinct();
}
- public int mutedTests() {
- TestOccurrencesRef testOccurrences = firstBuildInfo.testOccurrences;
- if (testOccurrences == null)
- return 0;
- final Integer muted = testOccurrences.muted;
-
- return muted == null ? 0 : muted;
- }
-
- public int totalTests() {
- final TestOccurrencesRef testOccurrences = firstBuildInfo.testOccurrences;
-
- if (testOccurrences == null)
- return 0;
-
- final Integer cnt = testOccurrences.count;
-
- return cnt == null ? 0 : cnt;
- }
-
public String getPrintableStatusString() {
StringBuilder builder = new StringBuilder();
builder.append("\t[").append(suiteName()).append("]\t");
@@ -343,16 +321,17 @@ public class MultBuildRunCtx implements ISuiteResults {
return stat == null ? null : stat.getSourceUpdateDuration();
}
+
+
+ @Nullable public String suiteName() {
+ return buildsStream().findFirst().map(SingleBuildRunCtx::suiteName).orElse(null);
+ }
+
/**
* @return aggregation project ID, such as "Ignite_Tests_20"
*/
@Nullable public String projectId() {
- final BuildType type = firstBuildInfo.getBuildType();
-
- if (type == null)
- return null;
-
- return type.getProjectId();
+ return buildsStream().findFirst().map(SingleBuildRunCtx::projectId).orElse(null);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
index 22ee337..7de870f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
@@ -190,4 +190,12 @@ public class SingleBuildRunCtx implements ISuiteResults {
public Stream<String> getAllTestNames() {
return buildCompacted.getAllTestNames(compactor);
}
+
+ public String suiteName() {
+ return buildCompacted.buildTypeName(compactor);
+ }
+
+ public String projectId() {
+ return buildCompacted.projectId(compactor);
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 4c4219d..1c12d96 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -113,9 +113,10 @@ public class BuildChainProcessor {
final List<Future<Stream<BuildRef>>> phase1Submitted = uniqueBuldsInvolved
.map((buildRef) -> svc.submit(
- () -> replaceWithRecent(teamcity, includeLatestRebuild, unique, buildRef, entryPoints.size())))
+ () -> replaceWithRecent(teamcity, teamcityIgnited, includeLatestRebuild, unique, buildRef, entryPoints.size())))
.collect(Collectors.toList());
+
final List<Future<? extends Stream<? extends BuildRef>>> phase2Submitted = phase1Submitted.stream()
.map(FutureUtil::getResult)
.map((s) -> svc.submit(
@@ -160,14 +161,9 @@ public class BuildChainProcessor {
ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap,
Stream<? extends BuildRef> list) {
list.forEach((BuildRef ref) -> {
- Build build = teamcity.getBuild(ref.getId());
-
- if (build == null || build.isFakeStub())
- return;
-
- MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(build.buildTypeId, k -> new MultBuildRunCtx(build));
+ MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId, k -> new MultBuildRunCtx(ref));
- ctx.addBuild(loadTestsAndProblems(teamcity, build, ctx, teamcityIgnited));
+ ctx.addBuild(loadTestsAndProblems(teamcity, ref, ctx, teamcityIgnited));
});
return list;
@@ -177,15 +173,26 @@ public class BuildChainProcessor {
* Runs deep collection of all related statistics for particular build.
*
* @param tcIgnited
- * @param build Build from history with references to tests.
+ * @param buildRef Build ref from history with references to tests.
* @return Full context.
*/
- public SingleBuildRunCtx loadTestsAndProblems(ITeamcity teamcity, @Nonnull Build build,
+ public SingleBuildRunCtx loadTestsAndProblems(ITeamcity teamcity, @Nonnull BuildRef buildRef,
@Deprecated MultBuildRunCtx mCtx, ITeamcityIgnited tcIgnited) {
- FatBuildCompacted buildCompacted = tcIgnited.getFatBuild(build.getId());
+ FatBuildCompacted buildCompacted = tcIgnited.getFatBuild(buildRef.getId());
+ Build buildFromFat = buildCompacted.toBuild(compactor);
+
+ SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildFromFat, buildCompacted, compactor);
+
+ if (!buildFromFat.isComposite())
+ mCtx.addTests(buildCompacted.getTestOcurrences(compactor).getTests());
- SingleBuildRunCtx ctx = new SingleBuildRunCtx(build, buildCompacted, compactor);
+
+ //todo migrate rest of the method to fat build from TC ignited
+ Build build = teamcity.getBuild(buildRef.getId());
+
+ if (build == null || build.isFakeStub())
+ return ctx;
if (build.problemOccurrences != null)
ctx.setProblems(teamcity.getProblems(build).getProblemsNonNull());
@@ -211,12 +218,6 @@ public class BuildChainProcessor {
}
}
- if (build.testOccurrences != null && !build.isComposite()) {
- TestOccurrencesFull tests = buildCompacted.getTestOcurrences(compactor);
-
- mCtx.addTests(tests.getTests());
- }
-
if (build.statisticsRef != null)
mCtx.setStat(teamcity.getBuildStatistics(build.statisticsRef.href));
@@ -225,23 +226,25 @@ public class BuildChainProcessor {
@NotNull
public static Stream< BuildRef> replaceWithRecent(ITeamcity teamcity,
- LatestRebuildMode includeLatestRebuild,
- Map<Integer, BuildRef> unique, BuildRef buildRef, int cntLimit) {
+ ITeamcityIgnited teamcityIgnited, LatestRebuildMode includeLatestRebuild,
+ Map<Integer, BuildRef> unique, BuildRef buildRef, int cntLimit) {
if (includeLatestRebuild == LatestRebuildMode.NONE)
return Stream.of(buildRef);
final String branch = getBranchOrDefault(buildRef.branchName);
- final List<BuildRef> builds = teamcity.getFinishedBuilds(buildRef.buildTypeId, branch);
+ //todo now it is a full scan without caching, probably it is better to make branch based index query & caching results
+ Stream<BuildRef> history = teamcityIgnited.getBuildHistory(buildRef.buildTypeId, branch)
+ .stream().filter(BuildRef::isNotCancelled).filter(BuildRef::isFinished);
if (includeLatestRebuild == LatestRebuildMode.LATEST) {
- BuildRef recentRef = builds.stream().max(Comparator.comparing(BuildRef::getId)).orElse(buildRef);
+ BuildRef recentRef = history.max(Comparator.comparing(BuildRef::getId)).orElse(buildRef);
return Stream.of(recentRef.isFakeStub() ? buildRef : recentRef);
}
if (includeLatestRebuild == LatestRebuildMode.ALL) {
- return builds.stream()
+ return history
.filter(ref -> !ref.isFakeStub())
.filter(ref -> ensureUnique(unique, ref))
.sorted(Comparator.comparing(BuildRef::getId).reversed())
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
index 668ddf8..f7dfba9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
@@ -33,6 +33,8 @@ public interface ITeamcityIgnited {
public String host();
/**
+ * Retun all builds for branch and suite, without relation to its status.
+ *
* @param buildTypeId Build type identifier.
* @param branchName Branch name.
* @return list of builds in history, includes all statuses: queued, running, etc
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 2dbae37..f3e2484 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -197,7 +197,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
runActualizeBuilds(srvId, false, collect);
// schedule full resync later
- scheduler.invokeLater(this::sheduleResync, 60, TimeUnit.SECONDS);
+ scheduler.invokeLater(this::sheduleResync, 20, TimeUnit.MINUTES);
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 29ec877..4a101a3 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -60,6 +60,7 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
private long queuedDate;
private int projectId = -1;
+ /** Suite Name for this builds. */
private int name = -1;
@Nullable private List<TestCompacted> tests;
@@ -140,8 +141,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
BuildType type = new BuildType();
type.id(res.buildTypeId());
- type.name(compactor.getStringFromId(name));
- type.projectId(compactor.getStringFromId(projectId));
+ type.name(buildTypeName(compactor));
+ type.projectId(projectId(compactor));
res.setBuildType(type);
if (tests != null) {
@@ -280,4 +281,12 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
return tests.stream().map(t -> t.getTestName(compactor));
}
+
+ public String buildTypeName(IStringCompactor compactor) {
+ return compactor.getStringFromId(name);
+ }
+
+ public String projectId(IStringCompactor compactor) {
+ return compactor.getStringFromId(projectId);
+ }
}