You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/27 15:35:58 UTC
[ignite-teamcity-bot] branch ignite-10030 updated (2d6f657 ->
e24abae)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch ignite-10030
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git.
from 2d6f657 IGNITE-10030 Started Background upload of changes, problems, and statistics
new 0144d97 IGNITE-10030 problems done in tests: Background upload of changes, problems, and statistics
new e24abae IGNITE-10030 problems done in prod: Background upload of changes, problems, and statistics
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../main/java/org/apache/ignite/ci/ITeamcity.java | 5 +-
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 3 +
.../apache/ignite/ci/IgniteTeamcityConnection.java | 1 +
.../apache/ignite/ci/analysis/FullChainRunCtx.java | 12 --
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 53 +++----
.../ignite/ci/analysis/SingleBuildRunCtx.java | 33 ++---
.../ci/tcbot/builds/CompareBuildsService.java | 4 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 9 +-
.../tcmodel/result/problems/ProblemOccurrence.java | 8 +
.../ignited/fatbuild/FatBuildCompacted.java | 37 ++++-
.../ignited/fatbuild/ProactiveFatBuildSync.java | 3 +-
.../ignited/fatbuild/ProblemCompacted.java | 163 +++++++++++++++++++++
.../teamcity/ignited/fatbuild/TestCompacted.java | 2 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 6 +-
.../src/test/resources/problemList.xml | 21 ++-
15 files changed, 269 insertions(+), 91 deletions(-)
create mode 100644 ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
[ignite-teamcity-bot] 02/02: IGNITE-10030 problems done in prod:
Background upload of changes, problems, and statistics
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10030
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit e24abae77954d2056389d03931dcbcbca869190f
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 27 18:35:54 2018 +0300
IGNITE-10030 problems done in prod: Background upload of changes, problems, and statistics
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 5 +-
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 3 ++
.../apache/ignite/ci/IgniteTeamcityConnection.java | 1 +
.../apache/ignite/ci/analysis/FullChainRunCtx.java | 12 -----
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 53 ++++++++--------------
.../ignite/ci/analysis/SingleBuildRunCtx.java | 33 ++++----------
.../ci/tcbot/builds/CompareBuildsService.java | 4 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 9 ++--
.../ignited/fatbuild/FatBuildCompacted.java | 13 +++++-
.../ignited/fatbuild/ProactiveFatBuildSync.java | 3 +-
.../ignited/fatbuild/ProblemCompacted.java | 29 ++++++++++--
11 files changed, 78 insertions(+), 87 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 79c1b2b..5cdfaea 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -133,10 +133,7 @@ public interface ITeamcity extends ITeamcityConn {
return "app/rest/latest/builds/id:" + Integer.toString(id);
}
- /**
- * @param build
- * @return
- */
+ @Deprecated
ProblemOccurrences getProblems(BuildRef build);
@Deprecated
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 8cdf0cd..9c321ad 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -1122,6 +1122,9 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
@Override public void calculateBuildStatistic(SingleBuildRunCtx ctx) {
+ if (ctx.buildId() == null)
+ return;
+
if (calculatedStatistic().containsKey(ctx.buildId()))
return;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 4a801ff..09f0e54 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -318,6 +318,7 @@ public class IgniteTeamcityConnection implements ITeamcity {
}
@Override
+ @AutoProfiling
public ProblemOccurrences getProblems(int buildId) {
return getJaxbUsingHref("app/rest/latest/problemOccurrences" +
"?locator=build:(id:" + buildId + ")" +
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java
index 0e04ccb..0d01506 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java
@@ -40,18 +40,6 @@ public class FullChainRunCtx {
fakeStub = chainResults.isFakeStub();
}
- @Deprecated
- public int buildProblems() {
- return (int)buildCfgsResults.stream().filter(MultBuildRunCtx::hasNontestBuildProblem).count();
- }
-
- public int timeoutsOomeCrashBuildProblems() {
- return (int)buildCfgsResults.stream().filter(context ->
- context.hasJvmCrashProblem()
- || context.hasTimeoutProblem()
- || context.hasOomeProblem()).count();
- }
-
public List<MultBuildRunCtx> suites() {
return buildCfgsResults;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index 0411fe2..9076484 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -24,29 +24,23 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
-import org.apache.ignite.ci.tcmodel.conf.BuildType;
+
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
-import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.util.CollectionUtil;
-import org.apache.ignite.ci.util.FutureUtil;
-import org.apache.ignite.ci.util.TimeUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import static java.util.stream.Stream.concat;
-
/**
* Run configuration execution results loaded from different API URLs.
* Includes tests and problem occurrences; if logs processing is done also contains last started test
@@ -55,6 +49,8 @@ public class MultBuildRunCtx implements ISuiteResults {
/** First build info. */
@Nonnull private final BuildRef firstBuildInfo;
+ private IStringCompactor compactor;
+
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();
@@ -77,8 +73,9 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Currently scheduled builds */
private Integer queuedBuildCount;
- public MultBuildRunCtx(@Nonnull final BuildRef buildInfo) {
+ public MultBuildRunCtx(@Nonnull final BuildRef buildInfo, IStringCompactor compactor) {
this.firstBuildInfo = buildInfo;
+ this.compactor = compactor;
}
public Stream<String> getCriticalFailLastStartedTest() {
@@ -106,23 +103,13 @@ public class MultBuildRunCtx implements ISuiteResults {
return firstBuildInfo.buildTypeId;
}
-
- @Deprecated
- //currently used only in old metrics
- public boolean hasNontestBuildProblem() {
- return allProblemsInAllBuilds().anyMatch(problem ->
- !problem.isFailedTests()
- && !problem.isSnapshotDepProblem()
- && !ProblemOccurrence.BUILD_FAILURE_ON_MESSAGE.equals(problem.type));
- //todo what to do with BuildFailureOnMessage, now it is ignored
- }
-
public boolean hasAnyBuildProblemExceptTestOrSnapshot() {
return allProblemsInAllBuilds()
- .anyMatch(p -> !p.isFailedTests() && !p.isSnapshotDepProblem());
+ .anyMatch(p -> !p.isFailedTests(compactor) && !p.isSnapshotDepProblem(compactor));
}
- @NotNull public Stream<ProblemOccurrence> allProblemsInAllBuilds() {
+ @NotNull
+ private Stream<ProblemCompacted> allProblemsInAllBuilds() {
return buildsStream().flatMap(SingleBuildRunCtx::getProblemsStream);
}
@@ -184,21 +171,21 @@ public class MultBuildRunCtx implements ISuiteResults {
addKnownProblemCnt(res, "Exit Code", getExitCodeProblemsCount());
{
- Stream<ProblemOccurrence> stream =
+ Stream<ProblemCompacted> stream =
allProblemsInAllBuilds().filter(p ->
- !p.isFailedTests()
- && !p.isSnapshotDepProblem()
- && !p.isExecutionTimeout()
- && !p.isJvmCrash()
- && !p.isExitCode()
- && !p.isJavaLevelDeadlock()
- && !p.isOome());
- Optional<ProblemOccurrence> bpOpt = stream.findAny();
+ !p.isFailedTests(compactor)
+ && !p.isSnapshotDepProblem(compactor)
+ && !p.isExecutionTimeout(compactor)
+ && !p.isJvmCrash(compactor)
+ && !p.isExitCode(compactor)
+ //&& !p.isJavaLevelDeadlock(compactor)
+ && !p.isOome(compactor));
+ Optional<ProblemCompacted> bpOpt = stream.findAny();
if (bpOpt.isPresent()) {
if (res.length() > 0)
res.append(", ");
- res.append(bpOpt.get().type).append(" ");
+ res.append(bpOpt.get().type(compactor)).append(" ");
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
index f34a53d..4bc48cd 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
@@ -26,12 +26,10 @@ import java.util.concurrent.Future;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import org.apache.ignite.ci.tcmodel.changes.Change;
-import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.jetbrains.annotations.Nullable;
@@ -41,26 +39,22 @@ import org.jetbrains.annotations.Nullable;
public class SingleBuildRunCtx implements ISuiteResults {
/** Build compacted. */
private FatBuildCompacted buildCompacted;
+
/** Compactor. */
private IStringCompactor compactor;
/** Logger check result future. */
private CompletableFuture<LogCheckResult> logCheckResFut;
- /** Build problems occurred during single build run. */
- @Nullable private List<ProblemOccurrence> problems;
-
/** Changes. */
private List<Change> changes = new ArrayList<>();
/**
- * @param build Build.
* @param buildCompacted Build compacted.
* @param compactor Compactor.
*/
- public SingleBuildRunCtx(Build build,
- FatBuildCompacted buildCompacted,
- IStringCompactor compactor) {
+ public SingleBuildRunCtx(FatBuildCompacted buildCompacted,
+ IStringCompactor compactor) {
this.buildCompacted = buildCompacted;
this.compactor = compactor;
}
@@ -78,26 +72,23 @@ public class SingleBuildRunCtx implements ISuiteResults {
}
private long getExecutionTimeoutCount() {
- return getProblemsStream().filter(ProblemOccurrence::isExecutionTimeout).count();
+ return getProblemsStream().filter(p -> p.isExecutionTimeout(compactor)).count();
}
- Stream<ProblemOccurrence> getProblemsStream() {
- if (problems == null)
- return Stream.empty();
-
- return problems.stream().filter(Objects::nonNull);
+ Stream<ProblemCompacted> getProblemsStream() {
+ return buildCompacted.problems().stream();
}
@Override public boolean hasJvmCrashProblem() {
- return getProblemsStream().anyMatch(ProblemOccurrence::isJvmCrash);
+ return getProblemsStream().anyMatch(p -> p.isJvmCrash(compactor));
}
@Override public boolean hasOomeProblem() {
- return getProblemsStream().anyMatch(ProblemOccurrence::isOome);
+ return getProblemsStream().anyMatch(p -> p.isOome(compactor));
}
@Override public boolean hasExitCodeProblem() {
- return getProblemsStream().anyMatch(ProblemOccurrence::isExitCode);
+ return getProblemsStream().anyMatch(p -> p.isExitCode(compactor));
}
@Override public String suiteId() {
@@ -149,10 +140,6 @@ public class SingleBuildRunCtx implements ISuiteResults {
return logCheckRes;
}
- public void setProblems(@Nullable List<ProblemOccurrence> problems) {
- this.problems = problems;
- }
-
public void addChange(Change change) {
if (change.isFakeStub())
return;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
index b383dc5..da17398 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
@@ -26,6 +26,7 @@ import org.apache.ignite.ci.analysis.MultBuildRunCtx;
import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider;
import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider;
@@ -41,6 +42,7 @@ public class CompareBuildsService {
@Inject ICredentialsProv prov;
@Inject BuildChainProcessor bcp;
@Inject ITeamcityIgnitedProvider tcIgnitedProv;
+ @Inject IStringCompactor compactor;
public List<String> tests0(String srvId, Integer buildId ) {
IAnalyticsEnabledTeamcity teamcity = helper.server(srvId, prov);
@@ -74,7 +76,7 @@ public class CompareBuildsService {
else {
logger.info("Loading tests for build {}.", build.getId());
- MultBuildRunCtx buildCtx = new MultBuildRunCtx(build);
+ MultBuildRunCtx buildCtx = new MultBuildRunCtx(build, compactor);
buildCtx.addBuild(bcp.loadTestsAndProblems(tc, build, buildCtx, server));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index cb7a886..7bdd497 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -23,7 +23,6 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -158,7 +157,8 @@ public class BuildChainProcessor {
ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap,
Stream<? extends BuildRef> list) {
list.forEach((BuildRef ref) -> {
- MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId, k -> new MultBuildRunCtx(ref));
+ MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId,
+ k -> new MultBuildRunCtx(ref, compactor));
ctx.addBuild(loadTestsAndProblems(teamcity, ref, ctx, teamcityIgnited));
});
@@ -177,9 +177,8 @@ public class BuildChainProcessor {
@Deprecated MultBuildRunCtx mCtx, ITeamcityIgnited tcIgnited) {
FatBuildCompacted buildCompacted = tcIgnited.getFatBuild(buildRef.getId());
- Build buildFromFat = buildCompacted.toBuild(compactor);
- SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildFromFat, buildCompacted, compactor);
+ SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildCompacted, compactor);
if (!buildCompacted.isComposite())
mCtx.addTests(buildCompacted.getTestOcurrences(compactor).getTests());
@@ -190,8 +189,6 @@ public class BuildChainProcessor {
if (build == null || build.isFakeStub())
return ctx;
- if (build.problemOccurrences != null)
- ctx.setProblems(teamcity.getProblems(build).getProblemsNonNull());
if (build.lastChanges != null) {
for (ChangeRef next : build.lastChanges.changes) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index dbb23b3..3cc65db 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -262,12 +262,14 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
name == that.name &&
Objects.equal(tests, that.tests) &&
Objects.equal(snapshotDeps, that.snapshotDeps) &&
- Objects.equal(flags, that.flags);
+ Objects.equal(flags, that.flags) &&
+ Objects.equal(problems, that.problems);
}
/** {@inheritDoc} */
@Override public int hashCode() {
- return Objects.hashCode(super.hashCode(), _ver, startDate, finishDate, queuedDate, projectId, name, tests, snapshotDeps, flags);
+ return Objects.hashCode(super.hashCode(), _ver, startDate, finishDate, queuedDate, projectId, name, tests,
+ snapshotDeps, flags, problems);
}
/**
@@ -312,6 +314,13 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
.collect(Collectors.toList());
}
+ public List<ProblemCompacted> problems() {
+ if (this.problems == null)
+ return Collections.emptyList();
+
+ return Collections.unmodifiableList(this.problems);
+ }
+
public void addProblems(IStringCompactor compactor, List<ProblemOccurrence> occurrences) {
if (occurrences.isEmpty())
return;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index e916891..3cd337a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -232,8 +232,7 @@ public class ProactiveFatBuildSync {
}
if (build.problemOccurrences != null) {
- ProblemOccurrences po = conn.getProblems(buildId);
- problems = new ArrayList<>(po.getProblemsNonNull());
+ problems = conn.getProblems(buildId).getProblemsNonNull();
} else
problems = Collections.emptyList();
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
index 85e6792..961dbe8 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
@@ -45,6 +45,9 @@ import static org.apache.ignite.ci.analysis.RunStat.extractIdPrefixed;
*
*/
public class ProblemCompacted {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(ProblemCompacted.class);
+
/** Id */
private int id = -1;
private int type = -1;
@@ -55,9 +58,6 @@ public class ProblemCompacted {
// later details may be needed @Nullable private byte[] details;
- /** Logger. */
- private static final Logger logger = LoggerFactory.getLogger(ProblemCompacted.class);
-
/**
* Default constructor.
*/
@@ -74,7 +74,7 @@ public class ProblemCompacted {
try {
final Integer problemId = extractIdPrefixed(problemIdStr, "problem:(id:", ")");
if (problemId != null)
- id = problemId ;
+ id = problemId;
} catch (Exception e) {
logger.error("Failed to handle TC response: " + problemIdStr, e);
}
@@ -139,4 +139,25 @@ public class ProblemCompacted {
public String type(IStringCompactor compactor) {
return compactor.getStringFromId(type);
}
+
+ public boolean isJvmCrash(IStringCompactor compactor) {
+ return compactor.getStringId(ProblemOccurrence.TC_JVM_CRASH) == type;
+ }
+
+ public boolean isOome(IStringCompactor compactor) {
+ return compactor.getStringId(ProblemOccurrence.TC_OOME) == type;
+ }
+
+ public boolean isExitCode(IStringCompactor compactor) {
+ return compactor.getStringId(ProblemOccurrence.TC_EXIT_CODE) == type;
+ }
+
+ public boolean isFailedTests(IStringCompactor compactor) {
+ return compactor.getStringId(ProblemOccurrence.TC_FAILED_TESTS) == type;
+ }
+
+ public boolean isSnapshotDepProblem(IStringCompactor compactor) {
+ return compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR) == type
+ || compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR_BUILD_PROCEEDS_TYPE) == type;
+ }
}
[ignite-teamcity-bot] 01/02: IGNITE-10030 problems done in tests:
Background upload of changes, problems, and statistics
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10030
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 0144d9730939315bea6dac09e62891828e33615c
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 27 18:04:11 2018 +0300
IGNITE-10030 problems done in tests: Background upload of changes, problems, and statistics
---
.../tcmodel/result/problems/ProblemOccurrence.java | 8 ++
.../ignited/fatbuild/FatBuildCompacted.java | 24 +++-
.../ignited/fatbuild/ProblemCompacted.java | 142 +++++++++++++++++++++
.../teamcity/ignited/fatbuild/TestCompacted.java | 2 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 6 +-
.../src/test/resources/problemList.xml | 21 ++-
6 files changed, 195 insertions(+), 8 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java
index 72433f4..efbd438 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java
@@ -77,4 +77,12 @@ public class ProblemOccurrence {
public boolean isJavaLevelDeadlock() {
return JAVA_LEVEL_DEADLOCK.equals(type);
}
+
+ public String id() {
+ return id;
+ }
+
+ public void id(String fullStrId) {
+ this.id = fullStrId;
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index bcad28e..dbb23b3 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.ignite.ci.analysis.IVersionedEntity;
import org.apache.ignite.ci.db.Persisted;
@@ -41,7 +42,7 @@ import org.jetbrains.annotations.Nullable;
@Persisted
public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEntity {
/** Latest version. */
- private static final int LATEST_VERSION = 2;
+ private static final int LATEST_VERSION = 3;
/** Default branch flag offset. */
public static final int DEF_BR_F = 0;
@@ -76,6 +77,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
private BitSet flags = new BitSet();
+ @Nullable private List<ProblemCompacted> problems;
+
/** {@inheritDoc} */
@Override public int version() {
return _ver;
@@ -301,10 +304,23 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
}
public List<ProblemOccurrence> problems(IStringCompactor compactor) {
- return Collections.emptyList(); //todo
+ if (this.problems == null)
+ return Collections.emptyList();
+
+ return this.problems.stream()
+ .map(pc -> pc.toProblemOccurrence(compactor, id()))
+ .collect(Collectors.toList());
}
- public void addProblems(IStringCompactor compactor, List<ProblemOccurrence> problems) {
- //todo
+ public void addProblems(IStringCompactor compactor, List<ProblemOccurrence> occurrences) {
+ if (occurrences.isEmpty())
+ return;
+
+ if (this.problems == null)
+ this.problems = new ArrayList<>();
+
+ occurrences.stream()
+ .map(p -> new ProblemCompacted(compactor, p))
+ .forEach(this.problems::add);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
new file mode 100644
index 0000000..85e6792
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Strings;
+import org.apache.ignite.ci.analysis.RunStat;
+import org.apache.ignite.ci.tcmodel.hist.BuildRef;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
+import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xerial.snappy.Snappy;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.BitSet;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import static org.apache.ignite.ci.analysis.RunStat.extractIdPrefixed;
+
+/**
+ *
+ */
+public class ProblemCompacted {
+ /** Id */
+ private int id = -1;
+ private int type = -1;
+ private int identity = -1;
+
+ /** Actual build id. */
+ private int actualBuildId = -1;
+
+ // later details may be needed @Nullable private byte[] details;
+
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(ProblemCompacted.class);
+
+ /**
+ * Default constructor.
+ */
+ public ProblemCompacted() {
+ }
+
+ /**
+ * @param compactor Compactor.
+ * @param problemOccurrence TestOccurrence.
+ */
+ public ProblemCompacted(IStringCompactor compactor, ProblemOccurrence problemOccurrence) {
+ String problemIdStr = problemOccurrence.id();
+ if (!Strings.isNullOrEmpty(problemIdStr)) {
+ try {
+ final Integer problemId = extractIdPrefixed(problemIdStr, "problem:(id:", ")");
+ if (problemId != null)
+ id = problemId ;
+ } catch (Exception e) {
+ logger.error("Failed to handle TC response: " + problemIdStr, e);
+ }
+ }
+
+ type = compactor.getStringId(problemOccurrence.type);
+ identity = compactor.getStringId(problemOccurrence.identity);
+
+
+ if (problemOccurrence.buildRef != null && problemOccurrence.buildRef.getId() != null)
+ actualBuildId = problemOccurrence.buildRef.getId();
+
+ }
+
+
+ public ProblemOccurrence toProblemOccurrence(IStringCompactor compactor, int buildId) {
+ ProblemOccurrence occurrence = new ProblemOccurrence();
+
+ String fullStrId =
+ "problem:(id:" + id + ")," +
+ " build:(id:" + buildId + ")";
+ occurrence.id(fullStrId);
+ occurrence.type = compactor.getStringFromId(type);
+ occurrence.identity = compactor.getStringFromId(identity);
+ occurrence.href = "problemOccurrences?locator=" + fullStrId;
+
+ if (actualBuildId > 0) {
+ BuildRef buildRef = new BuildRef();
+
+ buildRef.setId(actualBuildId);
+
+ occurrence.buildRef = buildRef;
+ }
+
+ return occurrence;
+ }
+
+ private int id() {
+ return id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ProblemCompacted that = (ProblemCompacted) o;
+ return id == that.id &&
+ type == that.type &&
+ identity == that.identity &&
+ actualBuildId == that.actualBuildId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(id, type, identity, actualBuildId);
+ }
+
+ public boolean isExecutionTimeout(IStringCompactor compactor) {
+ return compactor.getStringId(ProblemOccurrence.TC_EXECUTION_TIMEOUT) == type;
+ }
+
+ public String type(IStringCompactor compactor) {
+ return compactor.getStringFromId(type);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index 37a32bc..573b87c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -55,7 +55,7 @@ public class TestCompacted {
public static final int COMPRESS_TYPE_RFU3 = 10;
public static final int COMPRESS_TYPE_RFU4 = 11;
- /** Id in this build only. Does not idenfity test for its history */
+ /** Id in this build only. Does not identify test for its history */
private int idInBuild = -1;
private int name = -1;
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 6427cb4..88bb399 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -306,7 +306,11 @@ public class IgnitedTcInMemoryIntegrationTest {
assertEquals(testNamesRef, testNamesAct);
final List<ProblemOccurrence> problems = buildCompacted.problems(compactor);
- assertEquals(1, problems.size());
+ assertEquals(2, problems.size());
+
+ assertTrue(problems.stream().anyMatch(ProblemOccurrence::isFailedTests));
+ assertTrue(problems.stream().anyMatch(ProblemOccurrence::isExitCode));
+ assertTrue(problems.stream().noneMatch(ProblemOccurrence::isJvmCrash));
}
public void saveTmpFile(Object obj, String name) throws IOException, JAXBException {
diff --git a/ignite-tc-helper-web/src/test/resources/problemList.xml b/ignite-tc-helper-web/src/test/resources/problemList.xml
index 6bf4ca9..4a770f2 100644
--- a/ignite-tc-helper-web/src/test/resources/problemList.xml
+++ b/ignite-tc-helper-web/src/test/resources/problemList.xml
@@ -1,3 +1,20 @@
-<problemOccurrences count="1" href="http://ci.ignite.apache.org:443/app/rest/problemOccurrences?locator=build:(id:2153237)">
- <problemOccurrence id="problem:(id:3733),build:(id:2153237)" type="TC_FAILED_TESTS" identity="TC_FAILED_TESTS_bt1053" href="/app/rest/latest/problemOccurrences/problem:(id:3733),build:(id:2153237)"/>
+<!-- Syntetic list from:
+ https://ci.ignite.apache.org/app/rest/latest/problemOccurrences?locator=build:(id:2153237)&fields=problemOccurrence(id,type,identity,href,details,build(id))-->
+<problemOccurrences>
+ <problemOccurrence id="problem:(id:3481),build:(id:2136678)" type="TC_EXIT_CODE" identity="Inspection1"
+ href="/app/rest/latest/problemOccurrences/problem:(id:3481),build:(id:2136678)">
+ <details>Process exited with code 1</details>
+ <additionalData>teamcity.process.flow.id=1200755009805251</additionalData>
+ <problem id="3481" type="TC_EXIT_CODE" identity="Inspection1" href="/app/rest/latest/problems/id:3481"/>
+ <build id="2136678" buildTypeId="IgniteTests24Java8_InspectionsAop" number="165" status="FAILURE"
+ state="finished" branchName="refs/heads/master" defaultBranch="true"
+ href="/app/rest/latest/builds/id:2136678"
+ webUrl="http://ci.ignite.apache.org/viewLog.html?buildId=2136678&buildTypeId=IgniteTests24Java8_InspectionsAop"/>
+ </problemOccurrence>
+ <problemOccurrence id="problem:(id:3733),build:(id:2153237)" type="TC_FAILED_TESTS"
+ identity="TC_FAILED_TESTS_bt1053"
+ href="/app/rest/latest/problemOccurrences/problem:(id:3733),build:(id:2153237)">
+ <details>1 failed test detected</details>
+ <build id="2153237"/>
+ </problemOccurrence>
</problemOccurrences>
\ No newline at end of file