You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/25 15:09:11 UTC
[ignite-teamcity-bot] branch ignite-9848-load-all-builds updated:
Details storage with snappy added
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-load-all-builds by this push:
new faafe94 Details storage with snappy added
faafe94 is described below
commit faafe94093c8c735360a3d971d4cf6687edd9d99
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Oct 25 18:08:56 2018 +0300
Details storage with snappy added
---
ignite-tc-helper-web/build.gradle | 2 +
.../main/java/org/apache/ignite/ci/ITeamcity.java | 1 +
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 1 +
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 35 ++++---
.../ignite/ci/analysis/SingleBuildRunCtx.java | 14 ++-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 1 +
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 2 +-
.../ignited/fatbuild/FatBuildCompacted.java | 25 +++--
.../teamcity/ignited/fatbuild/TestCompacted.java | 109 +++++++++++++++++++--
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 2 +-
10 files changed, 159 insertions(+), 33 deletions(-)
diff --git a/ignite-tc-helper-web/build.gradle b/ignite-tc-helper-web/build.gradle
index 41adbc0..7a1382b 100644
--- a/ignite-tc-helper-web/build.gradle
+++ b/ignite-tc-helper-web/build.gradle
@@ -65,6 +65,8 @@ dependencies {
compile group: 'javax.inject', name: 'javax.inject', version: '1'
compile group: 'com.google.inject', name: 'guice', version: '4.2.0'
+
+ compile group: 'org.xerial.snappy', name: 'snappy-java', version: '1.1.7.2'
}
processResources {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 6fa106c..17eabb1 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -155,6 +155,7 @@ public interface ITeamcity extends ITeamcityConn {
Statistics getBuildStatistics(String href);
+ @Deprecated
CompletableFuture<TestOccurrenceFull> getTestFull(String href);
Change getChange(String href);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 11d0e04..47e9168 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -1122,6 +1122,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
}
/** {@inheritDoc} */
+ @AutoProfiling
@Override public void calculateBuildStatistic(SingleBuildRunCtx ctx) {
if (calculatedStatistic().containsKey(ctx.buildId()))
return;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index fe0fe8b..13d7956 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -88,16 +88,20 @@ public class MultBuildRunCtx implements ISuiteResults {
}
public Stream<String> getCriticalFailLastStartedTest() {
- return builds.stream().map(SingleBuildRunCtx::getCriticalFailLastStartedTest).filter(Objects::nonNull);
+ return buildsStream().map(SingleBuildRunCtx::getCriticalFailLastStartedTest).filter(Objects::nonNull);
+ }
+
+ public Stream<SingleBuildRunCtx> buildsStream() {
+ return builds.stream();
}
public Stream<Integer> getBuildsWithThreadDump() {
- return builds.stream().map(SingleBuildRunCtx::getBuildIdIfHasThreadDump).filter(Objects::nonNull);
+ return buildsStream().map(SingleBuildRunCtx::getBuildIdIfHasThreadDump).filter(Objects::nonNull);
}
public Stream<Map<String, TestLogCheckResult>> getLogsCheckResults() {
- return builds.stream().map(SingleBuildRunCtx::getTestLogCheckResult).filter(Objects::nonNull);
+ return buildsStream().map(SingleBuildRunCtx::getTestLogCheckResult).filter(Objects::nonNull);
}
public String suiteId() {
@@ -129,7 +133,7 @@ public class MultBuildRunCtx implements ISuiteResults {
}
@NotNull public Stream<ProblemOccurrence> allProblemsInAllBuilds() {
- return builds.stream().flatMap(SingleBuildRunCtx::getProblemsStream);
+ return buildsStream().flatMap(SingleBuildRunCtx::getProblemsStream);
}
public List<SingleBuildRunCtx> getBuilds() {
@@ -141,7 +145,7 @@ public class MultBuildRunCtx implements ISuiteResults {
}
private long getExecutionTimeoutCount() {
- return builds.stream().filter(ISuiteResults::hasTimeoutProblem).count();
+ return buildsStream().filter(ISuiteResults::hasTimeoutProblem).count();
}
public boolean hasJvmCrashProblem() {
@@ -149,7 +153,7 @@ public class MultBuildRunCtx implements ISuiteResults {
}
public long getJvmCrashProblemCount() {
- return builds.stream().filter(ISuiteResults::hasJvmCrashProblem).count();
+ return buildsStream().filter(ISuiteResults::hasJvmCrashProblem).count();
}
public boolean hasOomeProblem() {
@@ -161,16 +165,16 @@ public class MultBuildRunCtx implements ISuiteResults {
}
private long getExitCodeProblemsCount() {
- return builds.stream().filter(ISuiteResults::hasExitCodeProblem).count();
+ return buildsStream().filter(ISuiteResults::hasExitCodeProblem).count();
}
private long getOomeProblemCount() {
- return builds.stream().filter(ISuiteResults::hasOomeProblem).count();
+ return buildsStream().filter(ISuiteResults::hasOomeProblem).count();
}
public int failedTests() {
- return (int)tests.values().stream().mapToInt(MultTestFailureOccurrences::failuresCount)
- .filter(cnt -> cnt > 0).count();
+ return (int)buildsStream().flatMap(ctx -> ctx.getFailedNotMutedTestNames().stream()).distinct().count();
+
}
public int mutedTests() {
@@ -348,11 +352,13 @@ public class MultBuildRunCtx implements ISuiteResults {
* @param testOccurrenceInBuildId, something like: 'id:15666,build:(id:1093907)'
* @param fullFut
*/
+ @Deprecated
public void addTestInBuildToTestFull(String testOccurrenceInBuildId,
CompletableFuture<TestOccurrenceFull> fullFut) {
testFullMap.put(testOccurrenceInBuildId, fullFut);
}
+ @Deprecated
private Optional<TestOccurrenceFull> getFullTest(String testOccurrenceInBuildId) {
return Optional.ofNullable(testFullMap.get(testOccurrenceInBuildId))
.flatMap(fut ->
@@ -403,6 +409,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return val == null ? 0 : val.intValue();
}
+ @Deprecated
public Stream<TestOccurrenceFull> getFullTests(ITestFailureOccurrences occurrence) {
return occurrence.getOccurrenceIds()
.map(this::getFullTest)
@@ -414,7 +421,7 @@ public class MultBuildRunCtx implements ISuiteResults {
* @return Username's stream for users introduced changes in this commit
*/
public Stream<String> lastChangeUsers() {
- return builds.stream()
+ return buildsStream()
.flatMap(k -> k.getChanges().stream())
.map(change -> change.username)
.filter(Objects::nonNull);
@@ -426,7 +433,7 @@ public class MultBuildRunCtx implements ISuiteResults {
Stream<? extends Future<?>> stream3 = testFullMap.values().stream();
- Stream<? extends Future<?>> stream4 = builds.stream().flatMap(SingleBuildRunCtx::getFutures);
+ Stream<? extends Future<?>> stream4 = buildsStream().flatMap(SingleBuildRunCtx::getFutures);
return
concat(
@@ -439,7 +446,7 @@ public class MultBuildRunCtx implements ISuiteResults {
*/
public boolean isComposite() {
return !builds.isEmpty()
- && builds.stream().allMatch(SingleBuildRunCtx::isComposite);
+ && buildsStream().allMatch(SingleBuildRunCtx::isComposite);
}
/**
@@ -450,6 +457,6 @@ public class MultBuildRunCtx implements ISuiteResults {
}
public Stream<LogCheckResult> getLogChecksIfFinished() {
- return builds.stream().map(SingleBuildRunCtx::getLogCheckIfFinished).filter(Objects::nonNull);
+ return buildsStream().map(SingleBuildRunCtx::getLogCheckIfFinished).filter(Objects::nonNull);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
index 0bf4d46..5456e6a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.analysis;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import java.util.ArrayList;
import java.util.List;
@@ -27,7 +28,6 @@ import java.util.concurrent.Future;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import org.apache.ignite.ci.tcmodel.changes.Change;
-import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
@@ -64,6 +64,7 @@ public class SingleBuildRunCtx implements ISuiteResults {
}
public Integer buildId() {
+ Preconditions.checkNotNull(buildCompacted);
return buildCompacted.id() < 0 ? null : buildCompacted.id();
}
@@ -158,14 +159,16 @@ public class SingleBuildRunCtx implements ISuiteResults {
return changes;
}
+ @Deprecated
public void setTests(List<TestOccurrence> tests) {
this.tests = tests;
}
- public List<TestOccurrence> getTests() {
- return tests;
+ public List<? extends TestOccurrence> getTests() {
+ return buildCompacted.getTestOcurrences(compactor).getTests();
}
+
@Nonnull Stream<? extends Future<?>> getFutures() {
return logCheckResultFut == null ? Stream.empty() : Stream.of((Future<?>)logCheckResultFut);
}
@@ -177,4 +180,9 @@ public class SingleBuildRunCtx implements ISuiteResults {
public String getBranch() {
return compactor.getStringFromId(buildCompacted.branchName());
}
+
+
+ public List<String> getFailedNotMutedTestNames() {
+ return buildCompacted.getFailedNotMutedTestNames(compactor);
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index e34d4c5..2febb69 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -188,6 +188,7 @@ public class BuildChainProcessor {
@Deprecated MultBuildRunCtx mCtx, ITeamcityIgnited tcIgnited) {
FatBuildCompacted buildCompacted = tcIgnited.getFatBuild(build.getId());
+
SingleBuildRunCtx ctx = new SingleBuildRunCtx(build, buildCompacted, compactor);
if (build.problemOccurrences != null)
ctx.setProblems(teamcity.getProblems(build).getProblemsNonNull());
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 922333d..2552108 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -102,7 +102,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Override public FatBuildCompacted getFatBuild(int buildId) {
FatBuildCompacted buildPersisted = fatBuildDao.getFatBuild(srvIdMaskHigh, buildId);
- if (buildPersisted != null)
+ if (buildPersisted != null && !buildPersisted.isOutdatedEntityVersion())
return buildPersisted;
//todo some sort of locking to avoid double requests
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 9f3bbdc..0d588ec 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -19,18 +19,17 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild;
import com.google.common.base.Objects;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collections;
import java.util.List;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
import org.apache.ignite.ci.analysis.IVersionedEntity;
import org.apache.ignite.ci.db.Persisted;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.jetbrains.annotations.Nullable;
@@ -216,16 +215,16 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
/**
* @param compactor Compactor.
*/
- public TestOccurrences getTestOcurrences(IStringCompactor compactor) {
+ public TestOccurrencesFull getTestOcurrences(IStringCompactor compactor) {
if (tests == null)
- return new TestOccurrences();
+ return new TestOccurrencesFull();
- List<TestOccurrence> res = new ArrayList<>();
+ List<TestOccurrenceFull> res = new ArrayList<>();
for (TestCompacted compacted : tests)
res.add(compacted.toTestOccurrence(compactor, id()));
- TestOccurrences testOccurrences = new TestOccurrences();
+ TestOccurrencesFull testOccurrences = new TestOccurrencesFull();
testOccurrences.setTests(res);
testOccurrences.count = res.size();
@@ -266,4 +265,14 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
return flag != null && flag;
}
+
+ public List<String> getFailedNotMutedTestNames(IStringCompactor compactor) {
+ if (tests == null)
+ return Collections.emptyList();
+
+ return tests.stream()
+ .filter(t -> t.isFailedButNotMuted(compactor))
+ .map(t -> t.getTestName(compactor))
+ .collect(Collectors.toList());
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index 15d9b39..ad3ab08 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -19,13 +19,19 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild;
import com.google.common.base.Objects;
import com.google.common.base.Strings;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.BitSet;
import org.apache.ignite.ci.analysis.RunStat;
+import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
+import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.xerial.snappy.Snappy;
/**
*
@@ -45,10 +51,14 @@ public class TestCompacted {
private BitSet flags = new BitSet();
/** Test global, can be used for references. */
- private long testId;
+ private long testId = 0;
/** Actual build id. */
- private int actualBuildId;
+ private int actualBuildId = -1;
+
+ /** Snappy comressed test log Details. */
+ private byte[] details;
+
/** Logger. */
private static final Logger logger = LoggerFactory.getLogger(TestCompacted.class);
@@ -82,6 +92,22 @@ public class TestCompacted {
setFlag(CUR_MUTED_F, testOccurrence.currentlyMuted);
setFlag(CUR_INV_F, testOccurrence.currentlyInvestigated);
setFlag(IGNORED_F, testOccurrence.ignored);
+
+ if (testOccurrence.build != null && testOccurrence.build.getId() != null)
+ actualBuildId = testOccurrence.build.getId();
+
+ if (testOccurrence.test != null && testOccurrence.test.id != null)
+ testId = testOccurrence.test.id;
+
+ String details = testOccurrence.details;
+ if (!Strings.isNullOrEmpty(details)) {
+ try {
+ this.details = Snappy.compress(details.getBytes(StandardCharsets.UTF_8));
+ }
+ catch (Exception e) {
+ logger.error("Snappy.compress failed: " + e.getMessage(), e);
+ }
+ }
}
private void setFlag(int off, Boolean val) {
@@ -105,8 +131,8 @@ public class TestCompacted {
return flags.get(off + 1);
}
- public TestOccurrence toTestOccurrence(IStringCompactor compactor, int buildId) {
- TestOccurrence occurrence = new TestOccurrence();
+ public TestOccurrenceFull toTestOccurrence(IStringCompactor compactor, int buildId) {
+ TestOccurrenceFull occurrence = new TestOccurrenceFull();
String fullStrId = "id:" +
idInBuild() + ",build:(id:" +
@@ -118,14 +144,60 @@ public class TestCompacted {
occurrence.status = compactor.getStringFromId(status);
occurrence.href = "/app/rest/latest/testOccurrences/" + fullStrId;
- occurrence.muted = getFlag(MUTED_F);
+ occurrence.muted = getMutedFlag();
occurrence.currentlyMuted = getFlag(CUR_MUTED_F);
occurrence.currentlyInvestigated = getFlag(CUR_INV_F);
- occurrence.ignored = getFlag(IGNORED_F);
+ occurrence.ignored = getIgnoredFlag();
+
+ if (actualBuildId > 0) {
+ BuildRef buildRef = new BuildRef();
+
+ buildRef.setId(actualBuildId);
+
+ occurrence.build = buildRef;
+ }
+
+ if (testId != 0) {
+ TestRef test = new TestRef();
+
+ test.id = testId;
+
+ occurrence.test = test;
+ }
+
+ occurrence.details = getDetailsText();
return occurrence;
}
+ /**
+ *
+ */
+ @Nullable public String getDetailsText() {
+ if (details == null)
+ return "";
+
+ try {
+ byte[] uncompressed = Snappy.uncompress(details);
+ return new String(uncompressed, StandardCharsets.UTF_8);
+ }
+ catch (IOException e) {
+
+ logger.error("Snappy.uncompress failed: " + e.getMessage(), e);
+ return null;
+
+ }
+
+ }
+
+ public Boolean getIgnoredFlag() {
+ return getFlag(IGNORED_F);
+ }
+
+ public Boolean getMutedFlag() {
+ return getFlag(MUTED_F);
+ }
+
private int idInBuild() {
return idInBuild;
}
@@ -148,4 +220,29 @@ public class TestCompacted {
@Override public int hashCode() {
return Objects.hashCode(idInBuild, name, status, duration, flags);
}
+
+
+ public boolean isFailedButNotMuted(IStringCompactor compactor) {
+ return isFailedTest(compactor) && !(isMutedTest() || isIgnoredTest());
+ }
+
+ private boolean isIgnoredTest() {
+ Boolean flag = getIgnoredFlag();
+
+ return flag != null && flag;
+ }
+
+ private boolean isMutedTest() {
+ Boolean flag = getMutedFlag();
+
+ return flag != null && flag;
+ }
+
+ public boolean isFailedTest(IStringCompactor compactor) {
+ return compactor.getStringId(TestOccurrence.STATUS_SUCCESS) != status;
+ }
+
+ public String getTestName(IStringCompactor compactor) {
+ return compactor.getStringFromId(name);
+ }
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 05445c5..1543f76 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -279,7 +279,7 @@ public class IgnitedTcInMemoryIntegrationTest {
saveTmpFile(refBuild, "src/test/tmp/buildRef.xml");
saveTmpFile(actBuild, "src/test/tmp/buildAct.xml");
- TestOccurrences testsAct = fatBuild.getTestOcurrences(compactor);
+ TestOccurrencesFull testsAct = fatBuild.getTestOcurrences(compactor);
saveTmpFile(testsRef, "src/test/tmp/testListRef.xml");
saveTmpFile(testsAct, "src/test/tmp/testListAct.xml");