You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/27 17:06:35 UTC
[ignite-teamcity-bot] branch ignite-10030 updated: IGNITE-10030
statistics done in prod: Background upload of changes, problems,
and statistics
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10030
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-10030 by this push:
new f6a3be0 IGNITE-10030 statistics done in prod: Background upload of changes, problems, and statistics
f6a3be0 is described below
commit f6a3be031e8dbaa5e0d142ac665cbf7df12d33dc
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 27 20:06:34 2018 +0300
IGNITE-10030 statistics done in prod: Background upload of changes, problems, and statistics
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 14 +--
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 40 ++------
.../apache/ignite/ci/IgniteTeamcityConnection.java | 27 +++---
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 26 ++----
.../ignite/ci/analysis/SingleBuildRunCtx.java | 4 +
.../java/org/apache/ignite/ci/db/DbMigrations.java | 27 +++---
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 3 -
.../ignite/ci/tcmodel/conf/bt/Parameters.java | 5 +
.../apache/ignite/ci/tcmodel/conf/bt/Property.java | 4 +
.../ignite/ci/tcmodel/result/stat/Statistics.java | 8 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 4 +-
.../ignited/fatbuild/FatBuildCompacted.java | 26 ++++--
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 11 ++-
.../ignited/fatbuild/ProactiveFatBuildSync.java | 15 +--
.../ignited/fatbuild/StatisticsCompacted.java | 101 +++++++++++++++++++++
.../ignite/ci/teamcity/pure/ITeamcityConn.java | 5 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 8 +-
17 files changed, 214 insertions(+), 114 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 5cdfaea..dac1546 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -59,16 +59,6 @@ public interface ITeamcity extends ITeamcityConn {
CompletableFuture<List<BuildType>> getProjectSuites(String projectId);
-
- /**
- * @param projectId Suite ID (string without spaces).
- * @param branch Branch in TC identification.
- * @return List of builds in historical order, recent builds coming last.
- */
- default List<BuildRef> getFinishedBuilds(String projectId, String branch) {
- return getFinishedBuilds(projectId, branch, null, null, null);
- };
-
/**
* @param projectId suite ID (string without spaces).
* @param branch Branch name in TC identification.
@@ -77,6 +67,7 @@ public interface ITeamcity extends ITeamcityConn {
* @param sinceBuildId Some build ID in the past to to use as minimal build to export.
* @return list of builds in historical order, recent builds coming last.
*/
+ @Deprecated
List<BuildRef> getFinishedBuilds(String projectId, String branch, Date sinceDate, Date untilDate, Integer sinceBuildId);
/**
@@ -114,6 +105,7 @@ public interface ITeamcity extends ITeamcityConn {
* @param untilDate Until date.
* @return List of build numbers in historical order in date interval, recent builds coming last.
*/
+ @Deprecated
default int[] getBuildNumbersFromHistory(String projectId, String branchNameForHist, Date sinceDate, Date untilDate) {
return getFinishedBuilds(projectId, branchNameForHist, sinceDate, untilDate, null).stream().mapToInt(BuildRef::getId).toArray();
}
@@ -142,8 +134,6 @@ public interface ITeamcity extends ITeamcityConn {
@Deprecated
TestOccurrences getFailedTests(String href, int count, String normalizedBranch);
- Statistics getBuildStatistics(String href);
-
@Deprecated
CompletableFuture<TestOccurrenceFull> getTestFull(String href);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 9c321ad..043154c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -106,7 +106,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
@Deprecated
private static final String TEST_FULL = "testFull";
private static final String BUILD_PROBLEMS = "buildProblems";
- private static final String BUILD_STATISTICS = "buildStatistics";
private static final String BUILD_HIST_FINISHED = "buildHistFinished";
private static final String BUILD_HIST_FINISHED_OR_FAILED = "buildHistFinishedOrFailed";
public static final String BOT_DETECTED_ISSUES = "botDetectedIssues";
@@ -165,10 +164,8 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
buildsFailureRunStatCache(), testRunStatCache(),
testFullCache(),
buildProblemsCache(),
- buildStatisticsCache(),
buildHistCache(),
- buildHistIncFailedCache(),
- testRefsCache());
+ buildHistIncFailedCache());
}
@Override
@@ -249,12 +246,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return getOrCreateCacheV2(ignCacheNme(BUILD_PROBLEMS));
}
- /**
- * @return Build {@link Statistics} instances cache, 32 parts.
- */
- private IgniteCache<String, Statistics> buildStatisticsCache() {
- return getOrCreateCacheV2(ignCacheNme(BUILD_STATISTICS));
- }
/**
@@ -853,26 +844,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Override public Statistics getBuildStatistics(String href) {
- return loadIfAbsent(buildStatisticsCache(),
- href,
- href1 -> {
- try {
- return teamcity.getBuildStatistics(href1);
- }
- catch (Exception e) {
- if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
- e.printStackTrace();
- return new Statistics();// save null result, because persistence may refer to some unexistent build on TC
- }
- else
- throw e;
- }
- });
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
@Override public CompletableFuture<TestOccurrenceFull> getTestFull(String href) {
return CacheUpdateUtil.loadAsyncIfAbsent(
testFullCache(),
@@ -1185,6 +1156,11 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return teamcity.getProblems(buildId);
}
+ @Override
+ public Statistics getStatistics(int buildId) {
+ return teamcity.getStatistics(buildId);
+ }
+
/** {@inheritDoc} */
@Override public void setAuthToken(String tok) {
teamcity.setAuthToken(tok);
@@ -1226,8 +1202,8 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
}
/** {@inheritDoc} */
- @Override public List<BuildRef> getBuildRefs(String fullUrl, AtomicReference<String> nextPage) {
- return teamcity.getBuildRefs(fullUrl, nextPage);
+ @Override public List<BuildRef> getBuildRefsPage(String fullUrl, AtomicReference<String> nextPage) {
+ return teamcity.getBuildRefsPage(fullUrl, nextPage);
}
/** {@inheritDoc} */
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 09f0e54..9435540 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -325,6 +325,12 @@ public class IgniteTeamcityConnection implements ITeamcity {
"&fields=problemOccurrence(id,type,identity,href,details,build(id))", ProblemOccurrences.class);
}
+ @Override
+ @AutoProfiling
+ public Statistics getStatistics(int buildId) {
+ return getJaxbUsingHref("app/rest/latest/builds/id:" + buildId + "/statistics", Statistics.class);
+ }
+
private CompletableFuture<List<File>> unzip(CompletableFuture<File> zipFileFut) {
return zipFileFut.thenApplyAsync(ZipUtil::unZipToSameFolder, executor);
}
@@ -459,12 +465,6 @@ public class IgniteTeamcityConnection implements ITeamcity {
/** {@inheritDoc} */
@AutoProfiling
- @Override public Statistics getBuildStatistics(String href) {
- return getJaxbUsingHref(href, Statistics.class);
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
@Override public CompletableFuture<TestOccurrenceFull> getTestFull(String href) {
return supplyAsync(() -> getJaxbUsingHref(href, TestOccurrenceFull.class), executor);
}
@@ -520,13 +520,6 @@ public class IgniteTeamcityConnection implements ITeamcity {
/** {@inheritDoc} */
@AutoProfiling
- @Override public List<BuildRef> getFinishedBuilds(String projectId, String branch) {
-
- return getFinishedBuilds(projectId, branch, null, null, null);
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
@Override public List<BuildRef> getFinishedBuilds(String projectId,
String branch,
Date sinceDate,
@@ -561,6 +554,7 @@ public class IgniteTeamcityConnection implements ITeamcity {
return supplyAsync(() -> getBuildsInState(null, branch, BuildRef.STATE_QUEUED), executor);
}
+ @Deprecated
private List<BuildRef> getBuildsInState(
@Nullable final String projectId,
@Nullable final String branch,
@@ -579,6 +573,7 @@ public class IgniteTeamcityConnection implements ITeamcity {
@SuppressWarnings("WeakerAccess")
@AutoProfiling
+ @Deprecated
protected List<BuildRef> getBuildsInState(
@Nullable final String projectId,
@Nullable final String branch,
@@ -657,7 +652,7 @@ public class IgniteTeamcityConnection implements ITeamcity {
/** {@inheritDoc} */
@AutoProfiling
- @Override public List<BuildRef> getBuildRefs(String fullUrl, AtomicReference<String> outNextPage) {
+ @Override public List<BuildRef> getBuildRefsPage(String fullUrl, AtomicReference<String> outNextPage) {
String relPath = "app/rest/latest/builds?locator=defaultFilter:false";
String relPathSelected = Strings.isNullOrEmpty(fullUrl) ? relPath : fullUrl;
String url = host + (relPathSelected.startsWith("/") ? relPathSelected.substring(1) : relPathSelected);
@@ -680,10 +675,12 @@ public class IgniteTeamcityConnection implements ITeamcity {
* @param buildId Build id.
* @param testDtls request test details string
*/
- @NotNull public String testsStartHref(int buildId, boolean testDtls) {
+ @NotNull
+ private String testsStartHref(int buildId, boolean testDtls) {
String fieldList = "id,name," +
(testDtls ? "details," : "") +
"status,duration,muted,currentlyMuted,currentlyInvestigated,ignored,test(id),build(id)";
+
return "app/rest/latest/testOccurrences?locator=build:(id:" +
buildId + ")" +
"&fields=testOccurrence(" + fieldList + ")";
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index 9076484..c79ab3b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -17,13 +17,7 @@
package org.apache.ignite.ci.analysis;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
@@ -58,11 +52,6 @@ public class MultBuildRunCtx implements ISuiteResults {
@Deprecated
private final Map<String, MultTestFailureOccurrences> tests = new ConcurrentSkipListMap<>();
- /**
- * Statistics for last build.
- */
- @Nullable private Statistics stat;
-
public void addBuild(SingleBuildRunCtx ctx) {
builds.add(ctx);
}
@@ -266,16 +255,19 @@ public class MultBuildRunCtx implements ISuiteResults {
return firstBuildInfo.branchName;
}
- public void setStat(@Nullable Statistics stat) {
- this.stat = stat;
- }
-
/**
* @return last build duration.
*/
@Nullable
public Long getBuildDuration() {
- return stat == null ? null : stat.getBuildDuration();
+ final OptionalDouble average = buildsStream()
+ .mapToLong(SingleBuildRunCtx::getBuildDuration)
+ .filter(Objects::nonNull)
+ .average();
+ if(average.isPresent())
+ return (long) average.getAsDouble();
+
+ return null;
}
@Nullable public String suiteName() {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
index 4bc48cd..a471a8a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
@@ -193,4 +193,8 @@ public class SingleBuildRunCtx implements ISuiteResults {
public String projectId() {
return buildCompacted.projectId(compactor);
}
+
+ public Long getBuildDuration() {
+ return buildCompacted.buildDuration(compactor);
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 8b06abb..6441ab1 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -42,7 +42,6 @@ import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
-import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.web.rest.build.GetBuildTestFailures;
import org.apache.ignite.ci.web.rest.pr.GetPrTestFailures;
import org.apache.ignite.ci.web.rest.tracked.GetTrackedBranchTestResults;
@@ -64,6 +63,9 @@ public class DbMigrations {
public static final String TESTS = "tests";
@Deprecated
private static final String BUILD_RESULTS = "buildResults";
+
+ private static final String BUILD_STATISTICS = "buildStatistics";
+
public static final String TESTS_COUNT_7700 = ",count:7700";
//V1 caches, 1024 parts
@@ -108,17 +110,15 @@ public class DbMigrations {
}
public void dataMigration(
- IgniteCache<String, TestOccurrences> testOccurrencesCache, Consumer<TestOccurrences> saveTestToStat,
- Consumer<TestOccurrences> saveTestToLatest,
- Cache<String, Build> buildCache, Consumer<Build> saveBuildToStat,
- IgniteCache<SuiteInBranch, RunStat> suiteHistCache,
- IgniteCache<TestInBranch, RunStat> testHistCache,
- Cache<String, TestOccurrenceFull> testFullCache,
- Cache<String, ProblemOccurrences> problemsCache,
- Cache<String, Statistics> buildStatCache,
- Cache<SuiteInBranch, Expirable<List<BuildRef>>> buildHistCache,
- Cache<SuiteInBranch, Expirable<List<BuildRef>>> buildHistInFailedCache,
- Cache<String, TestRef> testRefsCache) {
+ IgniteCache<String, TestOccurrences> testOccurrencesCache, Consumer<TestOccurrences> saveTestToStat,
+ Consumer<TestOccurrences> saveTestToLatest,
+ Cache<String, Build> buildCache, Consumer<Build> saveBuildToStat,
+ IgniteCache<SuiteInBranch, RunStat> suiteHistCache,
+ IgniteCache<TestInBranch, RunStat> testHistCache,
+ Cache<String, TestOccurrenceFull> testFullCache,
+ Cache<String, ProblemOccurrences> problemsCache,
+ Cache<SuiteInBranch, Expirable<List<BuildRef>>> buildHistCache,
+ Cache<SuiteInBranch, Expirable<List<BuildRef>>> buildHistInFailedCache) {
doneMigrations = doneMigrationsCache();
@@ -346,7 +346,6 @@ public class DbMigrations {
}
});
applyV1toV2Migration(PROBLEMS, problemsCache);
- applyV1toV2Migration(STAT, buildStatCache);
applyV1toV2Migration(FINISHED_BUILDS, buildHistCache);
applyV1toV2Migration(FINISHED_BUILDS_INCLUDE_FAILED, buildHistInFailedCache);
@@ -407,6 +406,8 @@ public class DbMigrations {
applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD, TEAMCITY_BUILD_CACHE_NAME_OLD);
applyDestroyIgnCacheMigration(TESTS);
+ applyDestroyIgnCacheMigration(STAT);
+ applyDestroyIgnCacheMigration(BUILD_STATISTICS);
}
private void applyDestroyIgnCacheMigration(String cacheName) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 7bdd497..8318fa1 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -212,9 +212,6 @@ public class BuildChainProcessor {
}
}
- if (build.statisticsRef != null)
- mCtx.setStat(teamcity.getBuildStatistics(build.statisticsRef.href));
-
return ctx;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Parameters.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Parameters.java
index 3f9393a..b0980a7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Parameters.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Parameters.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.tcmodel.conf.bt;
import org.jetbrains.annotations.Nullable;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -42,4 +43,8 @@ public class Parameters {
Objects.equals(property.name, key)).findAny();
return any.map(Property::getValue).orElse(null);
}
+
+ public List<Property> properties() {
+ return Collections.unmodifiableList(this.properties);
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Property.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Property.java
index 107cfed..a3ee651 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Property.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/conf/bt/Property.java
@@ -36,4 +36,8 @@ public class Property {
public String getValue() {
return value;
}
+
+ public String name() {
+ return name;
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/stat/Statistics.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/stat/Statistics.java
index 1a72247..2bbe6fb 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/stat/Statistics.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/stat/Statistics.java
@@ -29,11 +29,14 @@ import org.jetbrains.annotations.Nullable;
@XmlRootElement(name = "properties")
@XmlAccessorType(XmlAccessType.FIELD)
public class Statistics extends Parameters {
+ public static final String BUILD_DURATION = "BuildDuration";
+ public static final String BUILD_STAGE_DURATION_SOURCES_UPDATE = "buildStageDuration:sourcesUpdate";
+
/**
* @return build duration in millis or null.
*/
@Nullable public Long getBuildDuration() {
- String duration = getParameter("BuildDuration");
+ String duration = getParameter(BUILD_DURATION);
if (duration == null)
return null;
@@ -44,11 +47,12 @@ public class Statistics extends Parameters {
* @return source update duration in millis.
*/
@Nullable public Long getSourceUpdateDuration() {
- String duration = getParameter( "buildStageDuration:sourcesUpdate");
+ String duration = getParameter(BUILD_STAGE_DURATION_SOURCES_UPDATE);
if (duration == null)
return null;
return Long.parseLong(duration);
}
+
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 5603b44..1762693 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -211,7 +211,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
protected String runActualizeBuildRefs(String srvId, boolean fullReindex,
@Nullable Set<Integer> mandatoryToReload) {
AtomicReference<String> outLinkNext = new AtomicReference<>();
- List<BuildRef> tcDataFirstPage = conn.getBuildRefs(null, outLinkNext);
+ List<BuildRef> tcDataFirstPage = conn.getBuildRefsPage(null, outLinkNext);
Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh, tcDataFirstPage);
int totalUpdated = buildsUpdated.size();
@@ -228,7 +228,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
while (outLinkNext.get() != null) {
String nextPageUrl = outLinkNext.get();
outLinkNext.set(null);
- List<BuildRef> tcDataNextPage = conn.getBuildRefs(nextPageUrl, outLinkNext);
+ List<BuildRef> tcDataNextPage = conn.getBuildRefsPage(nextPageUrl, outLinkNext);
Set<Long> curChunkBuildsSaved = buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
totalUpdated += curChunkBuildsSaved.size();
buildSync.scheduleBuildsLoad(conn, cacheKeysToBuildIds(curChunkBuildsSaved));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 3cc65db..bbbeb9f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -30,10 +30,12 @@ import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
+import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
@@ -42,7 +44,7 @@ import org.jetbrains.annotations.Nullable;
@Persisted
public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEntity {
/** Latest version. */
- private static final int LATEST_VERSION = 3;
+ private static final int LATEST_VERSION = 4;
/** Default branch flag offset. */
public static final int DEF_BR_F = 0;
@@ -79,6 +81,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
@Nullable private List<ProblemCompacted> problems;
+ @Nullable private StatisticsCompacted statistics;
+
/** {@inheritDoc} */
@Override public int version() {
return _ver;
@@ -106,7 +110,6 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
finishDate = build.getFinishDate() == null ? -1L : build.getFinishDate().getTime();
queuedDate = build.getQueuedDate() == null ? -1L : build.getQueuedDate().getTime();
-
BuildType type = build.getBuildType();
if (type != null) {
projectId = compactor.getStringId(type.getProjectId());
@@ -199,7 +202,6 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
}
}
-
/**
* @param off Offset.
* @param val Value.
@@ -262,14 +264,15 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
name == that.name &&
Objects.equal(tests, that.tests) &&
Objects.equal(snapshotDeps, that.snapshotDeps) &&
- Objects.equal(flags, that.flags) &&
- Objects.equal(problems, that.problems);
+ Objects.equal(flags, that.flags) &&
+ Objects.equal(problems, that.problems) &&
+ Objects.equal(statistics, that.statistics);
}
/** {@inheritDoc} */
@Override public int hashCode() {
return Objects.hashCode(super.hashCode(), _ver, startDate, finishDate, queuedDate, projectId, name, tests,
- snapshotDeps, flags, problems);
+ snapshotDeps, flags, problems, statistics);
}
/**
@@ -321,7 +324,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
return Collections.unmodifiableList(this.problems);
}
- public void addProblems(IStringCompactor compactor, List<ProblemOccurrence> occurrences) {
+ public void addProblems(IStringCompactor compactor,
+ @NotNull List<ProblemOccurrence> occurrences) {
if (occurrences.isEmpty())
return;
@@ -332,4 +336,12 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
.map(p -> new ProblemCompacted(compactor, p))
.forEach(this.problems::add);
}
+
+ public Long buildDuration(IStringCompactor compactor) {
+ return statistics == null ? null : statistics.buildDuration(compactor);
+ }
+
+ public void statistics(IStringCompactor compactor, Statistics statistics) {
+ this.statistics = new StatisticsCompacted(compactor, statistics);
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index 6469311..da8c789 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -32,6 +32,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
+import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.jetbrains.annotations.Nullable;
@@ -65,6 +66,7 @@ public class FatBuildDao {
* @param build Build data.
* @param tests TestOccurrences one or several pages.
* @param problems
+ * @param statistics
* @param existingBuild existing version of build in the DB.
* @return Fat Build saved (if modifications detected), otherwise null.
*/
@@ -72,7 +74,8 @@ public class FatBuildDao {
int buildId,
@NotNull Build build,
@NotNull List<TestOccurrencesFull> tests,
- @NotNull List<ProblemOccurrence> problems,
+ @Nullable List<ProblemOccurrence> problems,
+ @Nullable Statistics statistics,
@Nullable FatBuildCompacted existingBuild) {
Preconditions.checkNotNull(buildsCache, "init() was not called");
Preconditions.checkNotNull(build, "build can't be null");
@@ -82,7 +85,11 @@ public class FatBuildDao {
for (TestOccurrencesFull next : tests)
newBuild.addTests(compactor, next.getTests());
- newBuild.addProblems(compactor, problems);
+ if (problems != null)
+ newBuild.addProblems(compactor, problems);
+
+ if (statistics != null)
+ newBuild.statistics(compactor, statistics);
if (existingBuild == null || !existingBuild.equals(newBuild)) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 3cd337a..a99463d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -23,7 +23,7 @@ import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
-import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
+import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
@@ -215,7 +215,8 @@ public class ProactiveFatBuildSync {
Build build;
List<TestOccurrencesFull> tests = new ArrayList<>();
- List<ProblemOccurrence> problems;
+ List<ProblemOccurrence> problems = null;
+ Statistics statistics = null;
try {
build = conn.getBuild(buildId);
@@ -231,10 +232,11 @@ public class ProactiveFatBuildSync {
while (!Strings.isNullOrEmpty(nextHref));
}
- if (build.problemOccurrences != null) {
+ if (build.problemOccurrences != null)
problems = conn.getProblems(buildId).getProblemsNonNull();
- } else
- problems = Collections.emptyList();
+
+ if(build.statisticsRef!=null)
+ statistics = conn.getStatistics(buildId);
}
catch (Exception e) {
if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
@@ -252,7 +254,6 @@ public class ProactiveFatBuildSync {
}
else {
build = Build.createFakeStub();
- problems = Collections.emptyList();
}
} else {
logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
@@ -265,6 +266,6 @@ public class ProactiveFatBuildSync {
//if we are here because of some sort of outdated version of build,
// new save will be performed with new entity version for compacted build
- return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, problems, existingBuild);
+ return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, problems, statistics, existingBuild);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
new file mode 100644
index 0000000..52f1e91
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+
+import com.google.common.base.Strings;
+import org.apache.ignite.ci.tcmodel.conf.bt.Property;
+import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.internal.util.GridIntList;
+import org.apache.ignite.internal.util.GridLongList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class StatisticsCompacted {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(StatisticsCompacted.class);
+
+ private GridIntList keys;
+ private GridLongList values;
+
+ public StatisticsCompacted() {
+ }
+
+ public StatisticsCompacted(IStringCompactor compactor, Statistics statistics) {
+ final List<Property> properties = statistics.properties();
+ final int size = properties.size();
+ keys = new GridIntList(size);
+ values = new GridLongList(size);
+ int idx = 0;
+ for (Property next : properties) {
+ final String name = next.name();
+ if (Strings.isNullOrEmpty(name))
+ continue;
+
+ final String value = next.getValue();
+ if (Strings.isNullOrEmpty(value))
+ continue;
+ final long val;
+ try {
+ val = Long.parseLong(value);
+ } catch (Exception e) {
+ logger.error("Statistics value is not numeric " + name + " skipped " + e.getMessage(), e);
+ continue;
+ }
+
+ final int stringId = compactor.getStringId(name);
+
+ keys.add(stringId);
+ values.add(val);
+ idx++;
+ }
+ }
+
+ public Long buildDuration(IStringCompactor compactor) {
+ final Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
+ if (buildDurationId == null)
+ return null;
+
+ long value = findPropertyValue(buildDurationId);
+
+ if (value < 0) return null;
+
+ return value;
+ }
+
+ private long findPropertyValue(int propertyCode) {
+ final int size = keys.size();
+
+ long value = -1;
+ for (int i = 0; i < size; i++) {
+ final int nameid = keys.get(i);
+
+ if (nameid != propertyCode)
+ continue;
+
+ if (i >= values.size())
+ break;
+
+ value = values.get(i);
+ }
+
+ return value;
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
index 9737011..27f6fe9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
@@ -24,6 +24,7 @@ import javax.annotation.Nullable;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
+import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
/**
@@ -42,7 +43,7 @@ public interface ITeamcityConn {
public Build getBuild(int buildId);
- public List<BuildRef> getBuildRefs(String fullUrl, AtomicReference<String> nextPage);
+ public List<BuildRef> getBuildRefsPage(String fullUrl, AtomicReference<String> nextPage);
/**
* @param buildId Build id.
@@ -62,4 +63,6 @@ public interface ITeamcityConn {
public Build triggerBuild(String buildTypeId, @Nonnull String branchName, boolean cleanRebuild, boolean queueAtTop);
ProblemOccurrences getProblems(int buildId);
+
+ Statistics getStatistics(int buildId);
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 88bb399..e2649a5 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -42,6 +42,7 @@ import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
+import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
@@ -255,6 +256,7 @@ public class IgnitedTcInMemoryIntegrationTest {
Build refBuild = jaxbTestXml("/build.xml", Build.class);
TestOccurrencesFull testsRef = jaxbTestXml("/testList.xml", TestOccurrencesFull.class);
ProblemOccurrences problemsList = jaxbTestXml("/problemList.xml", ProblemOccurrences.class);
+ Statistics statistics = jaxbTestXml("/statistics.xml", Statistics.class);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override protected void configure() {
@@ -269,7 +271,7 @@ public class IgnitedTcInMemoryIntegrationTest {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(APACHE);
List<TestOccurrencesFull> occurrences = Collections.singletonList(testsRef);
FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh, refBuild.getId(), refBuild, occurrences,
- problemsList.getProblemsNonNull(), null);
+ problemsList.getProblemsNonNull(), statistics, null);
assertNotNull(buildCompacted);
FatBuildCompacted fatBuild = stor.getFatBuild(srvIdMaskHigh, 2153237);
@@ -311,6 +313,10 @@ public class IgnitedTcInMemoryIntegrationTest {
assertTrue(problems.stream().anyMatch(ProblemOccurrence::isFailedTests));
assertTrue(problems.stream().anyMatch(ProblemOccurrence::isExitCode));
assertTrue(problems.stream().noneMatch(ProblemOccurrence::isJvmCrash));
+
+ Long duration = buildCompacted.buildDuration(compactor);
+ assertNotNull(duration);
+ assertTrue(duration>10000L);
}
public void saveTmpFile(Object obj, String name) throws IOException, JAXBException {