You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/27 14:34:44 UTC
[ignite-teamcity-bot] 01/01: IGNITE-10030 Started Background upload
of changes, problems, and statistics
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10030
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 2d6f65709213479fd2affb1a29df119d2bbff54b
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 27 17:34:39 2018 +0300
IGNITE-10030 Started Background upload of changes, problems, and statistics
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 4 ---
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 27 ++++--------------
.../apache/ignite/ci/IgniteTeamcityConnection.java | 21 +++++++-------
.../ignited/fatbuild/FatBuildCompacted.java | 10 +++++++
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 18 ++++++------
.../ignited/fatbuild/ProactiveFatBuildSync.java | 33 ++++++++++++++++------
.../ignite/ci/teamcity/pure/ITeamcityConn.java | 3 ++
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 9 +++++-
.../src/test/resources/problemList.xml | 3 ++
9 files changed, 74 insertions(+), 54 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 12446b9..79c1b2b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -105,10 +105,6 @@ public interface ITeamcity extends ITeamcityConn {
/** */
@Deprecated
- CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String branch);
-
- /** */
- @Deprecated
CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch);
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 3487566..8cdf0cd 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -142,12 +142,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
/** cached loads of queued builds for branch. */
private ConcurrentMap<String, CompletableFuture<List<BuildRef>>> queuedBuildsFuts = new ConcurrentHashMap<>();
- /** cached running builds for branch. */
- private ConcurrentMap<String, Expirable<List<BuildRef>>> runningBuilds = new ConcurrentHashMap<>();
-
- /** cached loads of running builds for branch. */
- private ConcurrentMap<String, CompletableFuture<List<BuildRef>>> runningBuildsFuts = new ConcurrentHashMap<>();
-
/** */
private ConcurrentMap<SuiteInBranch, Long> lastQueuedHistory = new ConcurrentHashMap<>();
@@ -667,20 +661,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
}
- /** {@inheritDoc} */
- @Override public CompletableFuture<List<BuildRef>> getRunningBuilds(String branch) {
- int defaultSecs = 60;
- int secondsUseCached = getTriggerRelCacheValidSecs(defaultSecs);
-
- return loadAsyncIfAbsentOrExpired(
- runningBuilds,
- Strings.nullToEmpty(branch),
- runningBuildsFuts,
- teamcity::getRunningBuilds,
- secondsUseCached,
- secondsUseCached == defaultSecs);
- }
-
public static int getTriggerRelCacheValidSecs(int defaultSecs) {
long msSinceTrigger = System.currentTimeMillis() - lastTriggerMs;
long secondsSinceTrigger = TimeUnit.MILLISECONDS.toSeconds(msSinceTrigger);
@@ -1191,12 +1171,17 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Override public Build triggerBuild(String buildTypeId, String branchName, boolean cleanRebuild, boolean queueAtTop) {
+ @Override public Build triggerBuild(String buildTypeId, @NotNull String branchName, boolean cleanRebuild, boolean queueAtTop) {
lastTriggerMs = System.currentTimeMillis();
return teamcity.triggerBuild(buildTypeId, branchName, cleanRebuild, queueAtTop);
}
+ @Override
+ public ProblemOccurrences getProblems(int buildId) {
+ return teamcity.getProblems(buildId);
+ }
+
/** {@inheritDoc} */
@Override public void setAuthToken(String tok) {
teamcity.setAuthToken(tok);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 12810df..4a801ff 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -272,10 +272,10 @@ public class IgniteTeamcityConnection implements ITeamcity {
/** {@inheritDoc} */
@AutoProfiling
@Override public Build triggerBuild(
- String buildTypeId,
- @Nonnull String branchName,
- boolean cleanRebuild,
- boolean queueAtTop
+ String buildTypeId,
+ @NotNull @Nonnull String branchName,
+ boolean cleanRebuild,
+ boolean queueAtTop
) {
String triggeringOptions =
" <triggeringOptions" +
@@ -317,6 +317,13 @@ public class IgniteTeamcityConnection implements ITeamcity {
}
}
+ @Override
+ public ProblemOccurrences getProblems(int buildId) {
+ return getJaxbUsingHref("app/rest/latest/problemOccurrences" +
+ "?locator=build:(id:" + buildId + ")" +
+ "&fields=problemOccurrence(id,type,identity,href,details,build(id))", ProblemOccurrences.class);
+ }
+
private CompletableFuture<List<File>> unzip(CompletableFuture<File> zipFileFut) {
return zipFileFut.thenApplyAsync(ZipUtil::unZipToSameFolder, executor);
}
@@ -549,12 +556,6 @@ public class IgniteTeamcityConnection implements ITeamcity {
/** {@inheritDoc} */
@Override
- @AutoProfiling public CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String branch) {
- return supplyAsync(() -> getBuildsInState(null, branch, BuildRef.STATE_RUNNING), executor);
- }
-
- /** {@inheritDoc} */
- @Override
@AutoProfiling public CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch) {
return supplyAsync(() -> getBuildsInState(null, branch, BuildRef.STATE_QUEUED), executor);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 231ed5d..bcad28e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild;
import com.google.common.base.Objects;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
import org.apache.ignite.ci.analysis.IVersionedEntity;
@@ -27,6 +28,7 @@ import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
@@ -297,4 +299,12 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
public String projectId(IStringCompactor compactor) {
return compactor.getStringFromId(projectId);
}
+
+ public List<ProblemOccurrence> problems(IStringCompactor compactor) {
+ return Collections.emptyList(); //todo
+ }
+
+ public void addProblems(IStringCompactor compactor, List<ProblemOccurrence> problems) {
+ //todo
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index afe4812..6469311 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -24,20 +24,16 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-import javax.cache.Cache;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.validation.constraints.NotNull;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
-import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
-import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
-import org.apache.ignite.internal.util.GridIntList;
import org.jetbrains.annotations.Nullable;
/**
@@ -68,14 +64,16 @@ public class FatBuildDao {
* @param buildId
* @param build Build data.
* @param tests TestOccurrences one or several pages.
+ * @param problems
* @param existingBuild existing version of build in the DB.
* @return Fat Build saved (if modifications detected), otherwise null.
*/
public FatBuildCompacted saveBuild(long srvIdMaskHigh,
- int buildId,
- @NotNull Build build,
- List<TestOccurrencesFull> tests,
- @Nullable FatBuildCompacted existingBuild) {
+ int buildId,
+ @NotNull Build build,
+ @NotNull List<TestOccurrencesFull> tests,
+ @NotNull List<ProblemOccurrence> problems,
+ @Nullable FatBuildCompacted existingBuild) {
Preconditions.checkNotNull(buildsCache, "init() was not called");
Preconditions.checkNotNull(build, "build can't be null");
@@ -84,6 +82,8 @@ public class FatBuildDao {
for (TestOccurrencesFull next : tests)
newBuild.addTests(compactor, next.getTests());
+ newBuild.addProblems(compactor, problems);
+
if (existingBuild == null || !existingBuild.equals(newBuild)) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 8eac870..e916891 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -22,6 +22,8 @@ import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
@@ -213,18 +215,27 @@ public class ProactiveFatBuildSync {
Build build;
List<TestOccurrencesFull> tests = new ArrayList<>();
+ List<ProblemOccurrence> problems;
try {
build = conn.getBuild(buildId);
- String nextHref = null;
- do {
- boolean testDtls = !build.isComposite(); // don't query test details for compoite
- TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls);
- nextHref = page.nextHref();
+ if(build.testOccurrences!=null) {
+ String nextHref = null;
+ do {
+ boolean testDtls = !build.isComposite(); // don't query test details for compoite
+ TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls);
+ nextHref = page.nextHref();
- tests.add(page);
+ tests.add(page);
+ }
+ while (!Strings.isNullOrEmpty(nextHref));
}
- while (!Strings.isNullOrEmpty(nextHref));
+
+ if (build.problemOccurrences != null) {
+ ProblemOccurrences po = conn.getProblems(buildId);
+ problems = new ArrayList<>(po.getProblemsNonNull());
+ } else
+ problems = Collections.emptyList();
}
catch (Exception e) {
if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
@@ -237,9 +248,13 @@ public class ProactiveFatBuildSync {
build.setCancelled();
tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor));
+
+ problems = existingBuild.problems(compactor);
}
- else
+ else {
build = Build.createFakeStub();
+ problems = Collections.emptyList();
+ }
} else {
logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
@@ -251,6 +266,6 @@ public class ProactiveFatBuildSync {
//if we are here because of some sort of outdated version of build,
// new save will be performed with new entity version for compacted build
- return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, existingBuild);
+ return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, problems, existingBuild);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
index f832088..9737011 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
@@ -23,6 +23,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
/**
@@ -59,4 +60,6 @@ public interface ITeamcityConn {
* @param queueAtTop Put at the top of the build queue.
*/
public Build triggerBuild(String buildTypeId, @Nonnull String branchName, boolean cleanRebuild, boolean queueAtTop);
+
+ ProblemOccurrences getProblems(int buildId);
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 173a959..6427cb4 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -40,6 +40,8 @@ import org.apache.ignite.ci.di.scheduler.NoOpSheduler;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
+import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
@@ -252,6 +254,7 @@ public class IgnitedTcInMemoryIntegrationTest {
public void testFatBuild() throws JAXBException, IOException {
Build refBuild = jaxbTestXml("/build.xml", Build.class);
TestOccurrencesFull testsRef = jaxbTestXml("/testList.xml", TestOccurrencesFull.class);
+ ProblemOccurrences problemsList = jaxbTestXml("/problemList.xml", ProblemOccurrences.class);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override protected void configure() {
@@ -265,7 +268,8 @@ public class IgnitedTcInMemoryIntegrationTest {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(APACHE);
List<TestOccurrencesFull> occurrences = Collections.singletonList(testsRef);
- FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh, refBuild.getId(), refBuild, occurrences, null);
+ FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh, refBuild.getId(), refBuild, occurrences,
+ problemsList.getProblemsNonNull(), null);
assertNotNull(buildCompacted);
FatBuildCompacted fatBuild = stor.getFatBuild(srvIdMaskHigh, 2153237);
@@ -300,6 +304,9 @@ public class IgnitedTcInMemoryIntegrationTest {
Set<String> testNamesRef = new TreeSet<>();
testsRef.getTests().forEach(testOccurrence -> testNamesRef.add(testOccurrence.name));
assertEquals(testNamesRef, testNamesAct);
+
+ final List<ProblemOccurrence> problems = buildCompacted.problems(compactor);
+ assertEquals(1, problems.size());
}
public void saveTmpFile(Object obj, String name) throws IOException, JAXBException {
diff --git a/ignite-tc-helper-web/src/test/resources/problemList.xml b/ignite-tc-helper-web/src/test/resources/problemList.xml
new file mode 100644
index 0000000..6bf4ca9
--- /dev/null
+++ b/ignite-tc-helper-web/src/test/resources/problemList.xml
@@ -0,0 +1,3 @@
+<problemOccurrences count="1" href="http://ci.ignite.apache.org:443/app/rest/problemOccurrences?locator=build:(id:2153237)">
+ <problemOccurrence id="problem:(id:3733),build:(id:2153237)" type="TC_FAILED_TESTS" identity="TC_FAILED_TESTS_bt1053" href="/app/rest/latest/problemOccurrences/problem:(id:3733),build:(id:2153237)"/>
+</problemOccurrences>
\ No newline at end of file