You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/29 14:47:41 UTC
[ignite-teamcity-bot] branch ignite-9848-3 updated: IGNITE-9848:
Background upload of all builds from TC to the bot DB
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-3
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-3 by this push:
new 6326920 IGNITE-9848: Background upload of all builds from TC to the bot DB
6326920 is described below
commit 6326920c45aa73cab55d4426dbdd81243cfd3287
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Mon Oct 29 17:47:38 2018 +0300
IGNITE-9848: Background upload of all builds from TC to the bot DB
---
.../java/org/apache/ignite/ci/db/DbMigrations.java | 8 ++--
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 45 +++++++++++++++-------
.../ci/teamcity/ignited/BuildRefCompacted.java | 5 +++
.../ignited/fatbuild/FatBuildCompacted.java | 12 +++++-
.../ignited/fatbuild/ProactiveFatBuildSync.java | 3 --
5 files changed, 51 insertions(+), 22 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 6441ab1..824dc4a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -404,10 +404,10 @@ public class DbMigrations {
applyDestroyIgnCacheMigration(FINISHED_BUILDS_INCLUDE_FAILED);
applyDestroyIgnCacheMigration(TEST_OCCURRENCE_FULL);
- applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD, TEAMCITY_BUILD_CACHE_NAME_OLD);
applyDestroyIgnCacheMigration(TESTS);
applyDestroyIgnCacheMigration(STAT);
applyDestroyIgnCacheMigration(BUILD_STATISTICS);
+ applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD, TEAMCITY_BUILD_CACHE_NAME_OLD);
}
private void applyDestroyIgnCacheMigration(String cacheName) {
@@ -415,12 +415,12 @@ public class DbMigrations {
applyDestroyCacheMigration(cacheName, ignCacheNme);
}
- private void applyDestroyCacheMigration(String dispCacheName, String ignCacheNme) {
+ private void applyDestroyCacheMigration(String dispCacheName, String cacheNme) {
applyMigration("destroy-" + dispCacheName, () -> {
- IgniteCache<Object, Object> cache = ignite.cache(ignCacheNme);
+ IgniteCache<Object, Object> cache = ignite.cache(cacheNme);
if (cache == null) {
- System.err.println("cache not found");
+ System.err.println("cache [" + cacheNme + "] not found");
return;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index fe26954..07f5fe0 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.inject.Inject;
@@ -49,6 +50,7 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
+import org.apache.lucene.util.IntsRef;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
@@ -89,18 +91,14 @@ public class BuildChainProcessor {
if (entryPoints.isEmpty())
return new FullChainRunCtx(Build.createFakeStub());
- BuildRef next = entryPoints.iterator().next();
- Build results = teamcity.getBuild(next.href);
- FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(results);
Map<Integer, BuildRef> unique = new ConcurrentHashMap<>();
+ Map<Integer, FatBuildCompacted> builds = new ConcurrentHashMap<>();
Map<String, MultBuildRunCtx> buildsCtxMap = new ConcurrentHashMap<>();
Stream<? extends BuildRef> uniqueBuldsInvolved = entryPoints.stream()
- .parallel()
- .unordered()
- .flatMap(ref -> dependencies(teamcity, ref)).filter(Objects::nonNull)
- .flatMap(ref -> dependencies(teamcity, ref)).filter(Objects::nonNull)
+ .flatMap(ref -> dependencies(teamcityIgnited, builds, teamcity, ref)).filter(Objects::nonNull)
+ .flatMap(ref -> dependencies(teamcityIgnited, builds, teamcity, ref)).filter(Objects::nonNull)
.filter(ref -> ensureUnique(unique, ref))
;
@@ -140,6 +138,12 @@ public class BuildChainProcessor {
return runStat.getCriticalFailRate() * 3.14f + runStat.getFailRate();
};
+
+ BuildRef someEntryPoint = entryPoints.iterator().next();
+ FatBuildCompacted build = teamcityIgnited.getFatBuild(someEntryPoint.getId());
+ FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(build.toBuild(compactor));
+
+
contexts.sort(Comparator.comparing(function).reversed());
fullChainRunCtx.addAllSuites(contexts);
@@ -279,17 +283,32 @@ public class BuildChainProcessor {
return branch;
}
- @Nullable private static Stream<? extends BuildRef> dependencies(ITeamcity teamcity, BuildRef ref) {
- Build results = teamcity.getBuild(ref.href);
- if (results == null)
+ @Nullable private static Stream<? extends BuildRef> dependencies(
+ ITeamcityIgnited teamcityIgnited,
+ Map<Integer, FatBuildCompacted> builds,
+ ITeamcity teamcity, BuildRef ref) {
+ FatBuildCompacted buildCompacted = builds.get(ref.getId());
+ if (buildCompacted != null)
+ return Stream.of(ref); // already processed build, so just keep current ID in stream
+
+ FatBuildCompacted build = teamcityIgnited.getFatBuild(ref.getId());
+
+ if (build.isFakeStub())
return Stream.of(ref);
- List<BuildRef> deps = results.getSnapshotDependenciesNonNull();
+ int[] ints = build.snapshotDependencies();
+
+ List<BuildRef> deps = IntStream.of(ints).mapToObj(i-> {
+ BuildRef ref1 = new BuildRef();
+ ref1.setId(i);
+
+ return ref1;
+ }).collect(Collectors.toList());
- if(deps.isEmpty())
+ if (deps.isEmpty())
return Stream.of(ref);
- if(logger.isDebugEnabled())
+ if (logger.isDebugEnabled())
logger.debug("Snapshot deps found: " +
ref.suiteId() + "->" + deps.stream().map(BuildRef::suiteId).collect(Collectors.toList()));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index 589b13b..1c17d49 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -137,4 +137,9 @@ public class BuildRefCompacted {
public int state() {
return state;
}
+
+ /** */
+ public boolean isFakeStub() {
+ return id() < 0;
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 48eb899..26e3313 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -55,6 +55,7 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
/** flag offset. */
public static final int FAKE_BUILD_F = 4;
+ public static final int[] EMPTY = new int[0];
/** Entity fields version. */
private short _ver = LATEST_VERSION;
@@ -354,9 +355,16 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
}
public int[] changes() {
- if(changesIds==null)
- return new int[0];
+ if (changesIds == null)
+ return EMPTY;
return changesIds;
}
+
+ public int[] snapshotDependencies() {
+ if (snapshotDeps == null)
+ return EMPTY;
+
+ return snapshotDeps.clone();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 4a5743a..47d7480 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -97,7 +97,6 @@ public class ProactiveFatBuildSync {
}
-
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0})
@AutoProfiling
@@ -106,8 +105,6 @@ public class ProactiveFatBuildSync {
final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
- Arrays.parallelSort(buildRefKeys);
-
List<Integer> buildsIdsToLoad = new ArrayList<>();
int totalAskedToLoad = 0;