You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/20 14:22:30 UTC
[ignite-teamcity-bot] 07/07: Trusted tests & suite history
performance fixes: Build start time implementation using entry processor
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 5c777bebbe2277cd212716b6fecf5ea321c8d95d
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jun 20 17:08:15 2019 +0300
Trusted tests & suite history performance fixes: Build start time implementation using entry processor
---
.../apache/ignite/tcignited/build/FatBuildDao.java | 36 ++++++++-
.../ignite/tcignited/history/HistoryCollector.java | 93 ++++++++++------------
2 files changed, 73 insertions(+), 56 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 4424158..98ef8fd 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -19,6 +19,7 @@ package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -29,14 +30,19 @@ import java.util.stream.StreamSupport;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.cache.Cache;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcservice.model.changes.ChangesList;
import org.apache.ignite.tcservice.model.result.Build;
@@ -191,12 +197,36 @@ public class FatBuildDao {
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
+ private static Set<Long> buildsIdsToCacheKeys(int srvId, Collection<Integer> stream) {
+ return stream.stream()
+ .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet());
+ }
+ public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) {
+ Map<Long, EntryProcessorResult<Long>> map = buildsCache.invokeAll(buildsIdsToCacheKeys(srvId, ids), new GetStartTimeProc());
- private static Set<Long> buildsIdsToCacheKeys(int srvIdMaskHigh, Collection<Integer> stream) {
- return stream.stream()
- .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
+ HashMap<Integer, Long> res = new HashMap<>();
+
+ map.forEach((k, r)->{
+ Long aLong = r.get();
+ if(aLong!=null)
+ res.put(BuildRefDao.cacheKeyToBuildId(k), aLong);
+ });
+
+ return res;
}
+ private static class GetStartTimeProc implements CacheEntryProcessor<Long, FatBuildCompacted, Long> {
+ public GetStartTimeProc() {
+ }
+
+ @Override public Long process(MutableEntry<Long, FatBuildCompacted> entry,
+ Object... arguments) throws EntryProcessorException {
+ if (entry.getValue() == null)
+ return null;
+ FatBuildCompacted fatBuildCompacted = entry.getValue();
+ return fatBuildCompacted.getStartDateTs();
+ }
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index 68743b7..f5b720e 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -27,14 +27,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.MutableEntry;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
-import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
@@ -115,10 +111,10 @@ public class HistoryCollector {
* @param srvId
* @param buildTypeId
* @param normalizedBaseBranch
- * @param knownBuilds Known builds.
+ * @param knownBuilds Known builds, which already present in run history.
*/
@AutoProfiling
- protected Set<Integer> determineLatestBuildsFunction(
+ protected Set<Integer> determineLatestBuilds(
int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
String btId = compactor.getStringFromId(buildTypeId);
String branchId = compactor.getStringFromId(normalizedBaseBranch);
@@ -127,31 +123,50 @@ public class HistoryCollector {
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = compacted.stream()
.filter(b -> !knownBuilds.contains(b.id()))
- //todo filter queued, cancelled and so on
- .filter(
- bRef -> {
- //todo getAll
- //todo getStartTime From FatBuild
- Long startTime = runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
- if (startTime == null)
- return false;
-
- return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
- }
- ).map(BuildRefCompacted::id).collect(Collectors.toSet());
+ .filter(this::applicableForHistory)
+ .map(BuildRefCompacted::id).collect(Collectors.toSet());
+
+ Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, buildIds);
+
+ //todo filter queued, cancelled and so on
+ Set<Integer> buildInScope = buildIds.stream().filter(
+ bId -> {
+ //todo getAll
+ //todo getStartTime From FatBuild
+ Long startTime = buildStartTimeFromFatBuild.get(bId);
+ //todo cache in runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
+ if (startTime == null)
+ return false;
+
+ return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
+ }
+ ).collect(Collectors.toSet());
- System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+ System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " +
+ buildInScope+ " from " + buildIds.size());
return buildIds;
}
@AutoProfiling
+ protected Map<Integer, Long> getStartTimeFromFatBuild(int srvId, Set<Integer> buildIds) {
+ return fatBuildDao.getBuildStartTime(srvId, buildIds);
+ }
+
+ /**
+ * @param ref Build Reference or fat build.
+ */
+ private boolean applicableForHistory(BuildRefCompacted ref) {
+ return !ref.isFakeStub() && !ref.isCancelled(compactor) && ref.isFinished(compactor);
+ }
+
+ @AutoProfiling
public SuiteHistory loadSuiteHistory(int srvId,
int buildTypeId,
int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
- Set<Integer> buildIds = determineLatestBuildsFunction(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
+ Set<Integer> buildIds = determineLatestBuilds(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
@@ -162,8 +177,8 @@ public class HistoryCollector {
System.err.println("***** + Adding to persisted history for suite "
+ compactor.getStringFromId(buildTypeId)
- + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- addl.size() + " invocations");
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " +
+ addl.size() + " invocations from " + missedBuildsIds + " builds checked");
histDao.putAll(srvId, addl);
suiteRunHist.putAll(addl);
@@ -191,8 +206,6 @@ public class HistoryCollector {
}
-
-
@AutoProfiling
public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
@@ -200,6 +213,9 @@ public class HistoryCollector {
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
fatBuildDao.getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
+ if(!applicableForHistory(fatBuildCompacted))
+ return;
+
SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
@@ -258,33 +274,4 @@ public class HistoryCollector {
return hist;
}
*/
-
-
- private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
- private final int successStatusStrId;
-
- public HistoryCollectProcessor(int successStatusStrId) {
- this.successStatusStrId = successStatusStrId;
- }
-
- @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
- Object... arguments) throws EntryProcessorException {
- if (entry.getValue() == null)
- return null;
-
- Map<Integer, Invocation> hist = new HashMap<>();
- FatBuildCompacted fatBuildCompacted = entry.getValue();
- Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
- tests.forEach(
- testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
-
- hist.put(testCompacted.testName(), invocation);
- }
- );
-
- return hist;
- }
- }
-
}