You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/20 14:22:26 UTC
[ignite-teamcity-bot] 03/07: Trusted tests & suite history
performance fixes: Storage into TTL persisted cache
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 9a87e9cb54df4c4bf8ddb890cd1297a621ac1208
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Jun 19 20:48:54 2019 +0300
Trusted tests & suite history performance fixes: Storage into TTL persisted cache
---
.../apache/ignite/tcignited/ITeamcityIgnited.java | 6 +--
.../ignite/tcignited/TeamcityIgnitedImpl.java | 19 ++++----
.../ignite/tcignited/TeamcityIgnitedModule.java | 2 +
.../apache/ignite/tcignited/build/FatBuildDao.java | 56 +++++++++++++---------
.../ignite/tcignited/build/SuiteHistory.java | 2 -
.../ignite/tcignited/history/SuiteInvocation.java | 11 +++--
.../history/SuiteInvocationHistoryDao.java | 14 ++++--
7 files changed, 67 insertions(+), 43 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 38120db..f19a4ad 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -210,10 +210,10 @@ public interface ITeamcityIgnited {
* V.3.0 run history implementation based on scan of fat builds.
*
* @param testName Test name.
- * @param suiteName Suite name. Null suite name means suite not found
- * @param branchName Branch name.
+ * @param buildTypeId Suite (Build Type) ID, ID for compactor. Null suite name means suite not found.
+ * @param normalizedBaseBranch Branch name. This branch name does not support branches equivalence, only exact query will work.
*/
- @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName);
+ @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
/**
* @param suiteBuildTypeId Suite id.
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index def3851..707285f 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -433,20 +433,21 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
- @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName,
- @Nullable Integer branchName) {
- if (suiteName == null || branchName == null)
+ @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId,
+ @Nullable Integer normalizedBaseBranch) {
+ if (buildTypeId == null || normalizedBaseBranch == null)
return null;
- if (testName < 0 || suiteName < 0 || branchName < 0)
+ if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0)
return null;
Supplier<Set<Integer>> supplier = () -> {
- String btId = compactor.getStringFromId(suiteName);
- String branchId = compactor.getStringFromId(branchName);
+ String btId = compactor.getStringFromId(buildTypeId);
+ String branchId = compactor.getStringFromId(normalizedBaseBranch);
List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId);
long curTs = System.currentTimeMillis();
- Set<Integer> buildIds = compacted.stream().filter(
+ Set<Integer> buildIds = compacted.stream()
+ .filter(
bRef -> {
Long startTime = getBuildStartTime(bRef.id());
if (startTime == null)
@@ -456,12 +457,12 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
).map(BuildRefCompacted::id).collect(Collectors.toSet());
- System.err.println("Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+ System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
return buildIds;
};
- return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, suiteName, branchName);
+ return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, buildTypeId, normalizedBaseBranch);
}
/** {@inheritDoc} */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
index 707cab8..86cb573 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.tcignited.buildlog.BuildLogProcessorModule;
@@ -65,6 +66,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(MuteDao.class).in(new SingletonScope());
bind(MuteSync.class).in(new SingletonScope());
bind(BuildLogCheckResultDao.class).in(new SingletonScope());
+ bind(SuiteInvocationHistoryDao.class).in(new SingletonScope());
TcRealConnectionModule module = new TcRealConnectionModule();
if (conn != null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 341c3b9..af59fd9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -225,22 +225,21 @@ public class FatBuildDao {
/**
* @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
- * @param buildIdsSupplier Latest actual Build ids supplier.
+ * @param buildIdsSupplier Latest actual Build ids supplier. This supplier should handle all equivalent branches in it.
* @param testName Test name.
- * @param suiteName Suite name.
- * @param branchName Branch name.
+ * @param buildTypeId Suite (Build type) id.
+ * @param normalizedBaseBranch Branch name.
*/
@AutoProfiling
public IRunHistory getTestRunHist(int srvIdMaskHigh,
- Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) {
+ Supplier<Set<Integer>> buildIdsSupplier, int testName, int buildTypeId, int normalizedBaseBranch) {
- RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, suiteName, branchName);
+ RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
SuiteHistory hist;
-
try {
hist = runHistInMemCache.get(runHistKey,
- () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, suiteName, branchName));
+ () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, buildTypeId, normalizedBaseBranch));
}
catch (ExecutionException e) {
throw ExceptionUtil.propagateException(e);
@@ -252,9 +251,9 @@ public class FatBuildDao {
@AutoProfiling
public SuiteHistory loadSuiteHistory(int srvId,
Supplier<Set<Integer>> buildIdsSupplier,
- int suiteName,
- int branchName) {
- Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, suiteName, branchName);// todo RunHistSync.normalizeBranch();
+ int buildTypeId,
+ int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier);
@@ -262,33 +261,44 @@ public class FatBuildDao {
missedBuildsIds.removeAll(suiteRunHist.keySet());
- if (!missedBuildsIds.isEmpty())
- addSuiteInvocationsToHistory(srvId, suiteRunHist, missedBuildsIds);
+ if (!missedBuildsIds.isEmpty()) {
+ Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
+
+ System.err.println("***** + Adding to persisted history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ addl.size() + " invocations");
+
+ historyDao.putAll(srvId, addl);
+ suiteRunHist.putAll(addl);
+ }
SuiteHistory sumary = new SuiteHistory();
suiteRunHist.forEach((buildId, suiteInv) -> {
suiteInv.tests().forEach((tName, test) -> {
- RunHistCompacted compacted = sumary.testsHistory.computeIfAbsent(tName,
- k_ -> new RunHistCompacted());
-
- compacted.innerAddInvocation(test);
+ sumary.testsHistory.computeIfAbsent(tName,
+ k_ -> new RunHistCompacted()).innerAddInvocation(test);
});
});
+ System.err.println("***** History for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ sumary.size(igniteProvider.get()) + " bytes");
+
return sumary;
}
@AutoProfiling
- public void addSuiteInvocationsToHistory(int srvId, Map<Integer, SuiteInvocation> suiteRunHist,
- HashSet<Integer> missedBuildsIds) {
+ public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
+ HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
- Map<Long, FatBuildCompacted> buildsCacheAll = getAllFatBuilds(srvId, missedBuildsIds);
-
- buildsCacheAll.forEach((buildCacheKey, fatBuildCompacted) -> {
- SuiteInvocation sinv = new SuiteInvocation(fatBuildCompacted, compactor, (k, v) -> false);
+ getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
+ SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
tests.forEach(
@@ -301,6 +311,8 @@ public class FatBuildDao {
suiteRunHist.put(fatBuildCompacted.id(), sinv);
});
+
+ return suiteRunHist;
}
@AutoProfiling
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index e9e9b22..a7af3f8 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -33,8 +33,6 @@ public class SuiteHistory {
/** Tests history: Test name ID->RunHistory */
Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
- Map<Integer, SuiteInvocation> invocationMap = new TreeMap<>();
-
public int size(Ignite ignite) {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
return binary.length();
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
index e2cc0cd..0687a13 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -38,7 +38,7 @@ public class SuiteInvocation {
/** Suite name for queries */
@QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 1)})
- private int suiteName;
+ private int buildTypeId;
/** Teamcity branch name for queries */
@QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)})
@@ -52,10 +52,13 @@ public class SuiteInvocation {
public SuiteInvocation() {}
- public SuiteInvocation(FatBuildCompacted buildCompacted, IStringCompactor comp,
+ public SuiteInvocation(int srvId, int normalizedBaseBranch, FatBuildCompacted buildCompacted, IStringCompactor comp,
BiPredicate<Integer, Integer> filter) {
- buildStartTime = buildCompacted.getStartDateTs();
- suite = buildCompacted.toInvocation(comp, filter);
+ this.srvId = srvId;
+ this.normalizedBranchName = normalizedBaseBranch;
+ this.buildStartTime = buildCompacted.getStartDateTs();
+ this.suite = buildCompacted.toInvocation(comp, filter);
+ this.buildTypeId = buildCompacted.buildTypeId();
}
public void addTest(int testName, Invocation invocation) {
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
index 1cd066a..3b5b450 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
@@ -58,11 +58,11 @@ public class SuiteInvocationHistoryDao {
suiteHistory = ignite.getOrCreateCache(ccfg);
}
- public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int suiteName, int normalizedBranchName) {
+ public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int buildTypeId, int normalizedBranchName) {
java.util.Map<Integer, SuiteInvocation> map = new HashMap<>();
try (QueryCursor<Cache.Entry<Long, SuiteInvocation>> qryCursor = suiteHistory.query(
- new SqlQuery<Long, SuiteInvocation>(SuiteInvocation.class, "srvId = ? and suiteName = ? and normalizedBranchName = ?")
- .setArgs(srvId, suiteName, normalizedBranchName))) {
+ new SqlQuery<Long, SuiteInvocation>(SuiteInvocation.class, "srvId = ? and buildTypeId = ? and normalizedBranchName = ?")
+ .setArgs(srvId, buildTypeId, normalizedBranchName))) {
for (Cache.Entry<Long, SuiteInvocation> next : qryCursor) {
Long key = next.getKey();
@@ -73,4 +73,12 @@ public class SuiteInvocationHistoryDao {
return map;
}
+
+ public void putAll(int srvId, Map<Integer, SuiteInvocation> addl) {
+ Map<Long, SuiteInvocation> data = new HashMap<>();
+
+ addl.forEach((k, v) -> data.put(BuildRefDao.buildIdToCacheKey(srvId, k), v));
+
+ suiteHistory.putAll(data);
+ }
}