You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/11/27 15:48:59 UTC
[ignite-teamcity-bot] branch ignite-9542-new-run-stripe updated:
IGNITE-9542 Full missed run history search implemented
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9542-new-run-stripe
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9542-new-run-stripe by this push:
new da0ca86 IGNITE-9542 Full missed run history search implemented
da0ca86 is described below
commit da0ca866b3ff32dc791d290040f82faed936c449
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Nov 27 18:48:58 2018 +0300
IGNITE-9542 Full missed run history search implemented
---
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 19 ++---
.../ci/teamcity/ignited/buildref/BuildRefSync.java | 23 ++++++
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 2 +-
.../ignited/fatbuild/ProactiveFatBuildSync.java | 1 -
.../teamcity/ignited/runhist/RunHistCompacted.java | 5 ++
.../ignited/runhist/RunHistCompactedDao.java | 4 --
.../ci/teamcity/ignited/runhist/RunHistSync.java | 84 ++++++++++++++++++++--
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 63 ++++++++++++----
8 files changed, 168 insertions(+), 33 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 77a0893..e1022cd 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -95,6 +95,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Run history DAO. */
@Inject private RunHistCompactedDao runHistCompactedDao;
+ /** Run history sync. */
@Inject private RunHistSync runHistSync;
/** Strings compactor. */
@@ -331,6 +332,11 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
public void ensureActualizeRequested() {
scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
+
+ // schedule find missing later
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
+
+ runHistSync.invokeLaterFindMissingHistory(srvNme);
}
/** {@inheritDoc} */
@@ -397,13 +403,13 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
FatBuildCompacted savedVer = buildSync.loadBuild(conn, buildId, existingBuild, mode);
//build was modified, probably we need also to update reference accordingly
- if (savedVer != null)
- buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
+ if (savedVer == null)
+ return existingBuild;
- if (savedVer != null)
- runHistSync.saveToHistoryLater(srvIdMaskHigh, buildId, savedVer);
+ buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
+ runHistSync.saveToHistoryLater(srvIdMaskHigh, buildId, savedVer);
- return savedVer == null ? existingBuild : savedVer;
+ return savedVer;
}
protected FatBuildCompacted getFatBuildFromIgnite(int buildId) {
@@ -442,9 +448,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
void actualizeRecentBuildRefs() {
- // schedule find missing later
- buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
-
List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
Set<Integer> paginateUntil = new HashSet<>();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
new file mode 100644
index 0000000..fce67c7
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.teamcity.ignited.buildref;
+
+/**
+ *
+ */
+public class BuildRefSync {
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index aa34f3f..2c4df80 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -113,7 +113,7 @@ public class FatBuildDao {
}
@AutoProfiling
- protected void putFatBuild(long srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
+ public void putFatBuild(long srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 8c95421..6a63289 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -229,7 +229,6 @@ public class ProactiveFatBuildSync {
() -> findMissingBuildsFromBuildRef(srvName, conn), 360, TimeUnit.MINUTES);
}
-
/**
*
* @param conn TC connection to load data
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
index 96adae4..f93d618 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
@@ -20,11 +20,16 @@ package org.apache.ignite.ci.teamcity.ignited.runhist;
import com.google.common.base.MoreObjects;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.ci.analysis.IVersionedEntity;
+import org.apache.ignite.ci.db.Persisted;
import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
import javax.annotation.Nullable;
import java.util.List;
+/**
+ *
+ */
+@Persisted
public class RunHistCompacted implements IVersionedEntity, IRunHistory {
/** Latest version. */
private static final int LATEST_VERSION = 1;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
index c2d1e5d..46768db 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
@@ -60,10 +60,6 @@ public class RunHistCompactedDao {
testHistCache = ignite.getOrCreateCache(cfg);
}
- public void save(RunHistKey k, RunHistCompacted v) {
- testHistCache.put(k, v);
- }
-
public IRunHistory getTestRunHist(int srvIdMaskHigh, String name, String branch) {
final Integer testName = compactor.getStringIdIfPresent(name);
if (testName == null)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
index aefaf61..1408f80 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
@@ -17,20 +17,26 @@
package org.apache.ignite.ci.teamcity.ignited.runhist;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import javax.annotation.concurrent.GuardedBy;
import javax.inject.Inject;
import java.util.HashMap;
import java.util.Map;
+import org.jetbrains.annotations.NotNull;
/**
*
@@ -41,7 +47,14 @@ public class RunHistSync {
@Inject private IScheduler scheduler;
- @Inject private RunHistCompactedDao dao;
+ @Inject private RunHistCompactedDao histDao;
+
+ /** Build reference DAO. */
+ @Inject private BuildRefDao buildRefDao;
+
+ /** Build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
/** Build to save to history. */
@GuardedBy("this")
@@ -53,10 +66,7 @@ public class RunHistSync {
* @param build Build.
*/
public void saveToHistoryLater(int srvId, int buildId, FatBuildCompacted build) {
- if (!build.isFinished(compactor))
- return;
-
- if (build.isCancelled(compactor))
+ if (!validForStatistics(build))
return;
synchronized (this) {
@@ -95,7 +105,7 @@ public class RunHistSync {
build.getAllTests().forEach(t -> {
Invocation inv = t.toInvocation(compactor, build);
- final Boolean res = dao.addInvocation(srvId, t, build.id(), build.branchName(), inv);
+ final Boolean res = histDao.addInvocation(srvId, t, build.id(), build.branchName(), inv);
if(Boolean.FALSE.equals(res))
duplicates.incrementAndGet();
@@ -109,6 +119,68 @@ public class RunHistSync {
+ " invocations saved to DB " + duplicates.get() + " duplicates";
}
+ public void invokeLaterFindMissingHistory(String srvName) {
+ scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName),
+ () -> findMissingHistFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+ }
+
+ @NotNull
+ private String taskName(String taskName, String srvName) {
+ return RunHistSync.class.getSimpleName() +"." + taskName + "." + srvName;
+ }
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Find missing Build History", nameExtArgsIndexes = {0})
+ @AutoProfiling
+ protected String findMissingHistFromBuildRef(String srvId ) {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
+
+ List<Integer> buildsIdsToLoad = new ArrayList<>();
+ int totalAskedToLoad = 0;
+
+ for (int buildRefKey : buildRefKeys) {
+ if (!fatBuildDao.containsKey(srvIdMaskHigh, buildRefKey))
+ continue; // FAT Build not yet avaiable, skipping hist loading
+
+ if (buildsIdsToLoad.size() >= 100) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleHistLoad(srvIdMaskHigh, buildsIdsToLoad);
+ buildsIdsToLoad.clear();
+ }
+ buildsIdsToLoad.add(buildRefKey);
+ }
+
+ if (!buildsIdsToLoad.isEmpty()) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleHistLoad(srvIdMaskHigh, buildsIdsToLoad);
+ }
+
+ return "Invoked later load for " + totalAskedToLoad + " builds from " + srvId;
+ }
+
+ private void scheduleHistLoad(int srvIdMaskHigh, List<Integer> load) {
+ //todo implement
+ System.err.println("scheduleHistLoad: " + load.toString());
+
+ load.forEach(id -> {
+ FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, id);
+
+ if (validForStatistics(fatBuild))
+ saveToHistoryLater(srvIdMaskHigh, fatBuild.id(), fatBuild);
+ else
+ System.err.println("Build is not valid for stat: " + fatBuild.toString());
+ });
+ }
+
+ private boolean validForStatistics(FatBuildCompacted fatBuild) {
+ return fatBuild != null
+ && !fatBuild.isFakeStub()
+ && !fatBuild.isCancelled(compactor)
+ && fatBuild.isFinished(compactor);
+ }
+
/**
* Scope of work: builds to be loaded from a connection.
*/
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index dbbc09c..0a5e013 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -348,10 +348,8 @@ public class IgnitedTcInMemoryIntegrationTest {
@Test
public void testRunHistSaveLoad() {
- final String srvId = "apache";
- TeamcityIgnitedModule module = new TeamcityIgnitedModule();
- Injector injector = Guice.createInjector(module, new AbstractModule() {
+ Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new AbstractModule() {
@Override
protected void configure() {
bind(Ignite.class).toInstance(ignite);
@@ -359,11 +357,10 @@ public class IgnitedTcInMemoryIntegrationTest {
}
});
+ injector.getInstance(RunHistCompactedDao.class).init();
final IStringCompactor c = injector.getInstance(IStringCompactor.class);
- final RunHistCompactedDao dao = injector.getInstance(RunHistCompactedDao.class);
- dao.init();
-
+ final String srvId = "apache";
final String btId = "RunAll";
final String branch = ITeamcity.DEFAULT;
@@ -372,21 +369,61 @@ public class IgnitedTcInMemoryIntegrationTest {
final Map<Integer, FatBuildCompacted> buildsMap = tst.apacheBuilds();
- Map<RunHistKey, RunHistCompacted> resMap = new HashMap<>();
+ final RunHistSync histSync = injector.getInstance(RunHistSync.class);
+ buildsMap.forEach((id, build) -> histSync.saveToHistoryLater(ITeamcityIgnited.serverIdToInt(srvId), id, build));
- TeamcityIgnitedMock.initHistory(c, resMap, buildsMap, U.safeAbs(srvId.hashCode()));
+ final ITeamcityIgnitedProvider inst = injector.getInstance(ITeamcityIgnitedProvider.class);
+ final ITeamcityIgnited srv = inst.server(srvId, Mockito.mock(ICredentialsProv.class));
+ final IRunHistory testRunHist = srv.getTestRunHist(new TestInBranch(PrChainsProcessorTest.TEST_FLAKY_IN_MASTER, branch));
- System.out.println(resMap);
+ assertNotNull(testRunHist);
+ assertEquals(0.5, testRunHist.getFailRate(), 0.1);
+ }
- // resMap.forEach(dao::save);
- final RunHistSync histSync = injector.getInstance(RunHistSync.class);
- buildsMap.forEach((id, build) -> {
- histSync.saveToHistoryLater(U.safeAbs(srvId.hashCode()), id, build);
+
+ @Test
+ public void testHistoryBackgroundUpdateWorks() {
+ Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(Ignite.class).toInstance(ignite);
+ bind(IScheduler.class).to(DirectExecNoWaitSheduler.class).in(new SingletonScope());
+ }
});
+ injector.getInstance(RunHistCompactedDao.class).init();
+
+ final String srvId = "apache";
+ final String btId = "RunAll";
+ final String branch = ITeamcity.DEFAULT;
+
final ITeamcityIgnitedProvider inst = injector.getInstance(ITeamcityIgnitedProvider.class);
final ITeamcityIgnited srv = inst.server(srvId, Mockito.mock(ICredentialsProv.class));
+
+ FatBuildDao fatBuildDao = injector.getInstance(FatBuildDao.class);
+ fatBuildDao.init();
+
+ BuildRefDao buildRefDao = injector.getInstance(BuildRefDao.class);
+ buildRefDao.init();
+
+ final IStringCompactor c = injector.getInstance(IStringCompactor.class);
+
+
+ final PrChainsProcessorTest tst = new PrChainsProcessorTest();
+ tst.initBuildChain(c, btId, branch);
+
+ final Map<Integer, FatBuildCompacted> buildsMap = tst.apacheBuilds();
+
+ buildsMap.forEach((id, build) -> {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+ fatBuildDao.putFatBuild(srvIdMaskHigh, id, build);
+ buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(build));
+ });
+
+ final RunHistSync histSync = injector.getInstance(RunHistSync.class);
+ histSync.invokeLaterFindMissingHistory(srvId);
+
final IRunHistory testRunHist = srv.getTestRunHist(new TestInBranch(PrChainsProcessorTest.TEST_FLAKY_IN_MASTER, branch));
assertNotNull(testRunHist);