You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/27 10:14:53 UTC
[ignite-teamcity-bot] branch ignite-9848-2 created (now 9da98d1)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch ignite-9848-2
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git.
at 9da98d1 Standalone syncer class started; Find missing builds by refs started
This branch includes the following new commits:
new 9da98d1 Standalone syncer class started; Find missing builds by refs started
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ignite-teamcity-bot] 01/01: Standalone syncer class started;
Find missing builds by refs started
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-2
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 9da98d150c463bffede8afbbcec5551a55f69d79
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 27 13:14:50 2018 +0300
Standalone syncer class started; Find missing builds by refs started
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 1 -
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 21 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 190 ++++--------------
.../ci/teamcity/ignited/TeamcityIgnitedModule.java | 2 +
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 26 +++
.../ignited/fatbuild/ProactiveFatBuildSync.java | 221 +++++++++++++++++++++
.../ignite/ci/teamcity/pure/ITeamcityConn.java | 5 +
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 2 +-
8 files changed, 306 insertions(+), 162 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 92cdab8..12446b9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -59,7 +59,6 @@ public interface ITeamcity extends ITeamcityConn {
CompletableFuture<List<BuildType>> getProjectSuites(String projectId);
- String serverId();
/**
* @param projectId Suite ID (string without spaces).
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index d11f148..92b1684 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -17,13 +17,7 @@
package org.apache.ignite.ci.teamcity.ignited;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TreeMap;
+import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -223,4 +217,17 @@ public class BuildRefDao {
return false;
}
+
+ @AutoProfiling
+ public int[] getAllIds(int srvId) {
+ GridIntList res = new GridIntList(buildRefsCache.size());
+
+ StreamSupport.stream(buildRefsCache.spliterator(), false)
+ .map(Cache.Entry::getKey)
+ .filter(entry -> isKeyForServer(entry, srvId))
+ .map(BuildRefDao::cacheKeyToBuildId)
+ .forEach(res::add);
+
+ return res.array();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index f32d335..08e1b5f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -20,23 +20,6 @@ package org.apache.ignite.ci.teamcity.ignited;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.collect.Sets;
-import java.io.FileNotFoundException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.OptionalInt;
-import java.util.Set;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-import javax.annotation.concurrent.GuardedBy;
-import javax.inject.Inject;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
@@ -48,23 +31,30 @@ import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
import org.apache.ignite.ci.util.ExceptionUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.io.FileNotFoundException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Logger. */
- private static final Logger logger = LoggerFactory.getLogger(TestCompacted.class);
+ private static final Logger logger = LoggerFactory.getLogger(TeamcityIgnitedImpl.class);
/** Max build id diff to enforce reload during incremental refresh. */
public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000;
- public static final int FAT_BUILD_PROACTIVE_TASKS = 4;
/** Server id. */
- private String srvId;
+ private String srvNme;
/** Pure HTTP Connection API. */
private ITeamcityConn conn;
@@ -81,45 +71,27 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Build DAO. */
@Inject private FatBuildDao fatBuildDao;
- @Inject private IStringCompactor compactor;
+ @Inject private ProactiveFatBuildSync buildSync;
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
- @GuardedBy("this")
- private Set<Integer> buildToLoad = new HashSet<>();
-
public void init(String srvId, ITeamcityConn conn) {
- this.srvId = srvId;
+ this.srvNme = srvId;
this.conn = conn;
srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
buildRefDao.init(); //todo init somehow in auto
buildConditionDao.init();
fatBuildDao.init();
- }
-
- /**
- * Invoke load fat builds later, re-load provided builds.
- *
- * @param buildsToAskFromTc Builds to ask from tc.
- */
- public void scheduleBuildsLoad(Collection<Integer> buildsToAskFromTc) {
- if (buildsToAskFromTc.isEmpty())
- return;
-
- synchronized (this) {
- buildToLoad.addAll(buildsToAskFromTc);
- }
-
- int ldrToActivate = ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
-
- scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate), () -> loadFatBuilds(ldrToActivate), 2, TimeUnit.MINUTES);
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme);
}
- @NotNull public String taskName(String taskName) {
- return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." + srvId;
+
+ @NotNull
+ private String taskName(String taskName) {
+ return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." + srvNme;
}
/** {@inheritDoc} */
@@ -132,7 +104,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Override public List<BuildRef> getBuildHistory(
@Nullable String buildTypeId,
@Nullable String branchName) {
- scheduler.sheduleNamed(taskName("actualizeRecentBuilds"), this::actualizeRecentBuilds, 2, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
String bracnhNameQry ;
if (ITeamcity.DEFAULT.equals(branchName))
@@ -148,7 +120,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
Build build = conn.triggerBuild(buildTypeId, branchName, cleanRebuild, queueAtTop);
//todo may add additional parameter: load builds into DB in sync/async fashion
- runActualizeBuilds(srvId, false, Sets.newHashSet(build.getId()));
+ runActualizeBuildRefs(srvNme, false, Sets.newHashSet(build.getId()));
return build;
}
@@ -172,7 +144,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (existingBuild != null && !existingBuild.isOutdatedEntityVersion())
return existingBuild;
- FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild);
+ FatBuildCompacted savedVer = buildSync.reloadBuild(conn, buildId, existingBuild);
//build was modified, probably we need also to update reference accordindly
if (savedVer != null)
@@ -181,61 +153,12 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return savedVer == null ? existingBuild : savedVer;
}
- /**
- * @param buildId
- * @param existingBuild
- * @return new build if it was updated or null if no updates detected
- */
- public FatBuildCompacted reloadBuild(int buildId, @Nullable FatBuildCompacted existingBuild) {
- // System.err.println(Thread.currentThread().getName()+ ": Build " + buildId);
- //todo some sort of locking to avoid double requests
- Build build;
- List<TestOccurrencesFull> tests = new ArrayList<>();
- try {
- build = conn.getBuild(buildId);
-
- String nextHref = null;
- do {
- boolean testDtls = !build.isComposite(); // don't query test details for compoite
- TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls);
- nextHref = page.nextHref();
-
- tests.add(page);
- }
- while (!Strings.isNullOrEmpty(nextHref));
- }
- catch (Exception e) {
- if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
- logger.info("Loading build [" + buildId + "] for server [" + srvId + "] failed:" + e.getMessage(), e);
-
- if (existingBuild != null) {
- build = existingBuild.toBuild(compactor);
-
- if(build.isRunning() || build.isQueued())
- build.setCancelled();
-
- tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor));
- }
- else
- build = Build.createFakeStub();
- } else {
- logger.error("Loading build [" + buildId + "] for server [" + srvId + "] failed:" + e.getMessage(), e);
-
- e.printStackTrace();
-
- throw ExceptionUtil.propagateException(e);
- }
- }
- //if we are here because of some sort of outdated version of build,
- // new save will be performed with new entity version for compacted build
- return fatBuildDao.saveBuild(srvIdMaskHigh, buildId, build, tests, existingBuild);
- }
/**
*
*/
- void actualizeRecentBuilds() {
+ void actualizeRecentBuildRefs() {
List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
Set<Integer> paginateUntil = new HashSet<>();
@@ -252,23 +175,23 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
});
}
//schedule direct reload for Fat Builds for all queued too-old builds
- scheduleBuildsLoad(directUpload);
+ buildSync.scheduleBuildsLoad(srvNme, directUpload);
- runActualizeBuilds(srvId, false, paginateUntil);
+ runActualizeBuildRefs(srvNme, false, paginateUntil);
if(!paginateUntil.isEmpty()) {
//some builds may stuck in the queued or running, enforce loading as well
- scheduleBuildsLoad(paginateUntil);
+ buildSync.scheduleBuildsLoad(srvNme, paginateUntil);
}
// schedule full resync later
- scheduler.invokeLater(this::sheduleResync, 15, TimeUnit.MINUTES);
+ scheduler.invokeLater(this::sheduleResyncBuildRefs, 15, TimeUnit.MINUTES);
}
/**
*
*/
- private void sheduleResync() {
+ private void sheduleResyncBuildRefs() {
scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 120, TimeUnit.MINUTES);
}
@@ -276,25 +199,29 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
void fullReindex() {
- runActualizeBuilds(srvId, true, null);
+ runActualizeBuildRefs(srvNme, true, null);
+
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme);
}
+
/**
- * @param srvId Server id. todo to be added as composite name extend
+ * @param srvId Server id.
* @param fullReindex Reindex all builds from TC history.
* @param mandatoryToReload [in/out] Build ID should be found before end of sync. Ignored if fullReindex mode.
*
*/
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Actualize BuildRefs(srv, full resync)", nameExtArgsIndexes = {0, 1})
@AutoProfiling
- protected String runActualizeBuilds(String srvId, boolean fullReindex,
- @Nullable Set<Integer> mandatoryToReload) {
+ protected String runActualizeBuildRefs(String srvId, boolean fullReindex,
+ @Nullable Set<Integer> mandatoryToReload) {
AtomicReference<String> outLinkNext = new AtomicReference<>();
List<BuildRef> tcDataFirstPage = conn.getBuildRefs(null, outLinkNext);
Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh, tcDataFirstPage);
int totalUpdated = buildsUpdated.size();
- scheduleBuildsLoad(cacheKeysToBuildIds(buildsUpdated));
+ buildSync.scheduleBuildsLoad(srvNme, cacheKeysToBuildIds(buildsUpdated));
int totalChecked = tcDataFirstPage.size();
int neededToFind = 0;
@@ -310,7 +237,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
List<BuildRef> tcDataNextPage = conn.getBuildRefs(nextPageUrl, outLinkNext);
Set<Long> curChunkBuildsSaved = buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
totalUpdated += curChunkBuildsSaved.size();
- scheduleBuildsLoad(cacheKeysToBuildIds(curChunkBuildsSaved));
+ buildSync.scheduleBuildsLoad(srvNme, cacheKeysToBuildIds(curChunkBuildsSaved));
int savedCurChunk = curChunkBuildsSaved.size();
@@ -332,47 +259,4 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@NotNull private List<Integer> cacheKeysToBuildIds(Collection<Long> cacheKeysUpdated) {
return cacheKeysUpdated.stream().map(BuildRefDao::cacheKeyToBuildId).collect(Collectors.toList());
}
-
- /** */
- private void loadFatBuilds(int ldrNo) {
- Set<Integer> load;
-
- synchronized (this) {
- load = buildToLoad;
- buildToLoad = new HashSet<>();
- }
-
- doLoadBuilds(ldrNo, srvId, load);
- }
-
- @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1})
- @AutoProfiling
- protected String doLoadBuilds(int ldrNo, String srvId, Set<Integer> load) {
- if(load.isEmpty())
- return "Nothing to load";
-
- AtomicInteger err = new AtomicInteger();
- AtomicInteger ld = new AtomicInteger();
-
- Map<Long, FatBuildCompacted> builds = fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load);
-
- load.forEach(
- buildId -> {
- try {
- FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
-
- FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild);
-
- if (savedVer != null)
- ld.incrementAndGet();
- }
- catch (Exception e) {
- logger.error("", e);
- err.incrementAndGet();
- }
- }
- );
-
- return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err;
- }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
index e397d8f..9a5d68c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
@@ -20,6 +20,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.tcbot.condition.BuildConditionDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.teamcity.restcached.TcRestCachedModule;
import org.jetbrains.annotations.Nullable;
@@ -37,6 +38,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(BuildRefDao.class).in(new SingletonScope());
bind(BuildConditionDao.class).in(new SingletonScope());
bind(FatBuildDao.class).in(new SingletonScope());
+ bind(ProactiveFatBuildSync.class).in(new SingletonScope());
bind(IStringCompactor.class).to(IgniteStringCompactor.class).in(new SingletonScope());
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index fd8b8df..725c074 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -24,15 +24,20 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import javax.cache.Cache;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.validation.constraints.NotNull;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
+import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.internal.util.GridIntList;
import org.jetbrains.annotations.Nullable;
/**
@@ -120,4 +125,25 @@ public class FatBuildDao {
return buildsCache.getAll(ids);
}
+
+ /**
+ * @param key Key.
+ * @param srvId Server id.
+ */
+ private boolean isKeyForServer(Long key, int srvId) {
+ return key!=null && key >> 32 == srvId;
+ }
+
+ @AutoProfiling
+ public int[] getAllIds(int srvId) {
+ GridIntList res = new GridIntList(buildsCache.size());
+
+ StreamSupport.stream(buildsCache.spliterator(), false)
+ .map(Cache.Entry::getKey)
+ .filter(entry -> isKeyForServer(entry, srvId))
+ .map(BuildRefDao::cacheKeyToBuildId)
+ .forEach(res::add);
+
+ return res.array();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
new file mode 100644
index 0000000..2dd1757
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+
+import com.google.common.base.Strings;
+import com.google.common.base.Throwables;
+import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.di.MonitoredTask;
+import org.apache.ignite.ci.di.scheduler.IScheduler;
+import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
+import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
+import org.apache.ignite.ci.util.ExceptionUtil;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
+import javax.inject.Inject;
+import java.io.FileNotFoundException;
+import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ProactiveFatBuildSync {
+ public static final int FAT_BUILD_PROACTIVE_TASKS = 4;
+
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(ProactiveFatBuildSync.class);
+
+ /** Build reference DAO. */
+ @Inject
+ private BuildRefDao buildRefDao;
+
+ /** Build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
+ /** Scheduler. */
+ @Inject private IScheduler scheduler;
+
+ @Inject private IStringCompactor compactor;
+
+
+ @GuardedBy("this")
+ private Set<Integer> buildToLoad = new HashSet<>();
+
+
+ /**
+ * Invoke load fat builds later, re-load provided builds.
+ * @param srvNme
+ * @param buildsToAskFromTc Builds to ask from tc.
+ */
+ public void scheduleBuildsLoad(String srvNme , Collection<Integer> buildsToAskFromTc) {
+ if (buildsToAskFromTc.isEmpty())
+ return;
+
+ synchronized (this) {
+ buildToLoad.addAll(buildsToAskFromTc);
+ }
+
+ int ldrToActivate = ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
+
+ scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate, srvNme),
+ () -> loadFatBuilds(ldrToActivate, srvNme), 2, TimeUnit.MINUTES);
+
+ }
+
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0})
+ @AutoProfiling
+ protected String findMissingBuildsFromBuildRef(String srvId) {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
+ final int[] fatBuildKeys = fatBuildDao.getAllIds(srvIdMaskHigh);
+
+ Arrays.parallelSort(buildRefKeys);
+ Arrays.parallelSort(fatBuildKeys);
+ /* ;
+ */
+
+ return "";
+ }
+
+
+
+ /** */
+ private void loadFatBuilds(int ldrNo, String srvNme) {
+ Set<Integer> load;
+
+ synchronized (this) {
+ load = buildToLoad;
+ buildToLoad = new HashSet<>();
+ }
+
+ doLoadBuilds(ldrNo, srvNme, conn, load);
+ }
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1})
+ @AutoProfiling
+ protected String doLoadBuilds(int ldrNo, String srvId, ITeamcityConn conn, Set<Integer> load) {
+ if(load.isEmpty())
+ return "Nothing to load";
+
+ final int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ AtomicInteger err = new AtomicInteger();
+ AtomicInteger ld = new AtomicInteger();
+
+ Map<Long, FatBuildCompacted> builds = fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load);
+
+ load.forEach(
+ buildId -> {
+ try {
+ FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
+
+ FatBuildCompacted savedVer = reloadBuild(conn, buildId, existingBuild);
+
+ if (savedVer != null)
+ ld.incrementAndGet();
+ }
+ catch (Exception e) {
+ logger.error("", e);
+ err.incrementAndGet();
+ }
+ }
+ );
+
+ return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err;
+ }
+
+ @NotNull
+ private String taskName(String taskName, String srvName) {
+ return ProactiveFatBuildSync.class.getSimpleName() +"." + taskName + "." + srvName;
+ }
+
+ public void invokeLaterFindMissingByBuildRef(String srvName) {
+ scheduler.sheduleNamed(taskName("findMissingBuildsFromBuildRef", srvName),
+ () -> findMissingBuildsFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+ }
+
+ /**
+ *
+ * @param conn
+ * @param buildId
+ * @param existingBuild
+ * @return new build if it was updated or null if no updates detected
+ */
+ @SuppressWarnings({"WeakerAccess"})
+ @AutoProfiling
+ public FatBuildCompacted reloadBuild(ITeamcityConn conn, int buildId, @Nullable FatBuildCompacted existingBuild) {
+ // System.err.println(Thread.currentThread().getName()+ ": Build " + buildId);
+ //todo some sort of locking to avoid double requests
+
+ final String srvNme = conn.serverId();
+ final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvNme);
+
+ Build build;
+ List<TestOccurrencesFull> tests = new ArrayList<>();
+ try {
+ build = conn.getBuild(buildId);
+
+ String nextHref = null;
+ do {
+ boolean testDtls = !build.isComposite(); // don't query test details for compoite
+ TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls);
+ nextHref = page.nextHref();
+
+ tests.add(page);
+ }
+ while (!Strings.isNullOrEmpty(nextHref));
+ }
+ catch (Exception e) {
+ if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
+ logger.info("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
+
+ if (existingBuild != null) {
+ build = existingBuild.toBuild(compactor);
+
+ if(build.isRunning() || build.isQueued())
+ build.setCancelled();
+
+ tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor));
+ }
+ else
+ build = Build.createFakeStub();
+ } else {
+ logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
+
+ e.printStackTrace();
+
+ throw ExceptionUtil.propagateException(e);
+ }
+ }
+
+ //if we are here because of some sort of outdated version of build,
+ // new save will be performed with new entity version for compacted build
+ return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, existingBuild);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
index 78f3d34..f832088 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
@@ -30,6 +30,11 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
*/
public interface ITeamcityConn {
/**
+ * @return Internal server ID as string
+ */
+ String serverId();
+
+ /**
* @return Normalized Host address, ends with '/'.
*/
public String host();
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index b07e102..173a959 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -214,7 +214,7 @@ public class IgnitedTcInMemoryIntegrationTest {
for (int i = queuedBuildIdx; i < tcBuilds.size(); i++)
tcBuilds.get(i).state = BuildRef.STATE_FINISHED;
- teamcityIgnited.actualizeRecentBuilds();
+ teamcityIgnited.actualizeRecentBuildRefs();
List<BuildRef> hist = srv.getBuildHistory(buildTypeId, branchName);