You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/27 10:14:54 UTC
[ignite-teamcity-bot] 01/01: Standalone syncer class started;
Find missing builds by refs started
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-2
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 9da98d150c463bffede8afbbcec5551a55f69d79
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 27 13:14:50 2018 +0300
Standalone syncer class started; Find missing builds by refs started
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 1 -
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 21 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 190 ++++--------------
.../ci/teamcity/ignited/TeamcityIgnitedModule.java | 2 +
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 26 +++
.../ignited/fatbuild/ProactiveFatBuildSync.java | 221 +++++++++++++++++++++
.../ignite/ci/teamcity/pure/ITeamcityConn.java | 5 +
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 2 +-
8 files changed, 306 insertions(+), 162 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 92cdab8..12446b9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -59,7 +59,6 @@ public interface ITeamcity extends ITeamcityConn {
CompletableFuture<List<BuildType>> getProjectSuites(String projectId);
- String serverId();
/**
* @param projectId Suite ID (string without spaces).
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index d11f148..92b1684 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -17,13 +17,7 @@
package org.apache.ignite.ci.teamcity.ignited;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TreeMap;
+import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -223,4 +217,17 @@ public class BuildRefDao {
return false;
}
+
+ @AutoProfiling
+ public int[] getAllIds(int srvId) {
+ GridIntList res = new GridIntList(buildRefsCache.size());
+
+ StreamSupport.stream(buildRefsCache.spliterator(), false)
+ .map(Cache.Entry::getKey)
+ .filter(entry -> isKeyForServer(entry, srvId))
+ .map(BuildRefDao::cacheKeyToBuildId)
+ .forEach(res::add);
+
+ return res.array();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index f32d335..08e1b5f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -20,23 +20,6 @@ package org.apache.ignite.ci.teamcity.ignited;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.collect.Sets;
-import java.io.FileNotFoundException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.OptionalInt;
-import java.util.Set;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-import javax.annotation.concurrent.GuardedBy;
-import javax.inject.Inject;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
@@ -48,23 +31,30 @@ import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
import org.apache.ignite.ci.util.ExceptionUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.io.FileNotFoundException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Logger. */
- private static final Logger logger = LoggerFactory.getLogger(TestCompacted.class);
+ private static final Logger logger = LoggerFactory.getLogger(TeamcityIgnitedImpl.class);
/** Max build id diff to enforce reload during incremental refresh. */
public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000;
- public static final int FAT_BUILD_PROACTIVE_TASKS = 4;
/** Server id. */
- private String srvId;
+ private String srvNme;
/** Pure HTTP Connection API. */
private ITeamcityConn conn;
@@ -81,45 +71,27 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Build DAO. */
@Inject private FatBuildDao fatBuildDao;
- @Inject private IStringCompactor compactor;
+ @Inject private ProactiveFatBuildSync buildSync;
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
- @GuardedBy("this")
- private Set<Integer> buildToLoad = new HashSet<>();
-
public void init(String srvId, ITeamcityConn conn) {
- this.srvId = srvId;
+ this.srvNme = srvId;
this.conn = conn;
srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
buildRefDao.init(); //todo init somehow in auto
buildConditionDao.init();
fatBuildDao.init();
- }
-
- /**
- * Invoke load fat builds later, re-load provided builds.
- *
- * @param buildsToAskFromTc Builds to ask from tc.
- */
- public void scheduleBuildsLoad(Collection<Integer> buildsToAskFromTc) {
- if (buildsToAskFromTc.isEmpty())
- return;
-
- synchronized (this) {
- buildToLoad.addAll(buildsToAskFromTc);
- }
-
- int ldrToActivate = ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
-
- scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate), () -> loadFatBuilds(ldrToActivate), 2, TimeUnit.MINUTES);
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme);
}
- @NotNull public String taskName(String taskName) {
- return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." + srvId;
+
+ @NotNull
+ private String taskName(String taskName) {
+ return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." + srvNme;
}
/** {@inheritDoc} */
@@ -132,7 +104,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Override public List<BuildRef> getBuildHistory(
@Nullable String buildTypeId,
@Nullable String branchName) {
- scheduler.sheduleNamed(taskName("actualizeRecentBuilds"), this::actualizeRecentBuilds, 2, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
String bracnhNameQry ;
if (ITeamcity.DEFAULT.equals(branchName))
@@ -148,7 +120,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
Build build = conn.triggerBuild(buildTypeId, branchName, cleanRebuild, queueAtTop);
//todo may add additional parameter: load builds into DB in sync/async fashion
- runActualizeBuilds(srvId, false, Sets.newHashSet(build.getId()));
+ runActualizeBuildRefs(srvNme, false, Sets.newHashSet(build.getId()));
return build;
}
@@ -172,7 +144,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (existingBuild != null && !existingBuild.isOutdatedEntityVersion())
return existingBuild;
- FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild);
+ FatBuildCompacted savedVer = buildSync.reloadBuild(conn, buildId, existingBuild);
//build was modified, probably we need also to update reference accordindly
if (savedVer != null)
@@ -181,61 +153,12 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return savedVer == null ? existingBuild : savedVer;
}
- /**
- * @param buildId
- * @param existingBuild
- * @return new build if it was updated or null if no updates detected
- */
- public FatBuildCompacted reloadBuild(int buildId, @Nullable FatBuildCompacted existingBuild) {
- // System.err.println(Thread.currentThread().getName()+ ": Build " + buildId);
- //todo some sort of locking to avoid double requests
- Build build;
- List<TestOccurrencesFull> tests = new ArrayList<>();
- try {
- build = conn.getBuild(buildId);
-
- String nextHref = null;
- do {
- boolean testDtls = !build.isComposite(); // don't query test details for compoite
- TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls);
- nextHref = page.nextHref();
-
- tests.add(page);
- }
- while (!Strings.isNullOrEmpty(nextHref));
- }
- catch (Exception e) {
- if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
- logger.info("Loading build [" + buildId + "] for server [" + srvId + "] failed:" + e.getMessage(), e);
-
- if (existingBuild != null) {
- build = existingBuild.toBuild(compactor);
-
- if(build.isRunning() || build.isQueued())
- build.setCancelled();
-
- tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor));
- }
- else
- build = Build.createFakeStub();
- } else {
- logger.error("Loading build [" + buildId + "] for server [" + srvId + "] failed:" + e.getMessage(), e);
-
- e.printStackTrace();
-
- throw ExceptionUtil.propagateException(e);
- }
- }
- //if we are here because of some sort of outdated version of build,
- // new save will be performed with new entity version for compacted build
- return fatBuildDao.saveBuild(srvIdMaskHigh, buildId, build, tests, existingBuild);
- }
/**
*
*/
- void actualizeRecentBuilds() {
+ void actualizeRecentBuildRefs() {
List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
Set<Integer> paginateUntil = new HashSet<>();
@@ -252,23 +175,23 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
});
}
//schedule direct reload for Fat Builds for all queued too-old builds
- scheduleBuildsLoad(directUpload);
+ buildSync.scheduleBuildsLoad(srvNme, directUpload);
- runActualizeBuilds(srvId, false, paginateUntil);
+ runActualizeBuildRefs(srvNme, false, paginateUntil);
if(!paginateUntil.isEmpty()) {
//some builds may stuck in the queued or running, enforce loading as well
- scheduleBuildsLoad(paginateUntil);
+ buildSync.scheduleBuildsLoad(srvNme, paginateUntil);
}
// schedule full resync later
- scheduler.invokeLater(this::sheduleResync, 15, TimeUnit.MINUTES);
+ scheduler.invokeLater(this::sheduleResyncBuildRefs, 15, TimeUnit.MINUTES);
}
/**
*
*/
- private void sheduleResync() {
+ private void sheduleResyncBuildRefs() {
scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 120, TimeUnit.MINUTES);
}
@@ -276,25 +199,29 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
void fullReindex() {
- runActualizeBuilds(srvId, true, null);
+ runActualizeBuildRefs(srvNme, true, null);
+
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme);
}
+
/**
- * @param srvId Server id. todo to be added as composite name extend
+ * @param srvId Server id.
* @param fullReindex Reindex all builds from TC history.
* @param mandatoryToReload [in/out] Build ID should be found before end of sync. Ignored if fullReindex mode.
*
*/
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Actualize BuildRefs(srv, full resync)", nameExtArgsIndexes = {0, 1})
@AutoProfiling
- protected String runActualizeBuilds(String srvId, boolean fullReindex,
- @Nullable Set<Integer> mandatoryToReload) {
+ protected String runActualizeBuildRefs(String srvId, boolean fullReindex,
+ @Nullable Set<Integer> mandatoryToReload) {
AtomicReference<String> outLinkNext = new AtomicReference<>();
List<BuildRef> tcDataFirstPage = conn.getBuildRefs(null, outLinkNext);
Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh, tcDataFirstPage);
int totalUpdated = buildsUpdated.size();
- scheduleBuildsLoad(cacheKeysToBuildIds(buildsUpdated));
+ buildSync.scheduleBuildsLoad(srvNme, cacheKeysToBuildIds(buildsUpdated));
int totalChecked = tcDataFirstPage.size();
int neededToFind = 0;
@@ -310,7 +237,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
List<BuildRef> tcDataNextPage = conn.getBuildRefs(nextPageUrl, outLinkNext);
Set<Long> curChunkBuildsSaved = buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
totalUpdated += curChunkBuildsSaved.size();
- scheduleBuildsLoad(cacheKeysToBuildIds(curChunkBuildsSaved));
+ buildSync.scheduleBuildsLoad(srvNme, cacheKeysToBuildIds(curChunkBuildsSaved));
int savedCurChunk = curChunkBuildsSaved.size();
@@ -332,47 +259,4 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@NotNull private List<Integer> cacheKeysToBuildIds(Collection<Long> cacheKeysUpdated) {
return cacheKeysUpdated.stream().map(BuildRefDao::cacheKeyToBuildId).collect(Collectors.toList());
}
-
- /** */
- private void loadFatBuilds(int ldrNo) {
- Set<Integer> load;
-
- synchronized (this) {
- load = buildToLoad;
- buildToLoad = new HashSet<>();
- }
-
- doLoadBuilds(ldrNo, srvId, load);
- }
-
- @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1})
- @AutoProfiling
- protected String doLoadBuilds(int ldrNo, String srvId, Set<Integer> load) {
- if(load.isEmpty())
- return "Nothing to load";
-
- AtomicInteger err = new AtomicInteger();
- AtomicInteger ld = new AtomicInteger();
-
- Map<Long, FatBuildCompacted> builds = fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load);
-
- load.forEach(
- buildId -> {
- try {
- FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
-
- FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild);
-
- if (savedVer != null)
- ld.incrementAndGet();
- }
- catch (Exception e) {
- logger.error("", e);
- err.incrementAndGet();
- }
- }
- );
-
- return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err;
- }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
index e397d8f..9a5d68c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
@@ -20,6 +20,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.tcbot.condition.BuildConditionDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.teamcity.restcached.TcRestCachedModule;
import org.jetbrains.annotations.Nullable;
@@ -37,6 +38,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(BuildRefDao.class).in(new SingletonScope());
bind(BuildConditionDao.class).in(new SingletonScope());
bind(FatBuildDao.class).in(new SingletonScope());
+ bind(ProactiveFatBuildSync.class).in(new SingletonScope());
bind(IStringCompactor.class).to(IgniteStringCompactor.class).in(new SingletonScope());
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index fd8b8df..725c074 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -24,15 +24,20 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import javax.cache.Cache;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.validation.constraints.NotNull;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
+import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.internal.util.GridIntList;
import org.jetbrains.annotations.Nullable;
/**
@@ -120,4 +125,25 @@ public class FatBuildDao {
return buildsCache.getAll(ids);
}
+
+ /**
+ * @param key Key.
+ * @param srvId Server id.
+ */
+ private boolean isKeyForServer(Long key, int srvId) {
+ return key!=null && key >> 32 == srvId;
+ }
+
+ @AutoProfiling
+ public int[] getAllIds(int srvId) {
+ GridIntList res = new GridIntList(buildsCache.size());
+
+ StreamSupport.stream(buildsCache.spliterator(), false)
+ .map(Cache.Entry::getKey)
+ .filter(entry -> isKeyForServer(entry, srvId))
+ .map(BuildRefDao::cacheKeyToBuildId)
+ .forEach(res::add);
+
+ return res.array();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
new file mode 100644
index 0000000..2dd1757
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+
+import com.google.common.base.Strings;
+import com.google.common.base.Throwables;
+import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.di.MonitoredTask;
+import org.apache.ignite.ci.di.scheduler.IScheduler;
+import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
+import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
+import org.apache.ignite.ci.util.ExceptionUtil;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
+import javax.inject.Inject;
+import java.io.FileNotFoundException;
+import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ProactiveFatBuildSync {
+ public static final int FAT_BUILD_PROACTIVE_TASKS = 4;
+
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(ProactiveFatBuildSync.class);
+
+ /** Build reference DAO. */
+ @Inject
+ private BuildRefDao buildRefDao;
+
+ /** Build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
+ /** Scheduler. */
+ @Inject private IScheduler scheduler;
+
+ @Inject private IStringCompactor compactor;
+
+
+ @GuardedBy("this")
+ private Set<Integer> buildToLoad = new HashSet<>();
+
+
+ /**
+ * Invoke load fat builds later, re-load provided builds.
+ * @param srvNme
+ * @param buildsToAskFromTc Builds to ask from tc.
+ */
+ public void scheduleBuildsLoad(String srvNme , Collection<Integer> buildsToAskFromTc) {
+ if (buildsToAskFromTc.isEmpty())
+ return;
+
+ synchronized (this) {
+ buildToLoad.addAll(buildsToAskFromTc);
+ }
+
+ int ldrToActivate = ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
+
+ scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate, srvNme),
+ () -> loadFatBuilds(ldrToActivate, srvNme), 2, TimeUnit.MINUTES);
+
+ }
+
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0})
+ @AutoProfiling
+ protected String findMissingBuildsFromBuildRef(String srvId) {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
+ final int[] fatBuildKeys = fatBuildDao.getAllIds(srvIdMaskHigh);
+
+ Arrays.parallelSort(buildRefKeys);
+ Arrays.parallelSort(fatBuildKeys);
+ /* ;
+ */
+
+ return "";
+ }
+
+
+
+ /** */
+ private void loadFatBuilds(int ldrNo, String srvNme) {
+ Set<Integer> load;
+
+ synchronized (this) {
+ load = buildToLoad;
+ buildToLoad = new HashSet<>();
+ }
+
+ doLoadBuilds(ldrNo, srvNme, conn, load);
+ }
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1})
+ @AutoProfiling
+ protected String doLoadBuilds(int ldrNo, String srvId, ITeamcityConn conn, Set<Integer> load) {
+ if(load.isEmpty())
+ return "Nothing to load";
+
+ final int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ AtomicInteger err = new AtomicInteger();
+ AtomicInteger ld = new AtomicInteger();
+
+ Map<Long, FatBuildCompacted> builds = fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load);
+
+ load.forEach(
+ buildId -> {
+ try {
+ FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
+
+ FatBuildCompacted savedVer = reloadBuild(conn, buildId, existingBuild);
+
+ if (savedVer != null)
+ ld.incrementAndGet();
+ }
+ catch (Exception e) {
+ logger.error("", e);
+ err.incrementAndGet();
+ }
+ }
+ );
+
+ return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err;
+ }
+
+ @NotNull
+ private String taskName(String taskName, String srvName) {
+ return ProactiveFatBuildSync.class.getSimpleName() +"." + taskName + "." + srvName;
+ }
+
+ public void invokeLaterFindMissingByBuildRef(String srvName) {
+ scheduler.sheduleNamed(taskName("findMissingBuildsFromBuildRef", srvName),
+ () -> findMissingBuildsFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+ }
+
+ /**
+ *
+ * @param conn
+ * @param buildId
+ * @param existingBuild
+ * @return new build if it was updated or null if no updates detected
+ */
+ @SuppressWarnings({"WeakerAccess"})
+ @AutoProfiling
+ public FatBuildCompacted reloadBuild(ITeamcityConn conn, int buildId, @Nullable FatBuildCompacted existingBuild) {
+ // System.err.println(Thread.currentThread().getName()+ ": Build " + buildId);
+ //todo some sort of locking to avoid double requests
+
+ final String srvNme = conn.serverId();
+ final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvNme);
+
+ Build build;
+ List<TestOccurrencesFull> tests = new ArrayList<>();
+ try {
+ build = conn.getBuild(buildId);
+
+ String nextHref = null;
+ do {
+ boolean testDtls = !build.isComposite(); // don't query test details for compoite
+ TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls);
+ nextHref = page.nextHref();
+
+ tests.add(page);
+ }
+ while (!Strings.isNullOrEmpty(nextHref));
+ }
+ catch (Exception e) {
+ if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
+ logger.info("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
+
+ if (existingBuild != null) {
+ build = existingBuild.toBuild(compactor);
+
+ if(build.isRunning() || build.isQueued())
+ build.setCancelled();
+
+ tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor));
+ }
+ else
+ build = Build.createFakeStub();
+ } else {
+ logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
+
+ e.printStackTrace();
+
+ throw ExceptionUtil.propagateException(e);
+ }
+ }
+
+ //if we are here because of some sort of outdated version of build,
+ // new save will be performed with new entity version for compacted build
+ return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, existingBuild);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
index 78f3d34..f832088 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
@@ -30,6 +30,11 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
*/
public interface ITeamcityConn {
/**
+ * @return Internal server ID as string
+ */
+ String serverId();
+
+ /**
* @return Normalized Host address, ends with '/'.
*/
public String host();
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index b07e102..173a959 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -214,7 +214,7 @@ public class IgnitedTcInMemoryIntegrationTest {
for (int i = queuedBuildIdx; i < tcBuilds.size(); i++)
tcBuilds.get(i).state = BuildRef.STATE_FINISHED;
- teamcityIgnited.actualizeRecentBuilds();
+ teamcityIgnited.actualizeRecentBuildRefs();
List<BuildRef> hist = srv.getBuildHistory(buildTypeId, branchName);