You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/30 18:31:53 UTC
[ignite-teamcity-bot] branch master updated: IGNITE-10071
Background upload of a build and simultaneous bot restart may result in a
queued build persisted but actually build is finished - Fixes #55.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 89a1fa1 IGNITE-10071 Background upload of a build and simultaneous bot restart may result in a queued build persisted but actually build is finished - Fixes #55.
89a1fa1 is described below
commit 89a1fa1085b6c288452ac0f04001171a2d1b3213
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Oct 30 21:31:43 2018 +0300
IGNITE-10071 Background upload of a build and simultaneous bot restart may result in a queued build persisted but actually build is finished - Fixes #55.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
---
.../ignite/ci/runners/RemoteClientTmpHelper.java | 72 ++++++++++++++--------
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 7 +--
.../ci/teamcity/ignited/BuildRefCompacted.java | 11 ++++
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 2 +-
.../ci/teamcity/ignited/ITeamcityIgnited.java | 11 +++-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 16 +++--
.../ignited/fatbuild/FatBuildCompacted.java | 19 ++++++
.../ignited/fatbuild/ProblemCompacted.java | 24 +++-----
.../teamcity/ignited/fatbuild/TestCompacted.java | 33 ++++++----
9 files changed, 133 insertions(+), 62 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
index 043883a..9c40870 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
@@ -17,20 +17,28 @@
package org.apache.ignite.ci.runners;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import java.io.FileWriter;
+import java.io.IOException;
import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
-import org.apache.ignite.ci.IgnitePersistentTeamcity;
-import org.apache.ignite.ci.user.TcHelperUser;
-import org.apache.ignite.ci.user.UserAndSessionsStorage;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
+import org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.logger.slf4j.Slf4jLogger;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
public class RemoteClientTmpHelper {
+ private static boolean dumpDict = false;
+
private static void setupDisco(IgniteConfiguration cfg) {
final TcpDiscoverySpi spi = new TcpDiscoverySpi();
final int locPort = 54433;
@@ -44,7 +52,7 @@ public class RemoteClientTmpHelper {
cfg.setDiscoverySpi(spi);
}
- public static void main(String[] args) {
+ public static void main(String[] args) throws IOException {
final IgniteConfiguration cfg = new IgniteConfiguration();
setupDisco(cfg);
@@ -54,36 +62,48 @@ public class RemoteClientTmpHelper {
cfg.setClientMode(true);
final Ignite ignite = Ignition.start(cfg);
- ignite.cluster().active(true);
-
- IgniteCache<Object, Object> cache1 = ignite.cache(UserAndSessionsStorage.USERS);
- for (Cache.Entry<Object, Object> next : cache1) {
- System.out.println(next.getKey() + ": " + next.getValue());
-
- if (next.getKey().equals("someusername")) {
- TcHelperUser u = (TcHelperUser)next.getValue();
- u.resetCredentials();
-
- cache1.put(next.getKey(), u);
+ if(dumpDict) {
+ IgniteCache<String, Object> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE);
+ try (FileWriter writer = new FileWriter("Dictionary.txt")) {
+ for (Cache.Entry<String, Object> next1 : strings) {
+ writer.write(next1.getValue().toString()
+ + "\n");
+ }
}
}
+ IgniteCache<Long, FatBuildCompacted> cache1 = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
- IgniteCache<Object, Object> cache = ignite.cache(IgnitePersistentTeamcity.BOT_DETECTED_ISSUES);
- for (Cache.Entry<Object, Object> next : cache) {
- Object key = next.getKey();
- Object value = next.getValue();
+ int apache = ITeamcityIgnited.serverIdToInt("apache");
- if (key.toString().contains("GridCacheLifecycleAwareSelfTest.testLifecycleAware")) {
- /*boolean remove = cache.remove(key);
+ int id = 2200135;
+ int id1 = 2200209;
+ dumpFatBuild(cache1, apache, id);
+ dumpFatBuild(cache1, apache, id1);
- if (remove)
- System.err.println("Removed issue " + value);*/
- System.err.println("Issue: " + value);
- }
+ IgniteCache<Long, BuildRefCompacted> cache2 = ignite.cache(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME);
+ dumpBuildRef(cache2, apache, id);
+ dumpBuildRef(cache2, apache, id1);
+ ignite.close();
+
+ }
+
+ public static void dumpBuildRef(IgniteCache<Long, BuildRefCompacted> cache, int apache, int id) throws IOException {
+ long l = BuildRefDao.buildIdToCacheKey(apache, id);
+ BuildRefCompacted compacted = cache.get(l);
+ Preconditions.checkNotNull(compacted, "Can't find build by ID " + id);
+ try (FileWriter writer = new FileWriter("BuildRef " + id + ".txt")) {
+ writer.write(compacted.toString());
}
+ }
- ignite.close();
+ public static void dumpFatBuild(IgniteCache<Long, FatBuildCompacted> cache, int apache, int id) throws IOException {
+ long l = FatBuildDao.buildIdToCacheKey(apache, id);
+ FatBuildCompacted compacted = cache.get(l);
+ Preconditions.checkNotNull(compacted, "Can't find build by ID " + id);
+ try (FileWriter writer = new FileWriter("Build " + id + ".txt")) {
+ writer.write(compacted.toString());
+ }
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 1295510..1e8adb6 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -155,7 +155,6 @@ public class BuildChainProcessor {
return fullChainRunCtx;
}
-
@SuppressWarnings("WeakerAccess")
@AutoProfiling
protected void createCxt(ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap,
@@ -200,13 +199,13 @@ public class BuildChainProcessor {
final String branch = getBranchOrDefault(buildCompacted.branchName(compactor));
final String buildTypeId = buildCompacted.buildTypeId(compactor);
- Stream<BuildRefCompacted> history = teamcityIgnited.getBuildHistoryCompacted(buildTypeId, branch)
+ Stream<BuildRefCompacted> hist = teamcityIgnited.getBuildHistoryCompacted(buildTypeId, branch)
.stream()
.filter(t -> t.isNotCancelled(compactor))
.filter(t -> t.isFinished(compactor));
if (includeLatestRebuild == LatestRebuildMode.LATEST) {
- BuildRefCompacted recentRef = history.max(Comparator.comparing(BuildRefCompacted::id))
+ BuildRefCompacted recentRef = hist.max(Comparator.comparing(BuildRefCompacted::id))
.orElse(buildCompacted);
return Stream.of(recentRef)
@@ -214,7 +213,7 @@ public class BuildChainProcessor {
}
if (includeLatestRebuild == LatestRebuildMode.ALL) {
- return history
+ return hist
.sorted(Comparator.comparing(BuildRefCompacted::id).reversed())
.limit(cntLimit)
.map(b -> builds.computeIfAbsent(b.id(), teamcityIgnited::getFatBuild));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index 6b84357..a3aff88 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -16,6 +16,7 @@
*/
package org.apache.ignite.ci.teamcity.ignited;
+import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.ci.db.Persisted;
@@ -172,4 +173,14 @@ public class BuildRefCompacted {
public boolean isQueued(IStringCompactor compactor) {
return compactor.getStringId(STATE_QUEUED) == state();
}
+
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("id", id)
+ .add("buildTypeId", buildTypeId)
+ .add("branchName", branchName)
+ .add("status", status)
+ .add("state", state)
+ .toString();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index d871f84..0d7467a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -115,7 +115,7 @@ public class BuildRefDao {
* @param srvId Server id mask high.
* @param buildId Build id.
*/
- public long buildIdToCacheKey(long srvId, int buildId) {
+ public static long buildIdToCacheKey(long srvId, int buildId) {
return (long)buildId | srvId << 32;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
index 5fea687..cfdaa2e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
@@ -97,7 +97,16 @@ public interface ITeamcityIgnited {
/**
* @param id Id.
*/
- public FatBuildCompacted getFatBuild(int id);
+ public default FatBuildCompacted getFatBuild(int id) {
+ return getFatBuild(id, false);
+ }
+
+
+
+ /**
+ * @param id Id.
+ */
+ public FatBuildCompacted getFatBuild(int id, boolean acceptQueued);
public Collection<ChangeCompacted> getAllChanges(int[] changeIds);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 67ea0c2..ce05e93 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -83,6 +83,9 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Changes DAO. */
@Inject private ChangeSync changeSync;
+ /** Changes DAO. */
+ @Inject private IStringCompactor compactor;
+
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
@@ -172,13 +175,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return buildConditionDao.setBuildCondition(srvIdMaskHigh, cond);
}
- @Override public FatBuildCompacted getFatBuild(int buildId) {
+ /** {@inheritDoc} */
+ @Override public FatBuildCompacted getFatBuild(int buildId, boolean acceptQueued) {
ensureActualizeRequested();
FatBuildCompacted existingBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, buildId);
- //todo additionally check queued and running builds, refesh builds if they are queued.
- if (existingBuild != null && !existingBuild.isOutdatedEntityVersion())
- return existingBuild;
+ if (existingBuild != null && !existingBuild.isOutdatedEntityVersion()) {
+ boolean finished = !existingBuild.isRunning(compactor) && !existingBuild.isQueued(compactor);
+
+ if(finished || acceptQueued)
+ return existingBuild;
+ }
FatBuildCompacted savedVer = buildSync.reloadBuild(conn, buildId, existingBuild);
@@ -189,6 +196,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return savedVer == null ? existingBuild : savedVer;
}
+ /** {@inheritDoc} */
@AutoProfiling
@Override public Collection<ChangeCompacted> getAllChanges(int[] changeIds) {
final Map<Long, ChangeCompacted> all = changesDao.getAll(srvIdMaskHigh, changeIds);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index aeeb644..04f2a63 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -16,6 +16,7 @@
*/
package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import org.apache.ignite.ci.analysis.IVersionedEntity;
import org.apache.ignite.ci.db.Persisted;
@@ -374,4 +375,22 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
return snapshotDeps.clone();
}
+
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("_", super.toString())
+ .add("_ver", _ver)
+ .add("startDate", startDate)
+ .add("finishDate", finishDate)
+ .add("queuedDate", queuedDate)
+ .add("projectId", projectId)
+ .add("name", name)
+ .add("tests", tests)
+ .add("snapshotDeps", snapshotDeps)
+ .add("flags", flags)
+ .add("problems", problems)
+ .add("statistics", statistics)
+ .add("changesIds", changesIds)
+ .toString();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
index 961dbe8..f15c418 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
@@ -17,27 +17,14 @@
package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.base.Strings;
-import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
-import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
-import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.xerial.snappy.Snappy;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.BitSet;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
import static org.apache.ignite.ci.analysis.RunStat.extractIdPrefixed;
@@ -160,4 +147,13 @@ public class ProblemCompacted {
return compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR) == type
|| compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR_BUILD_PROCEEDS_TYPE) == type;
}
+
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("id", id)
+ .add("type", type)
+ .add("identity", identity)
+ .add("actualBuildId", actualBuildId)
+ .toString() + "\n";
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index dfd0c1e..753a0ba 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -17,18 +17,16 @@
package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.base.Strings;
-
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.BitSet;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
-
import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
@@ -197,18 +195,17 @@ public class TestCompacted {
logger.error("Snappy.uncompress failed: " + e.getMessage(), e);
return null;
}
- } else if(flag1 && !flag2) {
+ } else if(flag1 && !flag2)
return new String(details, StandardCharsets.UTF_8);
- } else if (!flag1 && flag2) {
+ else if (!flag1 && flag2) {
try {
final ByteArrayInputStream in = new ByteArrayInputStream(details);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try (final GZIPInputStream gzi = new GZIPInputStream(in)) {
byte[] outbuf = new byte[details.length];
int len;
- while ((len = gzi.read(outbuf, 0, outbuf.length)) != -1) {
+ while ((len = gzi.read(outbuf, 0, outbuf.length)) != -1)
bos.write(outbuf, 0, len);
- }
}
return new String(bos.toByteArray(), StandardCharsets.UTF_8);
@@ -216,13 +213,12 @@ public class TestCompacted {
logger.error("GZip.uncompress failed: " + e.getMessage(), e);
return null;
}
- } else {
+ } else
return null;
- }
}
- public void setDetails(String dtlsString) {
- if (Strings.isNullOrEmpty(dtlsString)) {
+ public void setDetails(String dtlsStr) {
+ if (Strings.isNullOrEmpty(dtlsStr)) {
this.details = null;
return;
}
@@ -232,7 +228,7 @@ public class TestCompacted {
byte[] snappy = null;
byte[] gzip = null;
try {
- uncompressed = dtlsString.getBytes(StandardCharsets.UTF_8);
+ uncompressed = dtlsStr.getBytes(StandardCharsets.UTF_8);
} catch (Exception e) {
logger.error("Set details failed: " + e.getMessage(), e);
return;
@@ -353,4 +349,17 @@ public class TestCompacted {
public Integer getDuration() {
return duration < 0 ? null : duration;
}
+
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("idInBuild", idInBuild)
+ .add("name", name)
+ .add("status", status)
+ .add("duration", duration)
+ .add("flags", flags)
+ .add("testId", testId)
+ .add("actualBuildId", actualBuildId)
+ .add("details", details)
+ .toString() + "\n";
+ }
}