You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/07/01 08:20:58 UTC
[ignite-teamcity-bot] branch master updated: Issue detection bug
fixed: build TS problem fixed
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 5573467 Issue detection bug fixed: build TS problem fixed
5573467 is described below
commit 5573467d04eedbe9f79dd2b19e9675b76af78985
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Mon Jul 1 11:20:58 2019 +0300
Issue detection bug fixed: build TS problem fixed
---
.../java/org/apache/ignite/ci/issue/ChangeUi.java | 10 +++
.../java/org/apache/ignite/ci/issue/Issue.java | 6 +-
.../ignite/ci/runners/RemoteClientTmpHelper.java | 80 ++++++++++++++++------
.../ignite/ci/tcbot/issue/IssueDetector.java | 59 +++++++++-------
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
5 files changed, 111 insertions(+), 46 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java
index 11496d9..5b433b5 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java
@@ -17,6 +17,8 @@
package org.apache.ignite.ci.issue;
+import com.google.common.base.MoreObjects;
+
public class ChangeUi {
public final String username;
public final String webUrl;
@@ -41,4 +43,12 @@ public class ChangeUi {
public String toPlainText() {
return username + " " + webUrl;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("username", username)
+ .add("webUrl", webUrl)
+ .toString();
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java
index 05236a0..1777b8f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java
@@ -57,7 +57,7 @@ public class Issue {
@Nullable
public String displayName;
- /** Build start timestamp. */
+ /** Build start timestamp. Builds which is older that 10 days not notified. */
@Nullable public Long buildStartTs;
/** Detected timestamp. */
@@ -66,11 +66,13 @@ public class Issue {
/** Set of build tags detected. */
public Set<String> buildTags = new TreeSet<>();
- public Issue(IssueKey issueKey, IssueType type) {
+ public Issue(IssueKey issueKey, IssueType type,
+ @Nullable Long buildStartTs) {
this.issueKey = issueKey;
this.detectedTs = System.currentTimeMillis();
this.type = type.code();
this.displayType = type.displayName();
+ this.buildStartTs = buildStartTs;
}
public void addChange(String username, String webUrl) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
index 87beec5..179d473 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
@@ -19,9 +19,9 @@ package org.apache.ignite.ci.runners;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
-import java.io.BufferedWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.time.Duration;
@@ -32,22 +32,25 @@ import javax.xml.bind.JAXBException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
+import org.apache.ignite.ci.issue.Issue;
+import org.apache.ignite.ci.issue.IssueKey;
+import org.apache.ignite.ci.issue.IssuesStorage;
import org.apache.ignite.ci.tcbot.user.UserAndSessionsStorage;
-import org.apache.ignite.tcservice.model.hist.BuildRef;
-import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
-import org.apache.ignite.tcignited.ITeamcityIgnited;
-import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
-import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.tcignited.build.FatBuildDao;
-import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.ci.user.TcHelperUser;
-import org.apache.ignite.tcservice.util.XmlUtil;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.logger.slf4j.Slf4jLogger;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.history.RunHistCompactedDao;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
+import org.apache.ignite.tcservice.model.result.Build;
+import org.apache.ignite.tcservice.util.XmlUtil;
import org.jetbrains.annotations.NotNull;
import static org.apache.ignite.tcignited.history.RunHistCompactedDao.BUILD_START_TIME_CACHE_NAME;
@@ -64,10 +67,38 @@ public class RemoteClientTmpHelper {
* @param args Args.
*/
public static void main(String[] args) {
- // mainDumpFatBuildStartTime(args);
+ // mainDumpIssues(args);
System.err.println("Please insert option of main");
}
+ public static void mainDumpIssues(String[] args) {
+ try (Ignite ignite = tcbotServerConnectedClient()) {
+ IgniteCache<IssueKey, Issue> bst = ignite.cache(IssuesStorage.BOT_DETECTED_ISSUES);
+ Iterator<Cache.Entry<IssueKey, Issue>> iter = bst.iterator();
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(),
+ "Issues.txt")))) {
+ while (iter.hasNext()) {
+ Cache.Entry<IssueKey, Issue> next = iter.next();
+
+ Issue val = next.getValue();
+ long ageDays = -1;
+ if (val != null && val.buildStartTs != null)
+ ageDays = Duration.ofMillis(System.currentTimeMillis() - val.buildStartTs).toDays();
+
+ writer.write(next.getKey() + " " + val + " " +
+ ageDays + "\n");
+ }
+
+ writer.flush();
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ dumpDictionary(ignite);
+ }
+ }
+
public static void mainDumpFatBuildStartTime(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
@@ -255,18 +286,12 @@ public class RemoteClientTmpHelper {
cfg.setDiscoverySpi(spi);
}
- public static void mainExport(String[] args) throws IOException {
+ public static void mainExport(String[] args) {
final Ignite ignite = tcbotServerConnectedClient();
- if (dumpDict) {
- IgniteCache<String, Object> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE);
- try (FileWriter writer = new FileWriter("Dictionary.txt")) {
- for (Cache.Entry<String, Object> next1 : strings) {
- writer.write(next1.getValue().toString()
- + "\n");
- }
- }
- }
+ if (dumpDict)
+ dumpDictionary(ignite);
+
if (false) {
IgniteCache<Long, FatBuildCompacted> cache1 = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
@@ -283,6 +308,21 @@ public class RemoteClientTmpHelper {
}
}
+ public static void dumpDictionary(Ignite ignite) {
+ IgniteCache<String, Object> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE);
+ try {
+ try (FileWriter writer = new FileWriter(new File(dumpsDir(), "Dictionary.txt"))) {
+ for (Cache.Entry<String, Object> next1 : strings) {
+ writer.write(next1.getValue().toString()
+ + "\n");
+ }
+ }
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
public static Ignite tcbotServerConnectedClient() {
final IgniteConfiguration cfg = new IgniteConfiguration();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index 62842f0..b545e14 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -19,28 +19,45 @@ package org.apache.ignite.ci.tcbot.issue;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
-import org.apache.ignite.ci.issue.*;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nonnull;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import org.apache.ignite.ci.issue.Issue;
+import org.apache.ignite.ci.issue.IssueKey;
+import org.apache.ignite.ci.issue.IssueType;
import org.apache.ignite.ci.jobs.CheckQueueJob;
import org.apache.ignite.ci.mail.EmailSender;
import org.apache.ignite.ci.mail.SlackSender;
-import org.apache.ignite.tcbot.engine.issue.EventTemplate;
-import org.apache.ignite.tcbot.engine.issue.EventTemplates;
-import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.ci.tcbot.user.IUserStorage;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.ci.user.TcHelperUser;
-import org.apache.ignite.tcbot.engine.ui.DsChainUi;
-import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
-import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
-import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.engine.conf.INotificationChannel;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.NotificationsConfig;
+import org.apache.ignite.tcbot.engine.issue.EventTemplate;
+import org.apache.ignite.tcbot.engine.issue.EventTemplates;
+import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
+import org.apache.ignite.tcbot.engine.ui.DsChainUi;
+import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
+import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
+import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
@@ -50,17 +67,6 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nonnull;
-import javax.inject.Inject;
-import javax.inject.Provider;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
@@ -150,9 +156,17 @@ public class IssueDetector {
long detected = issue.detectedTs == null ? 0 : issue.detectedTs;
long issueAgeMs = System.currentTimeMillis() - detected;
- return issueAgeMs <= TimeUnit.HOURS.toMillis(2);
+ long bound = TimeUnit.HOURS.toMillis(2);
+ //temporary to issue missed notifications
+ if (issue.buildStartTs == null)
+ bound = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS) / 2;
+
+ return issueAgeMs <= bound;
})
.filter(issue -> {
+ if (issue.buildStartTs == null)
+ return true; // exception due to bug in issue detection; field was not filled
+
long buildStartTs = issue.buildStartTs == null ? 0 : issue.buildStartTs;
long buildAgeMs = System.currentTimeMillis() - buildStartTs;
long maxBuildAgeToNotify = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS) / 2;
@@ -337,11 +351,10 @@ public class IssueDetector {
@NotNull
private Issue createIssueForSuite(ITeamcityIgnited tcIgnited, DsSuiteUi suiteFailure, String trackedBranch,
IssueKey issueKey, IssueType issType) {
- Issue issue = new Issue(issueKey, issType);
+ Issue issue = new Issue(issueKey, issType, tcIgnited.getBuildStartTs(issueKey.buildId));
issue.trackedBranchName = trackedBranch;
issue.displayName = suiteFailure.name;
issue.webUrl = suiteFailure.webToHist;
- issue.buildStartTs = tcIgnited.getBuildStartTs(issueKey.buildId);
issue.buildTags.addAll(suiteFailure.tags);
@@ -419,7 +432,7 @@ public class IssueDetector {
if (issuesStorage.containsIssueKey(issueKey))
return false; //duplicate
- Issue issue = new Issue(issueKey, type);
+ Issue issue = new Issue(issueKey, type, tcIgnited.getBuildStartTs(issueKey.buildId));
issue.trackedBranchName = trackedBranch;
issue.displayName = testFailure.testName;
issue.webUrl = testFailure.webUrl;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index e8b8258..26fc178 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
/** TC Bot Version. */
- public static final String VERSION = "20190626";
+ public static final String VERSION = "20190701";
/** Java version, where Web App is running. */
public String javaVer;