You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/07/02 11:28:09 UTC
[ignite-teamcity-bot] branch master updated: IGNITE-11947:
Extracting heuristics into constants
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 26622f3 IGNITE-11947: Extracting heuristics into constants
26622f3 is described below
commit 26622f3f5b9b65e51dad637b2fc745ded898daea
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Jul 2 14:27:59 2019 +0300
IGNITE-11947: Extracting heuristics into constants
---
.../ignite/ci/tcbot/issue/IssueDetector.java | 14 +++++--
.../ci/tcbot/chain/PrChainsProcessorTest.java | 3 +-
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 2 +-
.../org/apache/ignite/tcbot/common/TcBotConst.java | 46 ++++++++++++++++++++++
.../tcbot/engine/chain/TestCompactedMult.java | 3 +-
.../teamcity/ignited/runhist/InvocationData.java | 3 +-
.../teamcity/ignited/runhist/RunHistCompacted.java | 7 ++--
.../ignite/tcignited/history/HistoryCollector.java | 6 +--
.../tcignited/history/RunHistCompactedDao.java | 2 +
.../ignite/tcignited/history/RunHistSync.java | 6 +++
10 files changed, 79 insertions(+), 13 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index 5ec0a98..ad0c630 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -42,9 +42,9 @@ import org.apache.ignite.ci.mail.SlackSender;
import org.apache.ignite.ci.tcbot.user.IUserStorage;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.ci.user.TcHelperUser;
+import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.engine.conf.INotificationChannel;
@@ -160,7 +160,15 @@ public class IssueDetector {
long detected = issue.detectedTs == null ? 0 : issue.detectedTs;
long issueAgeMs = System.currentTimeMillis() - detected;
- long bound = TimeUnit.HOURS.toMillis(2);
+ //here boundary can be not an absolute, but some ts when particular notification channel config was changed
+ // alternatively boundary may depend to issue notification histroy
+
+ boolean neverNotified = issue.addressNotified == null || issue.addressNotified.isEmpty();
+ // if issue had a prior notification, limit age by 2 hours to avoid new addresses spamming.
+ // otherwise check last day issues if it is notifiable
+ long bound = TimeUnit.HOURS.toMillis(neverNotified
+ ? TcBotConst.NOTIFY_MAX_AGE_SINCE_DETECT_HOURS
+ : TcBotConst.NOTIFY_MAX_AGE_SINCE_DETECT_FOR_NOTIFIED_ISSUE_HOURS );
return issueAgeMs <= bound;
})
@@ -171,7 +179,7 @@ public class IssueDetector {
long buildStartTs = issue.buildStartTs == null ? 0 : issue.buildStartTs;
long buildAgeMs = System.currentTimeMillis() - buildStartTs;
- long maxBuildAgeToNotify = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS) / 2;
+ long maxBuildAgeToNotify = TimeUnit.DAYS.toMillis(TcBotConst.NOTIFY_MAX_AGE_SINCE_START_DAYS) / 2;
return buildAgeMs <= maxBuildAgeToNotify;
})
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
index f38ce55..d4e93f6 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
+import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.engine.pr.PrChainsProcessor;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.model.conf.BuildType;
@@ -222,7 +223,7 @@ public class PrChainsProcessorTest {
addBuildsToEmulatedStor(cache1InMaster);
}
- long ageMs = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS);
+ long ageMs = TimeUnit.DAYS.toMillis(TcBotConst.HISTORY_MAX_DAYS);
for (int i = 0; i < 134; i++) {
addBuildsToEmulatedStor(createFailedBuild(c, CACHE_1,
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
index 9a0d174..0c81bb9 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
@@ -179,7 +179,7 @@ public class TeamcityIgnitedMock {
Invocation inv = testCompacted.toInvocation(build, (k, v) -> true, successStatusStrId);
- hist.addInvocation(inv);
+ hist.innerAddInvocation(inv);
});
}
diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
new file mode 100644
index 0000000..3dd56ee
--- /dev/null
+++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.common;
+
+/**
+ * TC Bot constants: contains magic numbers for project.
+ *
+ * Usually it is a practically set up rules, heuristics used by the bot.
+ */
+public class TcBotConst {
+ /** Max days to keep test invocatoin data in run statistics: affects Bot Visa. */
+ public static final int HISTORY_MAX_DAYS = 21;
+
+ /** History collection process: build id per server ID border days. */
+ public static final int HISTORY_BUILD_ID_BORDER_DAYS = HISTORY_MAX_DAYS + 2;
+
+ /** Notify about failure: max build age days (since build start time). */
+ public static final int NOTIFY_MAX_AGE_SINCE_START_DAYS = HISTORY_MAX_DAYS / 2;
+
+ /** */
+ public static final int NOTIFY_MAX_AGE_SINCE_DETECT_HOURS = 2;
+
+ /** */
+ public static final int NOTIFY_MAX_AGE_SINCE_DETECT_FOR_NOTIFIED_ISSUE_HOURS = 24;
+
+ /** Flakyness status change border: Count of test change status before considered as flaky. */
+ public static final int FLAKYNESS_STATUS_CHANGE_BORDER = 1;
+
+ /** Non flaky test failure rate: less that this failure rate in base branch is still blocker border, percents. */
+ public static final double NON_FLAKY_TEST_FAIL_RATE_BLOCKER_BORDER_PERCENTS = 4.;
+}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
index 7d5d3c3..621c7cd 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
@@ -23,6 +23,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.history.IRunHistSummary;
@@ -102,7 +103,7 @@ public class TestCompactedMult implements IMultTestOccurrence {
boolean flaky = baseBranchStat.isFlaky();
float failRate = baseBranchStat.getFailRate();
- boolean lowFailureRate = failRate * 100.0f < 4.;
+ boolean lowFailureRate = failRate * 100.0f < TcBotConst.NON_FLAKY_TEST_FAIL_RATE_BLOCKER_BORDER_PERCENTS;
if (lowFailureRate && !flaky) {
String runStatPrintable = IRunStat.getPercentPrintable(failRate * 100.0f);
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
index f02d286..87f3542 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
@@ -27,6 +27,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.persistence.Persisted;
import org.apache.ignite.tcignited.history.RunStatus;
@@ -38,7 +39,7 @@ import javax.annotation.Nonnull;
@Persisted
public class InvocationData {
/** Max days to keep test invocatoin data in run statistics: affects Bot Visa. */
- public static final int MAX_DAYS = 21;
+ public static final int MAX_DAYS = TcBotConst.HISTORY_MAX_DAYS;
/** Muted. */
public static final int MUTED = RunStatus.RES_MUTED_FAILURE.getCode();
/** Failure. */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
index 4a0cd2e..b4b5d79 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
+import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.persistence.IVersionedEntity;
import org.apache.ignite.tcbot.persistence.Persisted;
import org.apache.ignite.tcignited.history.ChangesState;
@@ -36,7 +37,6 @@ import org.apache.ignite.tcignited.history.RunStatus;
public class RunHistCompacted implements IVersionedEntity, IRunHistory {
/** Latest version. */
private static final int LATEST_VERSION = 1;
- public static final int FLAKYNESS_STATUS_CHANGE_BORDER = 1;
/** Entity fields version. */
@SuppressWarnings("FieldCanBeLocal")
@@ -92,7 +92,7 @@ public class RunHistCompacted implements IVersionedEntity, IRunHistory {
@Override public String getFlakyComments() {
int statusChange = getStatusChangesWithoutCodeModification();
- if (statusChange < FLAKYNESS_STATUS_CHANGE_BORDER)
+ if (statusChange < TcBotConst.FLAKYNESS_STATUS_CHANGE_BORDER)
return null;
return "Test seems to be flaky: " +
@@ -120,7 +120,7 @@ public class RunHistCompacted implements IVersionedEntity, IRunHistory {
/** {@inheritDoc} */
@Override public boolean isFlaky() {
- return getStatusChangesWithoutCodeModification() >= FLAKYNESS_STATUS_CHANGE_BORDER;
+ return getStatusChangesWithoutCodeModification() >= TcBotConst.FLAKYNESS_STATUS_CHANGE_BORDER;
}
/** {@inheritDoc} */
@@ -132,6 +132,7 @@ public class RunHistCompacted implements IVersionedEntity, IRunHistory {
* @param inv Invocation.
* @return if test run is new and is not expired.
*/
+ @Deprecated
public boolean addInvocation(Invocation inv) {
return data.addInvocation(inv);
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index 096aa64..e9ca205 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -39,8 +39,8 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
-import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
+import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
@@ -176,7 +176,7 @@ public class HistoryCollector {
long ageInDays = Duration.ofMillis(curTs - startTime).toDays();
- if (ageInDays > InvocationData.MAX_DAYS + 2) {
+ if (ageInDays > TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS) {
AtomicInteger integer = biggestBuildIdOutOfHistoryScope.computeIfAbsent(srvId,
s -> {
AtomicInteger atomicInteger = new AtomicInteger();
@@ -190,7 +190,7 @@ public class HistoryCollector {
logger.info("History Collector: New border for server was set " + bId);
}
- return ageInDays < InvocationData.MAX_DAYS;
+ return ageInDays < TcBotConst.HISTORY_MAX_DAYS;
}
).collect(Collectors.toSet());
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index 27dc742..5a18465 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -72,6 +72,7 @@ public class RunHistCompactedDao {
private IgniteCache<RunHistKey, RunHistCompacted> testHistCache;
/** Suite history cache. */
+ @Deprecated
private IgniteCache<RunHistKey, RunHistCompacted> suiteHistCache;
/** Build start time. */
@@ -157,6 +158,7 @@ public class RunHistCompactedDao {
}
@AutoProfiling
+ @Deprecated
public Integer addTestInvocations(RunHistKey histKey, List<Invocation> list) {
if (list.isEmpty())
return 0;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
index 9a621ee..987473f 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java
@@ -104,6 +104,7 @@ public class RunHistSync {
* @param srvCode Server code (internal identification).
* @param build Build.
*/
+ @Deprecated
public void saveToHistoryLater(String srvCode, FatBuildCompacted build) {
if (!validForStatistics(build))
return;
@@ -201,6 +202,7 @@ public class RunHistSync {
return saveInvocationsMap(buildsSaveThisRun, testsSaveThisRun);
}
+ @Deprecated
@AutoProfiling
@Nonnull protected String saveInvocationsMap(
Map<RunHistKey, List<Invocation>> buildsSaveThisRun,
@@ -259,6 +261,7 @@ public class RunHistSync {
return res;
}
+ @Deprecated
private void saveInvocationList(Set<Integer> confirmedNewBuild,
Set<Integer> confirmedDuplicate,
AtomicInteger invocations,
@@ -297,6 +300,7 @@ public class RunHistSync {
duplicateOrExpired.addAndGet(invocationList.size() - cntAdded);
}
+ @Deprecated
public void invokeLaterFindMissingHistory(String srvName) {
scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName),
() -> findMissingHistFromBuildRef(srvName), 12, TimeUnit.HOURS);
@@ -310,6 +314,7 @@ public class RunHistSync {
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Find Missing Build History", nameExtArgsIndexes = {0})
@AutoProfiling
+ @Deprecated
protected String findMissingHistFromBuildRef(String srvId) {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
@@ -342,6 +347,7 @@ public class RunHistSync {
* @param srvNme Server name;
* @param load Build IDs to be loaded into history cache later.
*/
+ @Deprecated
private void scheduleHistLoad(String srvNme, List<Integer> load) {
load.forEach(id -> {
FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(ITeamcityIgnited.serverIdToInt(srvNme), id);