You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/11/21 17:56:56 UTC
[ignite-teamcity-bot] branch master updated: Removing not used
caches and code after compacted storage refactoring
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 22b16a2 Removing not used caches and code after compacted storage refactoring
22b16a2 is described below
commit 22b16a2bc5684c3c30aa60cce2b40f4c63d2d609
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Wed Nov 21 20:56:53 2018 +0300
Removing not used caches and code after compacted storage refactoring
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 11 +-
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 196 ---------------------
.../apache/ignite/ci/IgniteTeamcityConnection.java | 21 +--
.../java/org/apache/ignite/ci/db/DbMigrations.java | 78 ++------
4 files changed, 23 insertions(+), 283 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index f22a6bf..1820d14 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -75,20 +75,13 @@ public interface ITeamcity extends ITeamcityConn {
}
@NotNull static String buildHref(int id) {
- return "app/rest/latest/builds/id:" + Integer.toString(id);
+ return "app/rest/latest/builds/id:" + id;
}
-
- @Deprecated
- ProblemOccurrences getProblems(BuildRef build);
-
@Deprecated
public TestOccurrences getTests(String fullUrl);
@Deprecated
- TestOccurrences getFailedTests(String href, int count, String normalizedBranch);
-
- @Deprecated
- CompletableFuture<TestOccurrenceFull> getTestFull(String href);
+ TestOccurrences getFailedTests(String href, int cnt, String normalizedBranch);
@Deprecated
CompletableFuture<TestRef> getTestRef(FullQueryParams key);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 1fe97f5..c8b73d1 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -92,7 +92,6 @@ import static org.apache.ignite.ci.tcbot.chain.BuildChainProcessor.normalizeBran
*/
@Deprecated
public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITeamcity, ITcAnalytics {
-
//V2 caches, 32 parts (V1 caches were 1024 parts)
@Deprecated
private static final String TESTS_OCCURRENCES = "testOccurrences";
@@ -100,9 +99,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
private static final String CALCULATED_STATISTIC = "calculatedStatistic";
private static final String LOG_CHECK_RESULT = "logCheckResult";
private static final String ISSUES_USAGES_LIST = "issuesUsagesList";
- @Deprecated
- private static final String TEST_FULL = "testFull";
- private static final String BUILD_PROBLEMS = "buildProblems";
public static final String BOT_DETECTED_ISSUES = "botDetectedIssues";
public static final String TEST_REFS = "testRefs";
@@ -112,9 +108,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
private static final String BUILDS_FAILURE_RUN_STAT = "buildsFailureRunStat";
public static final String BUILDS = "builds";
- /** Number of builds to re-query from TC to be sure some builds in the middle are not lost. */
- private static final int MAX_BUILDS_IN_PAST_TO_RELOAD = 5;
-
@Inject
private Ignite ignite;
@@ -128,10 +121,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
private ITeamcity teamcity;
private String serverId;
- /**
- * cached loads of full test occurrence.
- */
- private ConcurrentMap<String, CompletableFuture<TestOccurrenceFull>> testOccFullFutures = new ConcurrentHashMap<>();
/** Cached loads of test refs.*/
private ConcurrentMap<String, CompletableFuture<TestRef>> testRefsFutures = new ConcurrentHashMap<>();
@@ -142,16 +131,10 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
/** cached loads of queued builds for branch. */
private ConcurrentMap<String, CompletableFuture<List<BuildRef>>> queuedBuildsFuts = new ConcurrentHashMap<>();
- /** */
- private ConcurrentMap<SuiteInBranch, Long> lastQueuedHistory = new ConcurrentHashMap<>();
-
//todo: not good code to keep it static
@Deprecated
private static long lastTriggerMs = System.currentTimeMillis();
- private static final boolean noLocks = true;
-
-
@Override public void init(ITeamcity conn) {
this.teamcity = conn;
this.serverId = conn.serverId();
@@ -163,8 +146,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
this::migrateOccurrencesToLatest,
buildsCache(), this::addBuildOccurrenceToFailuresStat,
buildsFailureRunStatCache(), testRunStatCache(),
- testFullCache(),
- buildProblemsCache(),
visasHistStorage.visas());
}
@@ -217,13 +198,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return getOrCreateCacheV2(ignCacheNme(TESTS_OCCURRENCES));
}
- /**
- * @return {@link TestOccurrenceFull} instances cache, 32 parts.
- */
- @Deprecated
- private IgniteCache<String, TestOccurrenceFull> testFullCache() {
- return getOrCreateCacheV2(ignCacheNme(TEST_FULL));
- }
/**
* @return {@link Configurations} instances cache, 32 parts.
@@ -239,13 +213,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return getOrCreateCacheV2(ignCacheNme(TEST_REFS));
}
- /**
- * @return Build {@link ProblemOccurrences} instances cache, 32 parts.
- */
- private IgniteCache<String, ProblemOccurrences> buildProblemsCache() {
- return getOrCreateCacheV2(ignCacheNme(BUILD_PROBLEMS));
- }
-
/** {@inheritDoc} */
@AutoProfiling
@Override public CompletableFuture<List<BuildType>> getProjectSuites(String projectId) {
@@ -284,132 +251,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
}
@Deprecated
- protected List<BuildRef> loadBuildHistory(IgniteCache<SuiteInBranch, Expirable<List<BuildRef>>> cache,
- int seconds,
- SuiteInBranch key,
- BiFunction<SuiteInBranch, Integer, List<BuildRef>> realLoad) {
- @Nullable Expirable<List<BuildRef>> persistedBuilds = readBuildHistEntry(cache, key);
-
- if (persistedBuilds != null
- && (isHistoryAgeLessThanSecs(key, seconds, persistedBuilds))) {
- ObjectInterner.internFields(persistedBuilds);
-
- return persistedBuilds.getData();
- }
-
- Lock lock = lockBuildHistEntry(cache, key);
-
- try {
- if (!noLocks) {
- if (persistedBuilds != null
- && (isHistoryAgeLessThanSecs(key, seconds, persistedBuilds))) {
- ObjectInterner.internFields(persistedBuilds);
-
- return persistedBuilds.getData();
- }
- }
-
- Integer sinceBuildId;
- if (persistedBuilds != null) {
- List<BuildRef> prevData = persistedBuilds.getData();
- if (prevData.size() >= MAX_BUILDS_IN_PAST_TO_RELOAD) {
- BuildRef buildRef = prevData.get(prevData.size() - MAX_BUILDS_IN_PAST_TO_RELOAD);
-
- sinceBuildId = buildRef.getId();
- } else
- sinceBuildId = null;
- } else
- sinceBuildId = null;
-
- List<BuildRef> dataFromRest;
- try {
- dataFromRest = realLoad.apply(key, sinceBuildId);
- } catch (Exception e) {
- Throwable rootCause = Throwables.getRootCause(e);
- if (rootCause instanceof FileNotFoundException) {
- System.err.println("Build history not found for build : " + key + ": " + rootCause.getMessage());
- dataFromRest = Collections.emptyList();
- } else if (rootCause instanceof BadRequestException) {
- //probably referenced build not found
- if (sinceBuildId != null)
- dataFromRest = realLoad.apply(key, null);
- else
- throw e;
- } else
- throw e;
- }
- final List<BuildRef> persistedList = persistedBuilds != null ? persistedBuilds.getData() : null;
- final List<BuildRef> buildRefs = mergeHistoryMaps(persistedList, dataFromRest);
-
- final Expirable<List<BuildRef>> newVal = new Expirable<>(0L, buildRefs);
-
- if (persistedList != null && persistedList.equals(buildRefs)) {
- lastQueuedHistory.put(key, System.currentTimeMillis());
-
- return buildRefs;
- }
- saveBuildHistoryEntry(cache, key, newVal);
-
- lastQueuedHistory.put(key, System.currentTimeMillis());
-
- return buildRefs;
- }
- finally {
- if (!noLocks)
- lock.unlock();
- }
- }
-
- private boolean isHistoryAgeLessThanSecs(SuiteInBranch key, int seconds, Expirable<List<BuildRef>> persistedBuilds) {
- Long loaded = lastQueuedHistory.get(key);
- if (loaded == null)
- return false;
-
- long ageMs = System.currentTimeMillis() - loaded;
-
- return ageMs < TimeUnit.SECONDS.toMillis(seconds);
- }
-
- @AutoProfiling
- @SuppressWarnings("WeakerAccess")
- protected <K> void saveBuildHistoryEntry(IgniteCache<K, Expirable<List<BuildRef>>> cache, K key, Expirable<List<BuildRef>> newVal) {
- cache.put(key, newVal);
- }
-
-
- @AutoProfiling
- @SuppressWarnings("WeakerAccess")
- protected <K> Expirable<List<BuildRef>> readBuildHistEntry(IgniteCache<K, Expirable<List<BuildRef>>> cache, K key) {
- return cache.get(key);
- }
-
- @AutoProfiling
- @SuppressWarnings("WeakerAccess")
- protected <K> Lock lockBuildHistEntry(IgniteCache<K, Expirable<List<BuildRef>>> cache, K key) {
- if (noLocks)
- return null;
-
- Lock lock = cache.lock(key);
-
- lock.lock();
-
- return lock;
- }
-
- @NotNull
- @AutoProfiling
- @SuppressWarnings("WeakerAccess")
- protected List<BuildRef> mergeHistoryMaps(@Nullable List<BuildRef> persistedVal, List<BuildRef> mostActualVal) {
- final SortedMap<Integer, BuildRef> merge = new TreeMap<>();
-
- if (persistedVal != null)
- persistedVal.forEach(b -> merge.put(b.getId(), b));
-
- mostActualVal.forEach(b -> merge.put(b.getId(), b)); //to overwrite data from persistence by values from REST
-
- return new ArrayList<>(merge.values());
- }
-
private <K, V> CompletableFuture<V> loadAsyncIfAbsentOrExpired(ConcurrentMap<K, Expirable<V>> cache,
K key,
ConcurrentMap<K, CompletableFuture<V>> cachedComputations,
@@ -568,21 +409,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return teamcity.host();
}
- /** {@inheritDoc}*/
- @AutoProfiling
- @Override public ProblemOccurrences getProblems(BuildRef buildRef) {
- return loadIfAbsent(
- buildProblemsCache(),
- "/app/rest/latest/problemOccurrences?locator=build:(id:" + buildRef.getId() + ")",
- k -> {
- ProblemOccurrences problems = teamcity.getProblems(buildRef);
-
- registerCriticalBuildProblemInStat(buildRef, problems);
-
- return problems;
- });
- }
-
private void registerCriticalBuildProblemInStat(BuildRef build, ProblemOccurrences problems) {
boolean criticalFail = problems.getProblemsNonNull().stream().anyMatch(occurrence ->
occurrence.isExecutionTimeout() || occurrence.isJvmCrash() || occurrence.isFailureOnMetric()
@@ -649,28 +475,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Override public CompletableFuture<TestOccurrenceFull> getTestFull(String href) {
- return CacheUpdateUtil.loadAsyncIfAbsent(
- testFullCache(),
- href,
- testOccFullFutures,
- href1 -> {
- try {
- return teamcity.getTestFull(href1);
- }
- catch (Exception e) {
- if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
- System.err.println("TestOccurrenceFull not found for href : " + href);
-
- return CompletableFuture.completedFuture(new TestOccurrenceFull());
- }
- throw e;
- }
- });
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
@Override public CompletableFuture<TestRef> getTestRef(FullQueryParams key) {
return CacheUpdateUtil.loadAsyncIfAbsent(
testRefsCache(),
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 5412b99..1a3e8d6 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -324,19 +324,21 @@ public class IgniteTeamcityConnection implements ITeamcity {
return getProblems(build.getId());
}
- @Override
+ /** {@inheritDoc} */
@AutoProfiling
- public ProblemOccurrences getProblems(int buildId) {
+ @Override public ProblemOccurrences getProblems(int buildId) {
return getJaxbUsingHref("app/rest/latest/problemOccurrences" +
"?locator=build:(id:" + buildId + ")" +
"&fields=problemOccurrence(id,type,identity,href,details,build(id))", ProblemOccurrences.class);
}
+ /** {@inheritDoc} */
@AutoProfiling
@Override public Statistics getStatistics(int buildId) {
return getJaxbUsingHref("app/rest/latest/builds/id:" + buildId + "/statistics", Statistics.class);
}
+ /** {@inheritDoc} */
@AutoProfiling
@Override public ChangesList getChangesList(int buildId) {
String href = "app/rest/latest/changes" +
@@ -462,15 +464,6 @@ public class IgniteTeamcityConnection implements ITeamcity {
return getJaxbUsingHref(href, Build.class);
}
- @AutoProfiling
- @Override public ProblemOccurrences getProblems(BuildRef buildRef) {
- ProblemOccurrences coll = getJaxbUsingHref("app/rest/latest/problemOccurrences?" +
- "locator=build:(id:" + buildRef.getId() + ")", ProblemOccurrences.class);
-
- coll.getProblemsNonNull().forEach(p -> p.buildRef = buildRef);
-
- return coll;
- }
/** {@inheritDoc} */
@AutoProfiling
@@ -480,12 +473,6 @@ public class IgniteTeamcityConnection implements ITeamcity {
/** {@inheritDoc} */
@AutoProfiling
- @Override public CompletableFuture<TestOccurrenceFull> getTestFull(String href) {
- return supplyAsync(() -> getJaxbUsingHref(href, TestOccurrenceFull.class), executor);
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
@Override public TestOccurrences getFailedTests(String href, int count, String normalizedBranch) {
return getTests(href + ",muted:false,status:FAILURE,count:" + count + "&fields=testOccurrence(id,name)");
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index a1a92ad..0486f2b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -19,7 +19,6 @@ package org.apache.ignite.ci.db;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -42,9 +41,7 @@ import org.apache.ignite.ci.issue.IssueKey;
import org.apache.ignite.ci.issue.IssuesStorage;
import org.apache.ignite.ci.observer.CompactBuildsInfo;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
import org.apache.ignite.ci.web.model.CompactContributionKey;
import org.apache.ignite.ci.web.model.CompactVisa;
@@ -106,17 +103,19 @@ public class DbMigrations {
private static final String BUILD_HIST_FINISHED_OR_FAILED = "buildHistFinishedOrFailed";
@Deprecated
- public static final String CURRENT_PR_FAILURES = "currentPrFailures";
-
- @Deprecated
public static final String TEAMCITY_BUILD_CACHE_NAME_OLD = "teamcityBuild";
/** */
@Deprecated
public static final String COMPACT_VISAS_HISTORY_CACHE_NAME = "compactVisasHistoryCache";
- private static final String CHANGE_INFO_FULL = "changeInfoFull";
- private static final String CHANGES_LIST = "changesList";
+ interface Old {
+ String TEST_FULL = "testFull";
+ String BUILD_PROBLEMS = "buildProblems";
+ String CHANGES_LIST = "changesList";
+ String CHANGE_INFO_FULL = "changeInfoFull";
+ String CURRENT_PR_FAILURES = "currentPrFailures";
+ }
public static final int SUITES_CNT = 100;
@@ -140,8 +139,6 @@ public class DbMigrations {
Cache<String, Build> buildCache, Consumer<Build> saveBuildToStat,
IgniteCache<SuiteInBranch, RunStat> suiteHistCache,
IgniteCache<TestInBranch, RunStat> testHistCache,
- Cache<String, TestOccurrenceFull> testFullCache,
- Cache<String, ProblemOccurrences> problemsCache,
Cache<CompactContributionKey, List<CompactVisaRequest>> visasCache) {
doneMigrations = doneMigrationsCache();
@@ -360,58 +357,14 @@ public class DbMigrations {
applyRemoveCache(GetTrackedBranchTestResults.TEST_FAILURES_SUMMARY_CACHE_NAME);
applyRemoveCache(GetBuildTestFailures.TEST_FAILURES_SUMMARY_CACHE_NAME);
- applyRemoveCache(DbMigrations.CURRENT_PR_FAILURES);
-
- applyMigration(TEST_OCCURRENCE_FULL + "-to-" + testFullCache.getName() + "V2", () -> {
- String cacheNme = ignCacheNme(TEST_OCCURRENCE_FULL);
- IgniteCache<String, TestOccurrenceFull> oldTestCache = ignite.cache(cacheNme);
-
- if (oldTestCache == null) {
- System.err.println("cache not found");
-
- return;
- }
+ applyRemoveCache(Old.CURRENT_PR_FAILURES);
+ applyDestroyIgnCacheMigration(TEST_OCCURRENCE_FULL);
+ /*
int size = oldTestCache.size();
if (size > 0) {
ignite.cluster().disableWal(testFullCache.getName());
-
try {
- int i = 0;
-
- Map<String, TestOccurrenceFull> batch = new HashMap<>();
- int maxFoundBuildId = 0;
-
- IgniteDataStreamer<String, TestOccurrenceFull> streamer = ignite.dataStreamer(testFullCache.getName());
-
- for (Cache.Entry<String, TestOccurrenceFull> entry : oldTestCache) {
- String key = entry.getKey();
-
- Integer buildId = RunStat.extractIdPrefixed(key, ",build:(id:", ")");
-
- if (buildId != null) {
- if (buildId > maxFoundBuildId)
- maxFoundBuildId = buildId;
-
- if (buildId < maxFoundBuildId - (RunStat.MAX_LATEST_RUNS * SUITES_CNT * 15))
- logger.info(serverId + " - Skipping entry " + i + " from " + size + ": " + key);
- else
- batch.put(entry.getKey(), entry.getValue());
- }
-
-
- i++;
-
- if (batch.size() >= 300)
- saveOneBatch(cacheNme, size, i, batch, streamer);
- }
-
- if (!batch.isEmpty())
- saveOneBatch(cacheNme, size, i, batch, streamer);
-
- streamer.flush();
-
- System.err.println("Removing data from old cache " + oldTestCache.getName());
oldTestCache.destroy();
}
@@ -419,8 +372,8 @@ public class DbMigrations {
ignite.cluster().enableWal(testFullCache.getName());
}
}
- });
- applyV1toV2Migration(PROBLEMS, problemsCache);
+ });*/
+ applyDestroyIgnCacheMigration(PROBLEMS);
applyDestroyIgnCacheMigration(FINISHED_BUILDS_INCLUDE_FAILED);
@@ -485,14 +438,17 @@ public class DbMigrations {
applyDestroyCacheMigration(BUILD_CONDITIONS_CACHE_NAME, BUILD_CONDITIONS_CACHE_NAME);
applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD, TEAMCITY_BUILD_CACHE_NAME_OLD);
- applyDestroyIgnCacheMigration(CHANGE_INFO_FULL);
- applyDestroyIgnCacheMigration(CHANGES_LIST);
+ applyDestroyIgnCacheMigration(Old.CHANGE_INFO_FULL);
+ applyDestroyIgnCacheMigration(Old.CHANGES_LIST);
applyDestroyIgnCacheMigration(FINISHED_BUILDS);
applyDestroyIgnCacheMigration(BUILD_HIST_FINISHED);
applyDestroyIgnCacheMigration(BUILD_HIST_FINISHED_OR_FAILED);
applyDestroyCacheMigration(COMPACT_VISAS_HISTORY_CACHE_NAME, COMPACT_VISAS_HISTORY_CACHE_NAME);
+
+ applyDestroyIgnCacheMigration(Old.BUILD_PROBLEMS);
+ applyDestroyIgnCacheMigration(Old.TEST_FULL);
}
private void applyDestroyIgnCacheMigration(String cacheName) {