You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/26 16:59:25 UTC
[ignite-teamcity-bot] branch ignite-9848-load-all-builds updated:
Guava based caching for hot methods results
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-load-all-builds by this push:
new 931e8cd Guava based caching for hot methods results
931e8cd is described below
commit 931e8cd171ac36d1ca7bb4ec7f5336c2ec7c9e79
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Oct 26 19:41:02 2018 +0300
Guava based caching for hot methods results
---
.../org/apache/ignite/ci/di/IgniteTcBotModule.java | 2 +
.../org/apache/ignite/ci/di/cache/GuavaCached.java | 48 ++++++++++
.../ignite/ci/di/cache/GuavaCachedInterceptor.java | 103 +++++++++++++++++++++
.../ignite/ci/di/cache/GuavaCachedModule.java | 37 ++++++++
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 2 +
.../ci/teamcity/ignited/IgniteStringCompactor.java | 11 ++-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 2 +
.../org/apache/ignite/ci/di/cache/CachingTest.java | 96 +++++++++++++++++++
8 files changed, 300 insertions(+), 1 deletion(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java
index 7f75f78..b2aa2d8 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java
@@ -30,6 +30,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.ci.ITcHelper;
import org.apache.ignite.ci.TcHelper;
import org.apache.ignite.ci.db.Ignite1Init;
+import org.apache.ignite.ci.di.cache.GuavaCachedModule;
import org.apache.ignite.ci.di.scheduler.SchedulerModule;
import org.apache.ignite.ci.github.ignited.GitHubIgnitedModule;
import org.apache.ignite.ci.issue.IssueDetector;
@@ -52,6 +53,7 @@ public class IgniteTcBotModule extends AbstractModule {
/** {@inheritDoc} */
@Override protected void configure() {
+ install(new GuavaCachedModule());
configProfiling();
configTaskMonitor();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCached.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCached.java
new file mode 100644
index 0000000..628d3fe
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCached.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.di.cache;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD)
+public @interface GuavaCached {
+ /**
+ * Specifies the maximum number of entries the cache may contain.
+ */
+ long maximumSize() default -1L;
+
+ /**
+ *
+ */
+ boolean softValues() default false;
+
+ /**
+ * Cache null as valid return value. For caching Ignite entries it is always require to set this parameter.
+ */
+ boolean cacheNullRval() default true;
+
+
+ /**
+ * Cache negative number values as valid return value. For caching Ignite entries it is always require to set this parameter.
+ */
+ boolean cacheNegativeNumbersRval() default true;
+
+ long expireAfterAccessSecs() default -1;
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCachedInterceptor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCachedInterceptor.java
new file mode 100644
index 0000000..adca7fc
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCachedInterceptor.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.di.cache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+import org.jetbrains.annotations.NotNull;
+
+public class GuavaCachedInterceptor implements MethodInterceptor {
+ private final ConcurrentMap<String, Cache<List, Optional>> caches = new ConcurrentHashMap<>();
+
+ @Override public Object invoke(MethodInvocation invocation) throws Throwable {
+ final Method invocationMtd = invocation.getMethod();
+ GuavaCached annotation = invocationMtd.getAnnotation(GuavaCached.class);
+
+ Cache<List, Optional> cache = caches.computeIfAbsent(cacheId(invocation), k -> {
+ CacheBuilder builder = CacheBuilder.newBuilder();
+
+ if(annotation.softValues())
+ builder = builder.softValues();
+
+ if(annotation.maximumSize()>0)
+ builder = builder.maximumSize(annotation.maximumSize());
+
+ if(annotation.expireAfterAccessSecs()>0)
+ builder.expireAfterAccess(annotation.expireAfterAccessSecs(), TimeUnit.SECONDS);
+
+ return builder.build();
+ });
+
+ List<Object> cacheKey = Arrays.asList(invocation.getArguments());
+
+ Optional optional = cache.get(cacheKey,
+ new Callable<Optional>() {
+ @Override public Optional call() throws Exception {
+ Object res;
+ try {
+ res = invocation.proceed();
+ }
+ catch (Throwable throwable) {
+ Throwables.propagateIfPossible(throwable, Exception.class);
+
+ throw new RuntimeException(throwable);
+ }
+ return Optional.ofNullable(res);
+ }
+ });
+
+ if (!annotation.cacheNullRval()) {
+ if (!optional.isPresent())
+ cache.invalidate(cacheKey);
+ }
+
+ if (!annotation.cacheNegativeNumbersRval()) {
+ if (optional.isPresent()) {
+ Object o = optional.get();
+ Preconditions.checkState(o instanceof Number, "Invalid return value of method: " + cacheKey);
+
+ Number num = (Number)o;
+ if (num.longValue() < 0)
+ cache.invalidate(cacheKey);
+ }
+ }
+
+ return optional.orElse(null);
+ }
+
+ @NotNull
+ private String cacheId(MethodInvocation invocation) {
+ final Method invocationMtd = invocation.getMethod();
+ final String cls = invocationMtd.getDeclaringClass().getName();
+ final String mtd = invocationMtd.getName();
+
+ return cls + "." + mtd;
+ }
+
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCachedModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCachedModule.java
new file mode 100644
index 0000000..3a3bfc0
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/cache/GuavaCachedModule.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.di.cache;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.matcher.Matchers;
+
+/**
+ *
+ */
+public class GuavaCachedModule extends AbstractModule {
+ /** {@inheritDoc} */
+ @Override protected void configure() {
+ GuavaCachedInterceptor cachedInterceptor = new GuavaCachedInterceptor();
+
+ bindInterceptor(Matchers.any(),
+ Matchers.annotatedWith(GuavaCached.class),
+ cachedInterceptor);
+
+ bind(GuavaCachedInterceptor.class).toInstance(cachedInterceptor);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index b61f25a..4980cc9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -39,6 +39,7 @@ import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.di.cache.GuavaCached;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.GridIntList;
@@ -177,6 +178,7 @@ public class BuildRefDao {
}
@AutoProfiling
+ @GuavaCached(softValues = true, maximumSize = 1000, expireAfterAccessSecs = 30)
public List<BuildRefCompacted> getBuildsForBranch(int srvId, String branchName) {
Integer branchNameId = compactor.getStringIdIfPresent(branchName);
if (branchNameId == null)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IgniteStringCompactor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IgniteStringCompactor.java
index 07d0036..f6c8545 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IgniteStringCompactor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IgniteStringCompactor.java
@@ -33,12 +33,18 @@ import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.di.cache.GuavaCached;
import org.apache.ignite.ci.util.ExceptionUtil;
import org.apache.ignite.ci.util.ObjectInterner;
import org.apache.ignite.configuration.CacheConfiguration;
import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class IgniteStringCompactor implements IStringCompactor {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(IgniteStringCompactor.class);
+
private final AtomicBoolean initGuard = new AtomicBoolean();
private final CountDownLatch initLatch = new CountDownLatch(1);
@@ -108,6 +114,7 @@ public class IgniteStringCompactor implements IStringCompactor {
/** {@inheritDoc} */
@AutoProfiling
+ @GuavaCached(cacheNegativeNumbersRval = false)
@Override public int getStringId(String val) {
if (val == null)
return -1;
@@ -116,7 +123,7 @@ public class IgniteStringCompactor implements IStringCompactor {
CompactorEntity entity = stringsCache.get(val);
if (entity != null)
- return entity == null ? -1 : entity.id;
+ return entity.id;
int codeCandidate = (int)seq.incrementAndGet();
@@ -127,6 +134,7 @@ public class IgniteStringCompactor implements IStringCompactor {
/** {@inheritDoc} */
@AutoProfiling
+ @GuavaCached(cacheNullRval = false)
@Override public String getStringFromId(int id) {
if (id < 0)
return null;
@@ -152,6 +160,7 @@ public class IgniteStringCompactor implements IStringCompactor {
}
/** {@inheritDoc} */
+ @GuavaCached(cacheNullRval = false, cacheNegativeNumbersRval = false)
@Override public Integer getStringIdIfPresent(String val) {
if (val == null)
return -1;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index f3e2484..0a42012 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -148,6 +148,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Override public FatBuildCompacted getFatBuild(int buildId) {
FatBuildCompacted existingBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, buildId);
+
+ //todo additionally check queued and running builds, refesh builds if they are queued.
if (existingBuild != null && !existingBuild.isOutdatedEntityVersion())
return existingBuild;
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/di/cache/CachingTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/di/cache/CachingTest.java
new file mode 100644
index 0000000..de38491
--- /dev/null
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/di/cache/CachingTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.di.cache;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class CachingTest {
+ @Test
+ public void testCacheWorks() {
+ Injector injector = Guice.createInjector(new GuavaCachedModule());
+
+ SomeWorker instance = injector.getInstance(SomeWorker.class);
+
+ for (int i = 0; i < 200; i++)
+ instance.doSmt();
+
+ assertEquals(1, instance.doSmtMtdCalls.get());
+
+ {
+ for (int i = 0; i < 200; i++)
+ instance.toString(i);
+
+ assertEquals(200, instance.toStringMtdCalls.get());
+
+ for (int i = 0; i < 200; i++)
+ instance.toString(i);
+
+ assertEquals(200, instance.toStringMtdCalls.get());
+
+ for (int i = 0; i < 200; i++)
+ instance.toString(-i - 1);
+
+ assertEquals(400, instance.toStringMtdCalls.get());
+ }
+
+
+ for (int i = 0; i < 100; i++)
+ instance.parseInt(Integer.toString(i%10));
+
+ assertEquals(10, instance.parseIntMtdCalls.get());
+
+ for (int i = 0; i < 100; i++)
+ instance.parseInt(Integer.toString(-(i % 10) - 1));
+
+ assertEquals(110, instance.parseIntMtdCalls.get());
+ }
+
+ public static class SomeWorker {
+ AtomicInteger doSmtMtdCalls = new AtomicInteger();
+ AtomicInteger toStringMtdCalls = new AtomicInteger();
+ AtomicInteger parseIntMtdCalls = new AtomicInteger();
+
+ @GuavaCached
+ public String doSmt() {
+ doSmtMtdCalls.incrementAndGet();
+
+ return "";
+ }
+
+ @GuavaCached(cacheNullRval = false)
+ public String toString(int i) {
+ toStringMtdCalls.incrementAndGet();
+ if (i < 0)
+ return null;
+ return Integer.toString(i);
+ }
+
+
+ @GuavaCached(cacheNegativeNumbersRval = false)
+ public int parseInt(String val) {
+ parseIntMtdCalls.incrementAndGet();
+
+ return Integer.parseInt(val);
+ }
+ }
+
+}