You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by tv...@apache.org on 2021/12/23 10:03:48 UTC
[commons-jcs] 03/04: Implement failover test
This is an automated email from the ASF dual-hosted git repository.
tv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-jcs.git
commit 1f5215ed32979c362e21f078cb14bc84554a7951
Author: Thomas Vandahl <tv...@apache.org>
AuthorDate: Thu Dec 23 10:55:45 2021 +0100
Implement failover test
---
.../auxiliary/remote/RemoteCacheNoWaitFacade.java | 70 ++++++-----
.../remote/RemoteCacheNoWaitFacadeUnitTest.java | 35 ++++++
.../auxiliary/remote/TestRemoteCacheFactory.java | 133 +++++++++++++++++++++
3 files changed, 207 insertions(+), 31 deletions(-)
diff --git a/commons-jcs-core/src/main/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacade.java b/commons-jcs-core/src/main/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacade.java
index ca02fd1..3715d61 100644
--- a/commons-jcs-core/src/main/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacade.java
+++ b/commons-jcs-core/src/main/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacade.java
@@ -51,6 +51,9 @@ public class RemoteCacheNoWaitFacade<K, V>
/** Provide factory instance to RemoteCacheFailoverRunner */
private final RemoteCacheFactory cacheFactory;
+ /** Attempt to restore primary connection (switched off for testing) */
+ protected boolean attemptRestorePrimary = true;
+
/** Time in ms to sleep between failover attempts */
private static final long idlePeriod = 20000L;
@@ -127,6 +130,23 @@ public class RemoteCacheNoWaitFacade<K, V>
protected void connectAndRestore()
{
final IRemoteCacheAttributes rca0 = getAuxiliaryCacheAttributes();
+ // Each RemoteCacheManager corresponds to one remote connection.
+ final List<RemoteLocation> failovers = rca0.getFailovers();
+ // we should probably check to see if there are any failovers,
+ // even though the caller should have already.
+
+ if ( failovers == null )
+ {
+ log.warn( "Remote is misconfigured, failovers was null." );
+ return;
+ }
+ if ( failovers.size() == 1 )
+ {
+ // if there is only the primary, return out of this
+ log.info( "No failovers defined, exiting failover runner." );
+ return;
+ }
+
final AtomicBoolean allright = new AtomicBoolean(false);
do
@@ -137,27 +157,9 @@ public class RemoteCacheNoWaitFacade<K, V>
if ( !allright.get() )
{
// Monitor each RemoteCacheManager instance one after the other.
- // Each RemoteCacheManager corresponds to one remote connection.
- final List<RemoteLocation> failovers = rca0.getFailovers();
- // we should probably check to see if there are any failovers,
- // even though the caller should have already.
-
- if ( failovers == null )
- {
- log.warn( "Remote is misconfigured, failovers was null." );
- return;
- }
- if ( failovers.size() == 1 )
- {
- // if there is only the primary, return out of this
- log.info( "No failovers defined, exiting failover runner." );
- return;
- }
-
final int fidx = rca0.getFailoverIndex();
- log.debug( "fidx = {0} failovers.size = {1}", () -> fidx, failovers::size);
+ log.debug( "fidx = {0} failovers.size = {1}", rca0::getFailoverIndex, failovers::size);
- // shouldn't we see if the primary is backup?
// If we don't check the primary, if it gets connected in the
// background,
// we will disconnect it only to put it right back
@@ -167,8 +169,9 @@ public class RemoteCacheNoWaitFacade<K, V>
// try them one at a time until successful
while (i.hasNext() && !allright.get())
{
+ final int failoverIndex = i.nextIndex();
final RemoteLocation server = i.next();
- log.debug( "Trying server [{0}] at failover index i = {1}", server, i );
+ log.debug("Trying server [{0}] at failover index i = {1}", server, failoverIndex);
final RemoteCacheAttributes rca = (RemoteCacheAttributes) rca0.clone();
rca.setRemoteLocation(server);
@@ -186,16 +189,16 @@ public class RemoteCacheNoWaitFacade<K, V>
// may need to do this more gracefully
log.debug( "resetting no wait" );
restorePrimaryServer((RemoteCacheNoWait<K, V>) ic);
- rca0.setFailoverIndex( i.nextIndex() );
+ rca0.setFailoverIndex(failoverIndex);
- log.debug( "setting ALLRIGHT to true" );
- if ( i.hasPrevious() )
+ log.debug("setting ALLRIGHT to true");
+ if (i.hasPrevious())
{
- log.debug( "Moving to Primary Recovery Mode, failover index = {0}", i );
+ log.debug("Moving to Primary Recovery Mode, failover index = {0}", failoverIndex);
}
else
{
- log.debug( "No need to connect to failover, the primary server is back up." );
+ log.debug("No need to connect to failover, the primary server is back up.");
}
allright.set(true);
@@ -216,19 +219,24 @@ public class RemoteCacheNoWaitFacade<K, V>
+ "primary server.", rca0::getFailoverIndex);
}
+ // Exit loop if in test mode
+ if (allright.get() && !attemptRestorePrimary)
+ {
+ break;
+ }
+
boolean primaryRestoredSuccessfully = false;
// if we are not connected to the primary, try.
- if ( rca0.getFailoverIndex() > 0 )
+ if (rca0.getFailoverIndex() > 0)
{
primaryRestoredSuccessfully = restorePrimary();
log.debug( "Primary recovery success state = {0}",
primaryRestoredSuccessfully );
}
- if ( !primaryRestoredSuccessfully )
+ if (!primaryRestoredSuccessfully)
{
- // Time driven mode: sleep between each round of recovery
- // attempt.
+ // Time driven mode: sleep between each round of recovery attempt.
try
{
log.warn( "Failed to reconnect to primary server. "
@@ -244,12 +252,12 @@ public class RemoteCacheNoWaitFacade<K, V>
// try to bring the listener back to the primary
}
- while ( rca0.getFailoverIndex() > 0 || !allright.get() );
+ while (rca0.getFailoverIndex() > 0 || !allright.get());
// continue if the primary is not restored or if things are not allright.
if ( log.isInfoEnabled() )
{
- final int failoverIndex = getAuxiliaryCacheAttributes().getFailoverIndex();
+ final int failoverIndex = rca0.getFailoverIndex();
log.info( "Exiting failover runner. Failover index = {0}", failoverIndex);
if ( failoverIndex <= 0 )
diff --git a/commons-jcs-core/src/test/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacadeUnitTest.java b/commons-jcs-core/src/test/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacadeUnitTest.java
index 4f7e4fc..d4ee3b9 100644
--- a/commons-jcs-core/src/test/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacadeUnitTest.java
+++ b/commons-jcs-core/src/test/java/org/apache/commons/jcs3/auxiliary/remote/RemoteCacheNoWaitFacadeUnitTest.java
@@ -3,6 +3,7 @@ package org.apache.commons.jcs3.auxiliary.remote;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.jcs3.auxiliary.AuxiliaryCache;
import org.apache.commons.jcs3.auxiliary.remote.behavior.IRemoteCacheAttributes;
/*
@@ -53,4 +54,38 @@ public class RemoteCacheNoWaitFacadeUnitTest
assertTrue( "Should be in the list.", facade.noWaits.contains( noWait ) );
assertSame( "Should have same facade.", facade, ((RemoteCache<String, String>)facade.noWaits.get(0).getRemoteCache()).getFacade() );
}
+
+ /**
+ * Verify that failover works
+ */
+ public void testFailover()
+ {
+ // SETUP
+ final IRemoteCacheAttributes cattr = new RemoteCacheAttributes();
+ cattr.setCacheName("testCache1");
+ cattr.setFailoverServers("localhost:1101,localhost:1102");
+ cattr.setReceive(false);
+
+ final TestRemoteCacheFactory factory = new TestRemoteCacheFactory();
+ factory.initialize();
+
+ final AuxiliaryCache<String, String> cache = factory.createCache(cattr, null, null, null);
+ final RemoteCacheNoWaitFacade<String, String> facade =
+ (RemoteCacheNoWaitFacade<String, String>) cache;
+ assertEquals("Should have two failovers.", 2, cattr.getFailovers().size());
+ assertEquals("Should have two managers.", 2, factory.managers.size());
+ assertEquals("Should have primary server.", 0, cattr.getFailoverIndex());
+ RemoteCacheNoWait<String, String> primary = facade.getPrimaryServer();
+
+ // Make primary unusable
+ facade.getPrimaryServer().getCacheEventQueue().destroy();
+ facade.attemptRestorePrimary = false;
+ facade.connectAndRestore();
+
+ // VERIFY
+ assertEquals("Should have two failovers.", 2, cattr.getFailovers().size());
+ assertEquals("Should have two managers.", 2, factory.managers.size());
+ assertEquals("Should have switched to secondary server.", 1, cattr.getFailoverIndex());
+ assertNotSame("Should have diferent primary now", primary, facade.getPrimaryServer());
+ }
}
diff --git a/commons-jcs-core/src/test/java/org/apache/commons/jcs3/auxiliary/remote/TestRemoteCacheFactory.java b/commons-jcs-core/src/test/java/org/apache/commons/jcs3/auxiliary/remote/TestRemoteCacheFactory.java
new file mode 100644
index 0000000..5620aad
--- /dev/null
+++ b/commons-jcs-core/src/test/java/org/apache/commons/jcs3/auxiliary/remote/TestRemoteCacheFactory.java
@@ -0,0 +1,133 @@
+package org.apache.commons.jcs3.auxiliary.remote;
+
+import java.io.IOException;
+import java.rmi.registry.Registry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.jcs3.auxiliary.remote.behavior.IRemoteCacheAttributes;
+import org.apache.commons.jcs3.engine.behavior.ICompositeCacheManager;
+import org.apache.commons.jcs3.engine.behavior.IElementSerializer;
+import org.apache.commons.jcs3.engine.logging.behavior.ICacheEventLogger;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Test RemoteCache factory that skips actual connection attempt
+ */
+public class TestRemoteCacheFactory extends RemoteCacheFactory
+{
+ /** Contains mappings of RemoteLocation instance to RemoteCacheManager instance. */
+ protected ConcurrentMap<RemoteLocation, RemoteCacheManager> managers;
+
+ /**
+ * Returns an instance of RemoteCacheManager for the given connection parameters.
+ * <p>
+ * Host and Port uniquely identify a manager instance.
+ * <p>
+ * @param cattr
+ *
+ * @return The instance value or null if no such manager exists
+ */
+ @Override
+ public RemoteCacheManager getManager( final IRemoteCacheAttributes cattr )
+ {
+ final RemoteCacheAttributes rca = (RemoteCacheAttributes) cattr.clone();
+ if (rca.getRemoteLocation() == null)
+ {
+ rca.setRemoteLocation("", Registry.REGISTRY_PORT);
+ }
+
+ return managers.get(rca.getRemoteLocation());
+ }
+
+ /**
+ * Returns an instance of RemoteCacheManager for the given connection parameters.
+ * <p>
+ * Host and Port uniquely identify a manager instance.
+ * <p>
+ * If the connection cannot be established, zombie objects will be used for future recovery
+ * purposes.
+ * <p>
+ * @param cattr the cache configuration object
+ * @param cacheMgr the cache manager
+ * @param cacheEventLogger the event logger
+ * @param elementSerializer the serializer to use for sending and receiving
+ *
+ * @return The instance value, never null
+ */
+ @Override
+ public RemoteCacheManager getManager( final IRemoteCacheAttributes cattr,
+ final ICompositeCacheManager cacheMgr,
+ final ICacheEventLogger cacheEventLogger,
+ final IElementSerializer elementSerializer )
+ {
+ final RemoteCacheAttributes rca = (RemoteCacheAttributes) cattr.clone();
+ if (rca.getRemoteLocation() == null)
+ {
+ rca.setRemoteLocation("", Registry.REGISTRY_PORT);
+ }
+
+ return managers.computeIfAbsent(rca.getRemoteLocation(), key -> {
+
+ return new TestRemoteCacheManager(rca, cacheMgr, null, cacheEventLogger, elementSerializer);
+ });
+ }
+
+ /**
+ * @see org.apache.commons.jcs3.auxiliary.AbstractAuxiliaryCacheFactory#initialize()
+ */
+ @Override
+ public void initialize()
+ {
+ managers = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * @see org.apache.commons.jcs3.auxiliary.AbstractAuxiliaryCacheFactory#dispose()
+ */
+ @Override
+ public void dispose()
+ {
+ managers.values().forEach(RemoteCacheManager::release);
+ managers.clear();
+ }
+
+ // Mock
+ public class TestRemoteCacheManager extends RemoteCacheManager
+ {
+ protected TestRemoteCacheManager(IRemoteCacheAttributes cattr, ICompositeCacheManager cacheMgr, RemoteCacheMonitor monitor, ICacheEventLogger cacheEventLogger,
+ IElementSerializer elementSerializer)
+ {
+ super(cattr, cacheMgr, monitor, cacheEventLogger, elementSerializer);
+ }
+
+ @Override
+ protected void lookupRemoteService() throws IOException
+ {
+ // Skip
+ }
+
+ @Override
+ public void removeRemoteCacheListener(IRemoteCacheAttributes cattr) throws IOException
+ {
+ // Skip
+ }
+ }
+}