You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cayenne.apache.org by aa...@apache.org on 2016/03/09 20:29:11 UTC

[1/4] cayenne git commit: Cayenne 4.0 connection pool is occasionally running out of connections

Repository: cayenne
Updated Branches:
  refs/heads/master 0e0da8064 -> b0631deb2


Cayenne 4.0 connection pool is occasionally running out of connections

* refactoring existing test. It doesn't seem to do the right thing

Project: http://git-wip-us.apache.org/repos/asf/cayenne/repo
Commit: http://git-wip-us.apache.org/repos/asf/cayenne/commit/2bc74c26
Tree: http://git-wip-us.apache.org/repos/asf/cayenne/tree/2bc74c26
Diff: http://git-wip-us.apache.org/repos/asf/cayenne/diff/2bc74c26

Branch: refs/heads/master
Commit: 2bc74c2645c71acf22125ad1e24d82728fd8f71d
Parents: 0e0da80
Author: Andrus Adamchik <an...@objectstyle.com>
Authored: Wed Mar 9 07:49:10 2016 -0800
Committer: Andrus Adamchik <an...@objectstyle.com>
Committed: Wed Mar 9 07:49:10 2016 -0800

----------------------------------------------------------------------
 .../apache/cayenne/datasource/PoolingDataSourceIT.java | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cayenne/blob/2bc74c26/cayenne-server/src/test/java/org/apache/cayenne/datasource/PoolingDataSourceIT.java
----------------------------------------------------------------------
diff --git a/cayenne-server/src/test/java/org/apache/cayenne/datasource/PoolingDataSourceIT.java b/cayenne-server/src/test/java/org/apache/cayenne/datasource/PoolingDataSourceIT.java
index 68bbef2..d7d8bf1 100644
--- a/cayenne-server/src/test/java/org/apache/cayenne/datasource/PoolingDataSourceIT.java
+++ b/cayenne-server/src/test/java/org/apache/cayenne/datasource/PoolingDataSourceIT.java
@@ -174,8 +174,8 @@ public class PoolingDataSourceIT extends BasePoolingDataSourceIT {
 		ExecutorService executor = Executors.newFixedThreadPool(tasks.length);
 
 		for (int j = 0; j < 100; j++) {
-			for (int i = 0; i < tasks.length; i++) {
-				executor.submit(tasks[i]);
+			for (PoolTask task : tasks) {
+				executor.submit(task);
 			}
 		}
 
@@ -192,8 +192,8 @@ public class PoolingDataSourceIT extends BasePoolingDataSourceIT {
 			throw new RuntimeException(e);
 		}
 
-		for (int i = 0; i < tasks.length; i++) {
-			assertEquals(100, tasks[i].i.get());
+		for (PoolTask task : tasks) {
+			assertEquals(100, task.i.get());
 		}
 	}
 
@@ -204,7 +204,6 @@ public class PoolingDataSourceIT extends BasePoolingDataSourceIT {
 		@Override
 		public void run() {
 
-			i.incrementAndGet();
 			try {
 				Connection c = dataSource.getConnection();
 				try {
@@ -220,6 +219,7 @@ public class PoolingDataSourceIT extends BasePoolingDataSourceIT {
 						} finally {
 							rs.close();
 						}
+
 					} finally {
 						st.close();
 					}
@@ -228,6 +228,9 @@ public class PoolingDataSourceIT extends BasePoolingDataSourceIT {
 					c.close();
 				}
 
+				// increment only after success
+				i.incrementAndGet();
+
 			} catch (SQLException e) {
 				e.printStackTrace();
 			}


[2/4] cayenne git commit: typo

Posted by aa...@apache.org.
typo

Project: http://git-wip-us.apache.org/repos/asf/cayenne/repo
Commit: http://git-wip-us.apache.org/repos/asf/cayenne/commit/ab6faeaa
Tree: http://git-wip-us.apache.org/repos/asf/cayenne/tree/ab6faeaa
Diff: http://git-wip-us.apache.org/repos/asf/cayenne/diff/ab6faeaa

Branch: refs/heads/master
Commit: ab6faeaa609b4e8e975563f1b519114a605097e5
Parents: 2bc74c2
Author: Andrus Adamchik <an...@objectstyle.com>
Authored: Wed Mar 9 07:53:05 2016 -0800
Committer: Andrus Adamchik <an...@objectstyle.com>
Committed: Wed Mar 9 07:53:05 2016 -0800

----------------------------------------------------------------------
 .../org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cayenne/blob/ab6faeaa/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
----------------------------------------------------------------------
diff --git a/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java b/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
index 0ffd34c..560348c 100644
--- a/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
+++ b/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
@@ -136,7 +136,7 @@ public class UnmanagedPoolingDataSource implements PoolingDataSource {
 		this.poolCap = new Semaphore(maxConnections);
 		this.maxIdleConnections = maxIdleConnections(minConnections, maxConnections);
 
-		// grow pull to min connections
+		// grow pool to min connections
 		try {
 			for (int i = 0; i < minConnections; i++) {
 				PoolAwareConnection c = createUnchecked();


[4/4] cayenne git commit: Cayenne 4.0 connection pool is occasionally running out of connections

Posted by aa...@apache.org.
Cayenne 4.0 connection pool is occasionally running out of connections

* reproduced in a unit test and fixed


Project: http://git-wip-us.apache.org/repos/asf/cayenne/repo
Commit: http://git-wip-us.apache.org/repos/asf/cayenne/commit/b0631deb
Tree: http://git-wip-us.apache.org/repos/asf/cayenne/tree/b0631deb
Diff: http://git-wip-us.apache.org/repos/asf/cayenne/diff/b0631deb

Branch: refs/heads/master
Commit: b0631deb251f036840d1ca3aee6d4ae50f2441bf
Parents: 0a9974b
Author: Andrus Adamchik <an...@objectstyle.com>
Authored: Wed Mar 9 07:49:10 2016 -0800
Committer: Andrus Adamchik <an...@objectstyle.com>
Committed: Wed Mar 9 13:28:54 2016 -0600

----------------------------------------------------------------------
 .../datasource/ManagedPoolingDataSource.java    |   4 +
 .../datasource/UnmanagedPoolingDataSource.java  |  19 ++-
 .../datasource/ManagedPoolingDataSourceIT.java  | 170 +++++++++++++------
 docs/doc/src/main/resources/RELEASE-NOTES.txt   |   1 +
 4 files changed, 135 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cayenne/blob/b0631deb/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
----------------------------------------------------------------------
diff --git a/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java b/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
index 1075e3d..367a6e0 100644
--- a/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
+++ b/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
@@ -62,6 +62,10 @@ public class ManagedPoolingDataSource implements PoolingDataSource, ScopeEventLi
 	int availableSize() {
 		return dataSourceManager.getDataSource().availableSize();
 	}
+	
+	int canExpandSize() {
+		return dataSourceManager.getDataSource().canExpandSize();
+	}
 
 	/**
 	 * Calls {@link #shutdown()} to drain the underlying pool, close open

http://git-wip-us.apache.org/repos/asf/cayenne/blob/b0631deb/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
----------------------------------------------------------------------
diff --git a/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java b/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
index 560348c..2d0e6e2 100644
--- a/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
+++ b/cayenne-server/src/main/java/org/apache/cayenne/datasource/UnmanagedPoolingDataSource.java
@@ -157,6 +157,10 @@ public class UnmanagedPoolingDataSource implements PoolingDataSource {
 		return available.size();
 	}
 
+	int canExpandSize() {
+		return poolCap.availablePermits();
+	}
+
 	@Override
 	public void close() {
 
@@ -264,7 +268,13 @@ public class UnmanagedPoolingDataSource implements PoolingDataSource {
 			return null;
 		}
 
-		PoolAwareConnection c = createWrapped();
+		PoolAwareConnection c;
+		try {
+			c = createWrapped();
+		} catch (SQLException e) {
+			poolCap.release();
+			throw e;
+		}
 
 		pool.put(c, 1);
 
@@ -335,8 +345,11 @@ public class UnmanagedPoolingDataSource implements PoolingDataSource {
 			return resetState(c);
 		}
 
-		throw new ConnectionUnavailableException(
-				"Can't obtain connection. Request to pool timed out. Total pool size: " + pool.size());
+		int poolSize = poolSize();
+		int canGrow = poolCap.availablePermits();
+
+		throw new ConnectionUnavailableException("Can't obtain connection. Request to pool timed out. Total pool size: "
+				+ poolSize + ", can expand by: " + canGrow);
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/cayenne/blob/b0631deb/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java
----------------------------------------------------------------------
diff --git a/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java b/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java
index 1580ce5..04a4540 100644
--- a/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java
+++ b/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java
@@ -1,7 +1,6 @@
 package org.apache.cayenne.datasource;
 
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -10,14 +9,18 @@ import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import javax.sql.DataSource;
 
 import org.apache.cayenne.unit.di.server.CayenneProjects;
 import org.apache.cayenne.unit.di.server.UseServerRuntime;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -28,42 +31,28 @@ import org.mockito.stubbing.OngoingStubbing;
 @UseServerRuntime(CayenneProjects.TESTMAP_PROJECT)
 public class ManagedPoolingDataSourceIT {
 
+	private static final Log LOGGER = LogFactory.getLog(ManagedPoolingDataSourceIT.class);
+
 	private int poolSize;
-	private Map<ExpiringConnection, Object> connections;
+	private OnOffDataSourceManager dataSourceManager;
 	private UnmanagedPoolingDataSource unmanagedPool;
 	private ManagedPoolingDataSource managedPool;
 
 	@Before
 	public void before() throws SQLException {
 
-		this.poolSize = 3;
-		this.connections = new ConcurrentHashMap<>();
-
-		DataSource mockDataSource = mock(DataSource.class);
-		when(mockDataSource.getConnection()).thenAnswer(new Answer<Connection>() {
-
-			@Override
-			public Connection answer(InvocationOnMock invocation) throws Throwable {
-				return createMockConnection();
-			}
-		});
+		this.poolSize = 4;
+		this.dataSourceManager = new OnOffDataSourceManager();
 
 		PoolingDataSourceParameters parameters = new PoolingDataSourceParameters();
 		parameters.setMaxConnections(poolSize);
-		parameters.setMinConnections(poolSize - 1);
-		parameters.setMaxQueueWaitTime(500);
+		parameters.setMinConnections(poolSize / 2);
+		parameters.setMaxQueueWaitTime(1000);
 		parameters.setValidationQuery("SELECT 1");
-		this.unmanagedPool = new UnmanagedPoolingDataSource(mockDataSource, parameters);
-
+		this.unmanagedPool = new UnmanagedPoolingDataSource(dataSourceManager.mockDataSource, parameters);
 		this.managedPool = new ManagedPoolingDataSource(unmanagedPool, 10000);
 	}
 
-	private Connection createMockConnection() throws SQLException {
-		ExpiringConnection connectionWrapper = new ExpiringConnection();
-		connections.put(connectionWrapper, 1);
-		return connectionWrapper.mockConnection;
-	}
-
 	@After
 	public void after() {
 		if (managedPool != null) {
@@ -71,53 +60,122 @@ public class ManagedPoolingDataSourceIT {
 		}
 	}
 
-	private void expireConnections() throws SQLException {
-		Iterator<ExpiringConnection> it = connections.keySet().iterator();
-		while (it.hasNext()) {
-			ExpiringConnection c = it.next();
-			it.remove();
-			c.expire();
+	private Collection<PoolTask> createTasks(int size) {
+		Collection<PoolTask> tasks = new ArrayList<>();
+
+		for (int i = 0; i < size; i++) {
+			tasks.add(new PoolTask());
 		}
+		return tasks;
 	}
 
 	@Test
-	public void testAbruptReset() throws SQLException {
+	public void testGetConnection_OnBackendShutdown() throws SQLException, InterruptedException {
+
+		// note that this assertion can only work reliably when the pool is inactive...
+		assertEquals(poolSize, managedPool.poolSize() + managedPool.canExpandSize());
+
+		Collection<PoolTask> tasks = createTasks(4);
+		ExecutorService executor = Executors.newFixedThreadPool(4);
+
+		for (int j = 0; j < 10; j++) {
+			for (PoolTask task : tasks) {
+				executor.submit(task);
+			}
+		}
+
+		dataSourceManager.off();
+		Thread.sleep(500);
+
+		for (int j = 0; j < 10; j++) {
+			for (PoolTask task : tasks) {
+				executor.submit(task);
+			}
+		}
 
-		assertTrue(managedPool.poolSize() > 0);
-		assertTrue(managedPool.availableSize() > 0);
+		Thread.sleep(100);
 
-		// make sure conn
-		expireConnections();
+		dataSourceManager.on();
 
-		// CAY-2067 ... this should work on an invalid pool
-		assertNotNull(managedPool.getConnection());
+		for (int j = 0; j < 10; j++) {
+			for (PoolTask task : tasks) {
+				executor.submit(task);
+			}
+		}
+
+		executor.shutdown();
+		executor.awaitTermination(2, TimeUnit.SECONDS);
+
+		// note that this assertion can only work reliably when the pool is inactive...
+		assertEquals(poolSize, managedPool.poolSize() + managedPool.canExpandSize());
 	}
 
-	static class ExpiringConnection {
+	class PoolTask implements Runnable {
 
-		private Connection mockConnection;
-		private OngoingStubbing<Statement> createStatementMock;
+		@Override
+		public void run() {
 
-		ExpiringConnection() throws SQLException {
-			this.mockConnection = mock(Connection.class);
-			this.createStatementMock = when(mockConnection.createStatement());
+			try (Connection c = managedPool.getConnection();) {
+				try (Statement s = c.createStatement()) {
+					try {
+						Thread.sleep(40);
+					} catch (InterruptedException e) {
+						// ignore
+					}
+				}
+			} catch (SQLException e) {
+				if (OnOffDataSourceManager.NO_CONNECTIONS_MESSAGE.equals(e.getMessage())) {
+					LOGGER.info("db down...");
+				} else {
+					LOGGER.warn("error getting connection", e);
+				}
+			}
+		}
+	}
 
-			createStatementMock.thenAnswer(new Answer<Statement>() {
-				@Override
-				public Statement answer(InvocationOnMock invocation) throws Throwable {
+	static class OnOffDataSourceManager {
+
+		static final String NO_CONNECTIONS_MESSAGE = "no connections at the moment";
 
-					ResultSet mockRs = mock(ResultSet.class);
-					when(mockRs.next()).thenReturn(true, false, false, false);
+		private DataSource mockDataSource;
+		private OngoingStubbing<Connection> createConnectionMock;
+
+		OnOffDataSourceManager() throws SQLException {
+			this.mockDataSource = mock(DataSource.class);
+			this.createConnectionMock = when(mockDataSource.getConnection());
+			on();
+		}
 
-					Statement mockStatement = mock(Statement.class);
-					when(mockStatement.executeQuery(anyString())).thenReturn(mockRs);
-					return mockStatement;
+		void off() throws SQLException {
+			createConnectionMock.thenAnswer(new Answer<Connection>() {
+				@Override
+				public Connection answer(InvocationOnMock invocation) throws Throwable {
+					throw new SQLException(NO_CONNECTIONS_MESSAGE);
 				}
 			});
 		}
 
-		void expire() throws SQLException {
-			createStatementMock.thenThrow(new SQLException("Expired"));
+		void on() throws SQLException {
+			createConnectionMock.thenAnswer(new Answer<Connection>() {
+				@Override
+				public Connection answer(InvocationOnMock invocation) throws Throwable {
+					Connection c = mock(Connection.class);
+					when(c.createStatement()).thenAnswer(new Answer<Statement>() {
+						@Override
+						public Statement answer(InvocationOnMock invocation) throws Throwable {
+
+							ResultSet mockRs = mock(ResultSet.class);
+							when(mockRs.next()).thenReturn(true, false, false, false);
+
+							Statement mockStatement = mock(Statement.class);
+							when(mockStatement.executeQuery(anyString())).thenReturn(mockRs);
+							return mockStatement;
+						}
+					});
+
+					return c;
+				}
+			});
 		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/cayenne/blob/b0631deb/docs/doc/src/main/resources/RELEASE-NOTES.txt
----------------------------------------------------------------------
diff --git a/docs/doc/src/main/resources/RELEASE-NOTES.txt b/docs/doc/src/main/resources/RELEASE-NOTES.txt
index c595659..3721d85 100644
--- a/docs/doc/src/main/resources/RELEASE-NOTES.txt
+++ b/docs/doc/src/main/resources/RELEASE-NOTES.txt
@@ -24,6 +24,7 @@ Bug Fixes:
 
 CAY-2064 Issue with BeanAccessor for classes with complex inheritance
 CAY-2066 Fixes for inner enums handling in ExtendedTypeMap
+CAY-2067 Cayenne 4.0 connection pool is occasionally running out of connections
 
 ----------------------------------
 Release: 4.0.M3


[3/4] cayenne git commit: Cayenne 4.0 connection pool is occasionally running out of connections

Posted by aa...@apache.org.
Cayenne 4.0 connection pool is occasionally running out of connections

* trying to reproduce


Project: http://git-wip-us.apache.org/repos/asf/cayenne/repo
Commit: http://git-wip-us.apache.org/repos/asf/cayenne/commit/0a9974bd
Tree: http://git-wip-us.apache.org/repos/asf/cayenne/tree/0a9974bd
Diff: http://git-wip-us.apache.org/repos/asf/cayenne/diff/0a9974bd

Branch: refs/heads/master
Commit: 0a9974bd1674857ca492789b46858ec3eb8249f4
Parents: ab6faea
Author: Andrus Adamchik <an...@objectstyle.com>
Authored: Wed Mar 9 07:49:10 2016 -0800
Committer: Andrus Adamchik <an...@objectstyle.com>
Committed: Wed Mar 9 08:52:55 2016 -0800

----------------------------------------------------------------------
 .../datasource/ManagedPoolingDataSource.java    |   8 ++
 .../datasource/ManagedPoolingDataSourceIT.java  | 123 +++++++++++++++++++
 2 files changed, 131 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cayenne/blob/0a9974bd/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
----------------------------------------------------------------------
diff --git a/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java b/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
index 3a794de..1075e3d 100644
--- a/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
+++ b/cayenne-server/src/main/java/org/apache/cayenne/datasource/ManagedPoolingDataSource.java
@@ -55,6 +55,14 @@ public class ManagedPoolingDataSource implements PoolingDataSource, ScopeEventLi
 		return dataSourceManager;
 	}
 
+	int poolSize() {
+		return dataSourceManager.getDataSource().poolSize();
+	}
+
+	int availableSize() {
+		return dataSourceManager.getDataSource().availableSize();
+	}
+
 	/**
 	 * Calls {@link #shutdown()} to drain the underlying pool, close open
 	 * connections and block the DataSource from creating any new connections.

http://git-wip-us.apache.org/repos/asf/cayenne/blob/0a9974bd/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java
----------------------------------------------------------------------
diff --git a/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java b/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java
new file mode 100644
index 0000000..1580ce5
--- /dev/null
+++ b/cayenne-server/src/test/java/org/apache/cayenne/datasource/ManagedPoolingDataSourceIT.java
@@ -0,0 +1,123 @@
+package org.apache.cayenne.datasource;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.sql.DataSource;
+
+import org.apache.cayenne.unit.di.server.CayenneProjects;
+import org.apache.cayenne.unit.di.server.UseServerRuntime;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.mockito.stubbing.OngoingStubbing;
+
+@UseServerRuntime(CayenneProjects.TESTMAP_PROJECT)
+public class ManagedPoolingDataSourceIT {
+
+	private int poolSize;
+	private Map<ExpiringConnection, Object> connections;
+	private UnmanagedPoolingDataSource unmanagedPool;
+	private ManagedPoolingDataSource managedPool;
+
+	@Before
+	public void before() throws SQLException {
+
+		this.poolSize = 3;
+		this.connections = new ConcurrentHashMap<>();
+
+		DataSource mockDataSource = mock(DataSource.class);
+		when(mockDataSource.getConnection()).thenAnswer(new Answer<Connection>() {
+
+			@Override
+			public Connection answer(InvocationOnMock invocation) throws Throwable {
+				return createMockConnection();
+			}
+		});
+
+		PoolingDataSourceParameters parameters = new PoolingDataSourceParameters();
+		parameters.setMaxConnections(poolSize);
+		parameters.setMinConnections(poolSize - 1);
+		parameters.setMaxQueueWaitTime(500);
+		parameters.setValidationQuery("SELECT 1");
+		this.unmanagedPool = new UnmanagedPoolingDataSource(mockDataSource, parameters);
+
+		this.managedPool = new ManagedPoolingDataSource(unmanagedPool, 10000);
+	}
+
+	private Connection createMockConnection() throws SQLException {
+		ExpiringConnection connectionWrapper = new ExpiringConnection();
+		connections.put(connectionWrapper, 1);
+		return connectionWrapper.mockConnection;
+	}
+
+	@After
+	public void after() {
+		if (managedPool != null) {
+			managedPool.close();
+		}
+	}
+
+	private void expireConnections() throws SQLException {
+		Iterator<ExpiringConnection> it = connections.keySet().iterator();
+		while (it.hasNext()) {
+			ExpiringConnection c = it.next();
+			it.remove();
+			c.expire();
+		}
+	}
+
+	@Test
+	public void testAbruptReset() throws SQLException {
+
+		assertTrue(managedPool.poolSize() > 0);
+		assertTrue(managedPool.availableSize() > 0);
+
+		// make sure conn
+		expireConnections();
+
+		// CAY-2067 ... this should work on an invalid pool
+		assertNotNull(managedPool.getConnection());
+	}
+
+	static class ExpiringConnection {
+
+		private Connection mockConnection;
+		private OngoingStubbing<Statement> createStatementMock;
+
+		ExpiringConnection() throws SQLException {
+			this.mockConnection = mock(Connection.class);
+			this.createStatementMock = when(mockConnection.createStatement());
+
+			createStatementMock.thenAnswer(new Answer<Statement>() {
+				@Override
+				public Statement answer(InvocationOnMock invocation) throws Throwable {
+
+					ResultSet mockRs = mock(ResultSet.class);
+					when(mockRs.next()).thenReturn(true, false, false, false);
+
+					Statement mockStatement = mock(Statement.class);
+					when(mockStatement.executeQuery(anyString())).thenReturn(mockRs);
+					return mockStatement;
+				}
+			});
+		}
+
+		void expire() throws SQLException {
+			createStatementMock.thenThrow(new SQLException("Expired"));
+		}
+	}
+}