You are viewing a plain text version of this content. The canonical link for it is here.
Posted to derby-commits@db.apache.org by km...@apache.org on 2013/04/04 03:12:55 UTC
svn commit: r1464247 - in /db/derby/code/trunk/java:
engine/org/apache/derby/iapi/types/
testing/org/apache/derbyTesting/functionTests/tests/memory/
Author: kmarsden
Date: Thu Apr 4 01:12:55 2013
New Revision: 1464247
URL: http://svn.apache.org/r1464247
Log:
DERBY-6096 OutOfMemoryError with Clob or Blob hash join: DataTypeDescriptor.estimatedMemoryUsage() has no case for BLOB or CLOB so would underestimate memory usage for those types at zero
Estimate BLOB/CLOB size at 10000 like other long types.
Modified:
db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/DataTypeDescriptor.java
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/ClobMemTest.java
Modified: db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/DataTypeDescriptor.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/DataTypeDescriptor.java?rev=1464247&r1=1464246&r2=1464247&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/DataTypeDescriptor.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/DataTypeDescriptor.java Thu Apr 4 01:12:55 2013
@@ -1491,6 +1491,8 @@ public final class DataTypeDescriptor im
switch (typeId.getTypeFormatId())
{
case StoredFormatIds.LONGVARBIT_TYPE_ID:
+ //DERBY-6096 Need to estimate usage for BLOB and clob
+ case StoredFormatIds.BLOB_TYPE_ID:
/* Who knows? Let's just use some big number */
return 10000.0;
@@ -1505,6 +1507,8 @@ public final class DataTypeDescriptor im
return (double) (2.0 * getMaximumWidth());
case StoredFormatIds.LONGVARCHAR_TYPE_ID:
+ //DERBY-6096 Need to estimate usage for BLOB and clob
+ case StoredFormatIds.CLOB_TYPE_ID:
/* Who knows? Let's just use some big number */
return 10000.0;
Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java?rev=1464247&r1=1464246&r2=1464247&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java (original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java Thu Apr 4 01:12:55 2013
@@ -32,6 +32,8 @@ import java.sql.Statement;
import java.util.Arrays;
import java.util.Properties;
import junit.framework.Test;
+import junit.framework.TestSuite;
+
import org.apache.derbyTesting.functionTests.util.streams.LoopingAlphabetStream;
import org.apache.derbyTesting.junit.BaseJDBCTestCase;
import org.apache.derbyTesting.junit.JDBC;
@@ -199,7 +201,11 @@ public class BlobMemTest extends BaseJDB
testBlobLength(false, 10000);
}
public static Test suite() {
- Test suite = TestConfiguration.defaultSuite(BlobMemTest.class);
+ TestSuite suite = new TestSuite();
+ // Just add Derby-6096 embedded as it takes time to run
+ suite.addTest(new BlobMemTest("xtestderby6096BlobhashJoin"));
+ suite.addTest(TestConfiguration.defaultSuite(BlobMemTest.class));
+
Properties p = new Properties();
// use small pageCacheSize so we don't run out of memory on the insert.
p.setProperty("derby.storage.pageCacheSize", "100");
@@ -319,4 +325,36 @@ public class BlobMemTest extends BaseJDB
assertEquals(lobSize, blobs[i].length());
}
}
+
+ /**
+ *
+ * DERBY-6096 Make blob hash join does not run out of memory.
+ * Prior to fix blobs were estimated at 0. We will test with
+ * 32K blobs even though the estimatedUsage is at 10k. The default
+ * max memory per table is only 1MB.
+ *
+ * @throws SQLException
+ */
+ public void xtestderby6096BlobhashJoin() throws SQLException {
+ byte[] b = new byte[32000];
+ Arrays.fill(b, (byte) 'a');
+ Statement s = createStatement();
+ s.execute("create table d6096(i int, b blob)");
+ PreparedStatement ps = prepareStatement("insert into d6096 values (?, ?)");
+ ps.setBytes(2, b);
+ for (int i = 0; i < 2000; i++) {
+ ps.setInt(1, i);
+ ps.execute();
+ }
+ ResultSet rs = s.executeQuery("select * from d6096 t1, d6096 t2 where t1.i=t2.i");
+ // just a single fetch will build the hash table and consume the memory.
+ assertTrue(rs.next());
+ // derby.tests.debug prints memory usage
+ System.gc();
+ println("TotalMemory:" + Runtime.getRuntime().totalMemory()
+ + " " + "Free Memory:"
+ + Runtime.getRuntime().freeMemory());
+ rs.close();
+ }
+
}
Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/ClobMemTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/ClobMemTest.java?rev=1464247&r1=1464246&r2=1464247&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/ClobMemTest.java (original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/ClobMemTest.java Thu Apr 4 01:12:55 2013
@@ -34,6 +34,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import junit.framework.Test;
+import junit.framework.TestSuite;
import org.apache.derbyTesting.functionTests.util.streams.LoopingAlphabetReader;
import org.apache.derbyTesting.junit.BaseJDBCTestCase;
@@ -182,7 +183,10 @@ public class ClobMemTest extends BaseJDB
}
public static Test suite() {
- Test suite = TestConfiguration.defaultSuite(ClobMemTest.class);
+ TestSuite suite = new TestSuite();
+ // Just add Derby-6096 embedded as it takes time to run
+ suite.addTest(new ClobMemTest("xtestderby6096ClobHashJoin"));
+ suite.addTest(TestConfiguration.defaultSuite(ClobMemTest.class));
Properties p = new Properties();
// use small pageCacheSize so we don't run out of memory on the insert.
p.setProperty("derby.storage.pageCacheSize", "100");
@@ -263,4 +267,38 @@ public class ClobMemTest extends BaseJDB
rollback();
}
+
+
+ /**
+ *
+ * DERBY-6096 Make clob hash join does not run out of memory.
+ * Prior to fix clobs were estimated at 0. We will test with
+ * 32K clobs even though the estimatedUsage is at 10k. The default
+ * max memory per table is only 1MB.
+ *
+ * @throws SQLException
+ */
+ public void xtestderby6096ClobHashJoin() throws SQLException {
+ char[] c = new char[32000];
+ Arrays.fill(c, 'a');
+ String cdata = new String(new char[32000]);
+ Statement s = createStatement();
+ s.execute("create table d6096(i int, c clob)");
+ PreparedStatement ps = prepareStatement("insert into d6096 values (?, ?)");
+ ps.setString(2, cdata);
+ for (int i = 0; i < 2000; i++) {
+ ps.setInt(1, i);
+ ps.execute();
+ }
+ ResultSet rs = s.executeQuery("select * from d6096 t1, d6096 t2 where t1.i=t2.i");
+ // just a single fetch will build the hash table and consume the memory.
+ assertTrue(rs.next());
+ // derby.tests.debug prints memory usage
+ System.gc();
+ println("TotalMemory:" + Runtime.getRuntime().totalMemory()
+ + " " + "Free Memory:"
+ + Runtime.getRuntime().freeMemory());
+ rs.close();
+ }
}
+