You are viewing a plain text version of this content. The canonical link for it is here.
Posted to derby-commits@db.apache.org by ka...@apache.org on 2011/04/14 11:08:09 UTC
svn commit: r1092067 - in /db/derby/code/trunk/java:
engine/org/apache/derby/impl/store/access/btree/index/
engine/org/apache/derby/impl/store/access/heap/
engine/org/apache/derby/impl/store/raw/data/
testing/org/apache/derbyTesting/functionTests/tests...
Author: kahatlen
Date: Thu Apr 14 09:08:08 2011
New Revision: 1092067
URL: http://svn.apache.org/viewvc?rev=1092067&view=rev
Log:
DERBY-2354: Unable to perform select query using DISTINCT on a read-only database
Made databases that are stored in a jar or a zip use a writable
directory for temporary files needed by some operations (typically
hash scans when the hash table gets too big to be kept in memory).
The writable directory can be controlled by the database property
derby.storage.tempDirectory.
Modified:
db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java
db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java
db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java
db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java
db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java
Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java?rev=1092067&r1=1092066&r2=1092067&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java Thu Apr 14 09:08:08 2011
@@ -36,6 +36,7 @@ import org.apache.derby.iapi.store.acces
import org.apache.derby.iapi.store.access.conglomerate.ConglomerateFactory;
import org.apache.derby.iapi.store.access.conglomerate.TransactionManager;
import org.apache.derby.iapi.store.access.ColumnOrdering;
+import org.apache.derby.iapi.store.access.TransactionController;
import org.apache.derby.iapi.store.raw.ContainerKey;
import org.apache.derby.iapi.store.raw.ContainerHandle;
@@ -173,7 +174,17 @@ public class B2IFactory implements Congl
{
B2I btree = null;
- if (xact_mgr.checkVersion(
+ if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 &&
+ xact_mgr.getAccessManager().isReadOnly())
+ {
+ // If this is a temporary conglomerate created for a read-only
+ // database, we don't really care which disk format we use, since
+ // it is not used for persisting data in the database. Use the
+ // current format. A special case is needed because checkVersion()
+ // throws an exception in read-only databases (DERBY-2354).
+ btree = new B2I();
+ }
+ else if (xact_mgr.checkVersion(
RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10,
RawStoreFactory.DERBY_STORE_MINOR_VERSION_4,
null))
Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java?rev=1092067&r1=1092066&r2=1092067&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java Thu Apr 14 09:08:08 2011
@@ -34,6 +34,7 @@ import org.apache.derby.iapi.store.acces
import org.apache.derby.iapi.store.access.conglomerate.TransactionManager;
import org.apache.derby.iapi.store.access.ColumnOrdering;
+import org.apache.derby.iapi.store.access.TransactionController;
import org.apache.derby.iapi.store.raw.ContainerHandle;
import org.apache.derby.iapi.store.raw.FetchDescriptor;
@@ -172,7 +173,17 @@ public class HeapConglomerateFactory imp
Heap heap = null;
- if (xact_mgr.checkVersion(
+ if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 &&
+ xact_mgr.getAccessManager().isReadOnly())
+ {
+ // If this is a temporary conglomerate created for a read-only
+ // database, we don't really care which disk format we use, since
+ // it is not used for persisting data in the database. Use the
+ // current format. A special case is needed because checkVersion()
+ // throws an exception in read-only databases (DERBY-2354).
+ heap = new Heap();
+ }
+ else if (xact_mgr.checkVersion(
RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10,
RawStoreFactory.DERBY_STORE_MINOR_VERSION_3,
null))
Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java?rev=1092067&r1=1092066&r2=1092067&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java Thu Apr 14 09:08:08 2011
@@ -347,7 +347,15 @@ abstract class FileContainer
*/
public Cacheable setIdentity(Object key) throws StandardException
{
- return setIdent((ContainerKey) key);
+ ContainerKey newIdentity = (ContainerKey) key;
+
+ // If the new identity represents a temporary container, switch to
+ // TempRAFContainer.
+ if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
+ return new TempRAFContainer(dataFactory).setIdent(newIdentity);
+ }
+
+ return setIdent(newIdentity);
}
/**
@@ -389,13 +397,16 @@ abstract class FileContainer
public Cacheable createIdentity(Object key, Object createParameter)
throws StandardException
{
- if (SanityManager.DEBUG)
- {
- SanityManager.ASSERT(
- !(key instanceof PageKey), "PageKey input to create container");
- }
+ ContainerKey newIdentity = (ContainerKey) key;
+
+ // If the new identity represents a temporary container, switch to
+ // TempRAFContainer.
+ if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
+ TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory);
+ return tmpContainer.createIdent(newIdentity, createParameter);
+ }
- return createIdent((ContainerKey) key, createParameter);
+ return createIdent(newIdentity, createParameter);
}
Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java?rev=1092067&r1=1092066&r2=1092067&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java Thu Apr 14 09:08:08 2011
@@ -22,14 +22,7 @@
package org.apache.derby.impl.store.raw.data;
import org.apache.derby.iapi.reference.SQLState;
-import org.apache.derby.impl.store.raw.data.BaseContainer;
-import org.apache.derby.impl.store.raw.data.BaseContainerHandle;
-import org.apache.derby.impl.store.raw.data.BasePage;
-
-import org.apache.derby.iapi.services.cache.Cacheable;
-import org.apache.derby.iapi.services.context.ContextService;
-import org.apache.derby.iapi.services.monitor.Monitor;
-import org.apache.derby.iapi.services.diag.Performance;
+
import org.apache.derby.iapi.services.sanity.SanityManager;
import org.apache.derby.iapi.services.io.FormatIdUtil;
@@ -40,18 +33,12 @@ import org.apache.derby.iapi.error.Stand
import org.apache.derby.iapi.store.raw.ContainerHandle;
import org.apache.derby.iapi.store.raw.ContainerKey;
-import org.apache.derby.iapi.store.raw.Loggable;
import org.apache.derby.iapi.store.raw.log.LogInstant;
-import org.apache.derby.iapi.store.raw.xact.RawTransaction;
-import org.apache.derby.io.StorageFactory;
-import org.apache.derby.io.WritableStorageFactory;
import org.apache.derby.io.StorageFile;
import org.apache.derby.io.StorageRandomAccessFile;
import org.apache.derby.iapi.services.io.FileUtil;
-import java.util.Vector;
-import java.io.DataInput;
import java.io.IOException;
import java.io.File;
import java.io.FileNotFoundException;
@@ -127,44 +114,6 @@ class RAFContainer extends FileContainer
}
/*
- ** Methods of Cacheable
- */
-
- /**
- Set container's identity
- @exception StandardException Standard Derby error policy
- */
- public Cacheable setIdentity(Object key) throws StandardException {
-
- ContainerKey newIdentity = (ContainerKey) key;
-
- // if this is an open for a temp container then return an object of that type
- if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
-
- TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory);
- return tmpContainer.setIdent(newIdentity);
- }
-
- return setIdent(newIdentity);
- }
-
- /**
- @exception StandardException Standard Derby error policy
- */
- public Cacheable createIdentity(Object key, Object createParameter) throws StandardException {
-
- ContainerKey newIdentity = (ContainerKey) key;
-
- if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
- TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory);
- return tmpContainer.createIdent(newIdentity, createParameter);
- }
-
- return createIdent(newIdentity, createParameter);
- }
-
-
- /*
** Container creation, opening, and closing
*/
Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java?rev=1092067&r1=1092066&r2=1092067&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java Thu Apr 14 09:08:08 2011
@@ -65,7 +65,8 @@ class TempRAFContainer extends RAFContai
ContainerKey newIdentity = (ContainerKey) key;
if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) {
- RAFContainer realContainer = new RAFContainer(dataFactory);
+ FileContainer realContainer =
+ (FileContainer) dataFactory.newContainerObject();
return realContainer.setIdent(newIdentity);
}
@@ -81,7 +82,7 @@ class TempRAFContainer extends RAFContai
ContainerKey newIdentity = (ContainerKey) key;
if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) {
- RAFContainer realContainer = new RAFContainer(dataFactory);
+ Cacheable realContainer = dataFactory.newContainerObject();
return realContainer.createIdentity(newIdentity, createParameter);
}
Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java?rev=1092067&r1=1092066&r2=1092067&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java (original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java Thu Apr 14 09:08:08 2011
@@ -28,18 +28,15 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.DriverManager;
-
+import java.sql.PreparedStatement;
import junit.framework.Test;
import junit.framework.TestSuite;
-import org.apache.derbyTesting.functionTests.tests.jdbcapi.BatchUpdateTest;
import org.apache.derbyTesting.junit.BaseJDBCTestCase;
import org.apache.derbyTesting.junit.CleanDatabaseTestSetup;
import org.apache.derbyTesting.junit.JDBC;
import org.apache.derbyTesting.junit.SecurityManagerSetup;
-import org.apache.derbyTesting.junit.TestConfiguration;
-import org.apache.derbyTesting.junit.Utilities;
public class DBInJarTest extends BaseJDBCTestCase {
@@ -105,6 +102,97 @@ public class DBInJarTest extends BaseJDB
}
}
+ /**
+ * Test various queries that use a hash table that may be spilled to disk
+ * if it grows too big. Regression test case for DERBY-2354.
+ */
+ public void testSpillHashToDisk() throws SQLException {
+ createDerby2354Database();
+
+ Connection jarConn =
+ DriverManager.getConnection("jdbc:derby:jar:(d2354db.jar)d2354db");
+
+ Statement stmt = jarConn.createStatement();
+
+ // The following statement used to fail with "Feature not implemented"
+ // or "Container was opened in read-only mode" before DERBY-2354. It
+ // only fails if the hash table used for duplicate elimination spills
+ // to disk, which happens if the hash table gets bigger than 1% of the
+ // total amount of memory allocated to the JVM. This means it won't
+ // expose the bug if the JVM runs with very high memory settings (but
+ // it has been tested with 1 GB heap size and then it did spill to
+ // disk).
+ JDBC.assertDrainResults(
+ stmt.executeQuery("select distinct x from d2354"),
+ 40000);
+
+ // Hash joins have the same problem. Force the big table to be used as
+ // the inner table in the hash join.
+ JDBC.assertEmpty(stmt.executeQuery(
+ "select * from --DERBY-PROPERTIES joinOrder = FIXED\n" +
+ "sysibm.sysdummy1 t1(x),\n" +
+ "d2354 t2 --DERBY-PROPERTIES joinStrategy = HASH\n" +
+ "where t1.x = t2.x"));
+
+ // Scrollable result sets keep the rows they've visited in a hash
+ // table, so they may also need to store data on disk temporarily.
+ Statement scrollStmt = jarConn.createStatement(
+ ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+ JDBC.assertDrainResults(
+ scrollStmt.executeQuery("select * from d2354"),
+ 40000);
+
+ stmt.close();
+ scrollStmt.close();
+ jarConn.close();
+
+ // Cleanup. Shut down the database and delete it.
+ shutdownDB("jdbc:derby:jar:(d2354db.jar)d2354db;shutdown=true");
+ removeFiles(new String[] {
+ System.getProperty("derby.system.home") + "/d2354db.jar"
+ });
+ }
+
+ /**
+ * Create a database in a jar for use in {@code testSpillHashToDisk}.
+ */
+ private void createDerby2354Database() throws SQLException {
+ // First create an ordinary database with a table.
+ Connection conn =
+ DriverManager.getConnection("jdbc:derby:d2354db;create=true");
+ conn.setAutoCommit(false);
+ Statement s = conn.createStatement();
+ s.execute("create table d2354 (x varchar(100))");
+ s.close();
+
+ // Insert 40000 unique values into the table. The values should be
+ // unique so that they all occupy an entry in the hash table used by
+ // the DISTINCT query in the test, and thereby increase the likelihood
+ // of spilling to disk.
+ PreparedStatement insert =
+ conn.prepareStatement(
+ "insert into d2354 values ? || " +
+ "'some extra data to increase the size of the table'");
+ for (int i = 0; i < 40000; i++) {
+ insert.setInt(1, i);
+ insert.executeUpdate();
+ }
+ insert.close();
+
+ conn.commit();
+ conn.close();
+
+ // Shut down the database and archive it in a jar file.
+ shutdownDB("jdbc:derby:d2354db;shutdown=true");
+
+ createStatement().execute(
+ "CALL CREATEARCHIVE('d2354db.jar', 'd2354db', 'd2354db')");
+
+ // Clean up the original database directory. We don't need it anymore
+ // now that we have archived it in a jar file.
+ removeDirectory(
+ new File(System.getProperty("derby.system.home") + "/d2354db"));
+ }
protected static Test baseSuite(String name) {
TestSuite suite = new TestSuite(name);