You are viewing a plain text version of this content. The canonical link for it is here.
Posted to derby-commits@db.apache.org by ma...@apache.org on 2012/09/25 03:35:58 UTC
svn commit: r1389677 - in /db/derby/code/branches/10.7: ./
java/engine/org/apache/derby/impl/store/access/btree/index/
java/engine/org/apache/derby/impl/store/access/heap/
java/engine/org/apache/derby/impl/store/raw/data/
java/testing/org/apache/derbyT...
Author: mamta
Date: Tue Sep 25 01:35:57 2012
New Revision: 1389677
URL: http://svn.apache.org/viewvc?rev=1389677&view=rev
Log:
DERBY-2354 (Unable to perform select query using DISTINCT on a read-only database)
Backporting this jira from 10.8 commit. Additionally, I hand backported changes to BaseTestCase and DropDatabaseSetup from another checkin so DBInJarTest has access to supporting method from BaseTestCase
Modified:
db/derby/code/branches/10.7/ (props changed)
db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java
db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java
db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java
db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java
db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java
db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java
db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java
db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java
Propchange: db/derby/code/branches/10.7/
------------------------------------------------------------------------------
Merged /db/derby/code/trunk:r1092067
Merged /db/derby/code/branches/10.8:r1092649
Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java (original)
+++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java Tue Sep 25 01:35:57 2012
@@ -36,6 +36,7 @@ import org.apache.derby.iapi.store.acces
import org.apache.derby.iapi.store.access.conglomerate.ConglomerateFactory;
import org.apache.derby.iapi.store.access.conglomerate.TransactionManager;
import org.apache.derby.iapi.store.access.ColumnOrdering;
+import org.apache.derby.iapi.store.access.TransactionController;
import org.apache.derby.iapi.store.raw.ContainerKey;
import org.apache.derby.iapi.store.raw.ContainerHandle;
@@ -173,7 +174,17 @@ public class B2IFactory implements Congl
{
B2I btree = null;
- if (xact_mgr.checkVersion(
+ if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 &&
+ xact_mgr.getAccessManager().isReadOnly())
+ {
+ // If this is a temporary conglomerate created for a read-only
+ // database, we don't really care which disk format we use, since
+ // it is not used for persisting data in the database. Use the
+ // current format. A special case is needed because checkVersion()
+ // throws an exception in read-only databases (DERBY-2354).
+ btree = new B2I();
+ }
+ else if (xact_mgr.checkVersion(
RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10,
RawStoreFactory.DERBY_STORE_MINOR_VERSION_4,
null))
Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java (original)
+++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java Tue Sep 25 01:35:57 2012
@@ -34,6 +34,7 @@ import org.apache.derby.iapi.store.acces
import org.apache.derby.iapi.store.access.conglomerate.TransactionManager;
import org.apache.derby.iapi.store.access.ColumnOrdering;
+import org.apache.derby.iapi.store.access.TransactionController;
import org.apache.derby.iapi.store.raw.ContainerHandle;
import org.apache.derby.iapi.store.raw.FetchDescriptor;
@@ -172,7 +173,17 @@ public class HeapConglomerateFactory imp
Heap heap = null;
- if (xact_mgr.checkVersion(
+ if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 &&
+ xact_mgr.getAccessManager().isReadOnly())
+ {
+ // If this is a temporary conglomerate created for a read-only
+ // database, we don't really care which disk format we use, since
+ // it is not used for persisting data in the database. Use the
+ // current format. A special case is needed because checkVersion()
+ // throws an exception in read-only databases (DERBY-2354).
+ heap = new Heap();
+ }
+ else if (xact_mgr.checkVersion(
RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10,
RawStoreFactory.DERBY_STORE_MINOR_VERSION_3,
null))
Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java (original)
+++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java Tue Sep 25 01:35:57 2012
@@ -344,7 +344,15 @@ abstract class FileContainer
*/
public Cacheable setIdentity(Object key) throws StandardException
{
- return setIdent((ContainerKey) key);
+ ContainerKey newIdentity = (ContainerKey) key;
+
+ // If the new identity represents a temporary container, switch to
+ // TempRAFContainer.
+ if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
+ return new TempRAFContainer(dataFactory).setIdent(newIdentity);
+ }
+
+ return setIdent(newIdentity);
}
/**
@@ -386,13 +394,16 @@ abstract class FileContainer
public Cacheable createIdentity(Object key, Object createParameter)
throws StandardException
{
- if (SanityManager.DEBUG)
- {
- SanityManager.ASSERT(
- !(key instanceof PageKey), "PageKey input to create container");
- }
+ ContainerKey newIdentity = (ContainerKey) key;
+
+ // If the new identity represents a temporary container, switch to
+ // TempRAFContainer.
+ if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
+ TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory);
+ return tmpContainer.createIdent(newIdentity, createParameter);
+ }
- return createIdent((ContainerKey) key, createParameter);
+ return createIdent(newIdentity, createParameter);
}
Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java (original)
+++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java Tue Sep 25 01:35:57 2012
@@ -22,14 +22,7 @@
package org.apache.derby.impl.store.raw.data;
import org.apache.derby.iapi.reference.SQLState;
-import org.apache.derby.impl.store.raw.data.BaseContainer;
-import org.apache.derby.impl.store.raw.data.BaseContainerHandle;
-import org.apache.derby.impl.store.raw.data.BasePage;
-
-import org.apache.derby.iapi.services.cache.Cacheable;
-import org.apache.derby.iapi.services.context.ContextService;
-import org.apache.derby.iapi.services.monitor.Monitor;
-import org.apache.derby.iapi.services.diag.Performance;
+
import org.apache.derby.iapi.services.sanity.SanityManager;
import org.apache.derby.iapi.services.io.FormatIdUtil;
@@ -37,18 +30,12 @@ import org.apache.derby.iapi.error.Stand
import org.apache.derby.iapi.store.raw.ContainerHandle;
import org.apache.derby.iapi.store.raw.ContainerKey;
-import org.apache.derby.iapi.store.raw.Loggable;
import org.apache.derby.iapi.store.raw.log.LogInstant;
-import org.apache.derby.iapi.store.raw.xact.RawTransaction;
-import org.apache.derby.io.StorageFactory;
-import org.apache.derby.io.WritableStorageFactory;
import org.apache.derby.io.StorageFile;
import org.apache.derby.io.StorageRandomAccessFile;
import org.apache.derby.iapi.services.io.FileUtil;
-import java.util.Vector;
-import java.io.DataInput;
import java.io.IOException;
import java.io.File;
import java.io.RandomAccessFile;
@@ -114,44 +101,6 @@ class RAFContainer extends FileContainer
}
/*
- ** Methods of Cacheable
- */
-
- /**
- Set container's identity
- @exception StandardException Standard Derby error policy
- */
- public Cacheable setIdentity(Object key) throws StandardException {
-
- ContainerKey newIdentity = (ContainerKey) key;
-
- // if this is an open for a temp container then return an object of that type
- if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
-
- TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory);
- return tmpContainer.setIdent(newIdentity);
- }
-
- return setIdent(newIdentity);
- }
-
- /**
- @exception StandardException Standard Derby error policy
- */
- public Cacheable createIdentity(Object key, Object createParameter) throws StandardException {
-
- ContainerKey newIdentity = (ContainerKey) key;
-
- if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) {
- TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory);
- return tmpContainer.createIdent(newIdentity, createParameter);
- }
-
- return createIdent(newIdentity, createParameter);
- }
-
-
- /*
** Container creation, opening, and closing
*/
Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java (original)
+++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java Tue Sep 25 01:35:57 2012
@@ -65,7 +65,8 @@ class TempRAFContainer extends RAFContai
ContainerKey newIdentity = (ContainerKey) key;
if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) {
- RAFContainer realContainer = new RAFContainer(dataFactory);
+ FileContainer realContainer =
+ (FileContainer) dataFactory.newContainerObject();
return realContainer.setIdent(newIdentity);
}
@@ -81,7 +82,7 @@ class TempRAFContainer extends RAFContai
ContainerKey newIdentity = (ContainerKey) key;
if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) {
- RAFContainer realContainer = new RAFContainer(dataFactory);
+ Cacheable realContainer = dataFactory.newContainerObject();
return realContainer.createIdentity(newIdentity, createParameter);
}
Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java (original)
+++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java Tue Sep 25 01:35:57 2012
@@ -28,18 +28,15 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.DriverManager;
-
+import java.sql.PreparedStatement;
import junit.framework.Test;
import junit.framework.TestSuite;
-import org.apache.derbyTesting.functionTests.tests.jdbcapi.BatchUpdateTest;
import org.apache.derbyTesting.junit.BaseJDBCTestCase;
import org.apache.derbyTesting.junit.CleanDatabaseTestSetup;
import org.apache.derbyTesting.junit.JDBC;
import org.apache.derbyTesting.junit.SecurityManagerSetup;
-import org.apache.derbyTesting.junit.TestConfiguration;
-import org.apache.derbyTesting.junit.Utilities;
public class DBInJarTest extends BaseJDBCTestCase {
@@ -105,6 +102,97 @@ public class DBInJarTest extends BaseJDB
}
}
+ /**
+ * Test various queries that use a hash table that may be spilled to disk
+ * if it grows too big. Regression test case for DERBY-2354.
+ */
+ public void testSpillHashToDisk() throws SQLException {
+ createDerby2354Database();
+
+ Connection jarConn =
+ DriverManager.getConnection("jdbc:derby:jar:(d2354db.jar)d2354db");
+
+ Statement stmt = jarConn.createStatement();
+
+ // The following statement used to fail with "Feature not implemented"
+ // or "Container was opened in read-only mode" before DERBY-2354. It
+ // only fails if the hash table used for duplicate elimination spills
+ // to disk, which happens if the hash table gets bigger than 1% of the
+ // total amount of memory allocated to the JVM. This means it won't
+ // expose the bug if the JVM runs with very high memory settings (but
+ // it has been tested with 1 GB heap size and then it did spill to
+ // disk).
+ JDBC.assertDrainResults(
+ stmt.executeQuery("select distinct x from d2354"),
+ 40000);
+
+ // Hash joins have the same problem. Force the big table to be used as
+ // the inner table in the hash join.
+ JDBC.assertEmpty(stmt.executeQuery(
+ "select * from --DERBY-PROPERTIES joinOrder = FIXED\n" +
+ "sysibm.sysdummy1 t1(x),\n" +
+ "d2354 t2 --DERBY-PROPERTIES joinStrategy = HASH\n" +
+ "where t1.x = t2.x"));
+
+ // Scrollable result sets keep the rows they've visited in a hash
+ // table, so they may also need to store data on disk temporarily.
+ Statement scrollStmt = jarConn.createStatement(
+ ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+ JDBC.assertDrainResults(
+ scrollStmt.executeQuery("select * from d2354"),
+ 40000);
+
+ stmt.close();
+ scrollStmt.close();
+ jarConn.close();
+
+ // Cleanup. Shut down the database and delete it.
+ shutdownDB("jdbc:derby:jar:(d2354db.jar)d2354db;shutdown=true");
+ removeFiles(new String[] {
+ System.getProperty("derby.system.home") + "/d2354db.jar"
+ });
+ }
+
+ /**
+ * Create a database in a jar for use in {@code testSpillHashToDisk}.
+ */
+ private void createDerby2354Database() throws SQLException {
+ // First create an ordinary database with a table.
+ Connection conn =
+ DriverManager.getConnection("jdbc:derby:d2354db;create=true");
+ conn.setAutoCommit(false);
+ Statement s = conn.createStatement();
+ s.execute("create table d2354 (x varchar(100))");
+ s.close();
+
+ // Insert 40000 unique values into the table. The values should be
+ // unique so that they all occupy an entry in the hash table used by
+ // the DISTINCT query in the test, and thereby increase the likelihood
+ // of spilling to disk.
+ PreparedStatement insert =
+ conn.prepareStatement(
+ "insert into d2354 values ? || " +
+ "'some extra data to increase the size of the table'");
+ for (int i = 0; i < 40000; i++) {
+ insert.setInt(1, i);
+ insert.executeUpdate();
+ }
+ insert.close();
+
+ conn.commit();
+ conn.close();
+
+ // Shut down the database and archive it in a jar file.
+ shutdownDB("jdbc:derby:d2354db;shutdown=true");
+
+ createStatement().execute(
+ "CALL CREATEARCHIVE('d2354db.jar', 'd2354db', 'd2354db')");
+
+ // Clean up the original database directory. We don't need it anymore
+ // now that we have archived it in a jar file.
+ removeDirectory(
+ new File(System.getProperty("derby.system.home") + "/d2354db"));
+ }
protected static Test baseSuite(String name) {
TestSuite suite = new TestSuite(name);
Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java (original)
+++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java Tue Sep 25 01:35:57 2012
@@ -700,6 +700,15 @@ public abstract class BaseTestCase
{
DropDatabaseSetup.removeDirectory(dir);
}
+
+ /**
+ * Remove all the files in the list
+ * @param list the list contains all the files
+ */
+ public static void removeFiles(String[] list)
+ {
+ DropDatabaseSetup.removeFiles(list);
+ }
/**
* Fail; attaching an exception for more detail on cause.
Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java?rev=1389677&r1=1389676&r2=1389677&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java (original)
+++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java Tue Sep 25 01:35:57 2012
@@ -118,4 +118,19 @@ class DropDatabaseSetup extends BaseTest
BaseJDBCTestCase.assertDirectoryDeleted(dir);
}
+
+ /**
+ * Remove all the files in the list
+ * @param list the list of files that will be deleted
+ **/
+ static void removeFiles(String[] list) {
+ for (int i = 0; i < list.length; i++) {
+ try {
+ File dfile = new File(list[i].toString());
+ assertTrue(list[i].toString(), dfile.delete());
+ } catch (IllegalArgumentException e) {
+ fail("open file error");
+ }
+ }
+ }
}