You are viewing a plain text version of this content. The canonical link for it is here.
Posted to derby-commits@db.apache.org by mi...@apache.org on 2006/01/29 21:52:39 UTC
svn commit: r373356 - in /db/derby/code/trunk/java:
engine/org/apache/derby/impl/jdbc/
testing/org/apache/derbyTesting/functionTests/master/
testing/org/apache/derbyTesting/functionTests/tests/largedata/
Author: mikem
Date: Sun Jan 29 12:52:34 2006
New Revision: 373356
URL: http://svn.apache.org/viewcvs?rev=373356&view=rev
Log:
DERBY-599
committing on behalf of: Sunitha Kambhampati
Problem:
setBlob(i,blob) does not set the length of the stream in the blob and but instead passes a -1 for stream length.
During the normalization process, setBlob.normalize(DTD,DVD) calls SQLBlob.setWith.
The setWidth is called in order to compare the length of the blob value to the maximum width of the column and to throw a truncation error in case the value wont fit into the column. setWidth() calls SQLBinary.getLength(). If tvalue is a stream, the getLength() method checks for streamLength value and if the streamLength value is set to -1 (ie unknown) , it calls getBytes().length() which calls getValue() and this is where the entire stream is getting materialized.
This patch fixes DERBY-599 so using setBlob call, will not materialize the entire blob into memory.
- changes to setBlob to pass the length of the blob value instead of -1. The length of the blob value passed into setBlob can be obtained by calling Blob.length() which returns a long.
- move the -ve length check from setBinaryStream to setBinaryStreamInternal since setBlob will not pass a -1 for length.
- change setBinaryStreamInternal to take the length parameter as a long instead of int.
- Currently Derby allows max value of 2G-1 ( Max value of an int) for blobs. Add check to ensure that if a stream with a length > max value of int is passed, an error is thrown. Use an existing error message
'The resulting value is outside the range for the data type {0}'
Modified:
db/derby/code/trunk/java/engine/org/apache/derby/impl/jdbc/EmbedPreparedStatement.java
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/LobLimits.out
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/LobLimits.java
Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/jdbc/EmbedPreparedStatement.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/engine/org/apache/derby/impl/jdbc/EmbedPreparedStatement.java?rev=373356&r1=373355&r2=373356&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/jdbc/EmbedPreparedStatement.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/jdbc/EmbedPreparedStatement.java Sun Jan 29 12:52:34 2006
@@ -709,26 +709,40 @@
default:
throw dataTypeConversion(parameterIndex, "java.io.InputStream");
}
- if (length < 0) //we are doing the check here and not in setBinaryStreamInternal becuase setBlob needs to pass -1 for length.
- throw newSQLException(SQLState.NEGATIVE_STREAM_LENGTH);
setBinaryStreamInternal(parameterIndex, x, length);
}
protected void setBinaryStreamInternal(int parameterIndex, InputStream x,
- int length)
+ long length)
throws SQLException
{
+
+ if ( length < 0 )
+ throw newSQLException(SQLState.NEGATIVE_STREAM_LENGTH);
+
checkStatus();
int jdbcTypeId = getParameterJDBCType(parameterIndex);
if (x == null) {
setNull(parameterIndex, jdbcTypeId);
return;
}
+
+ // max number of bytes that can be set to be inserted
+ // in Derby is 2Gb-1 (ie Integer.MAX_VALUE).
+ // (e.g into a blob column).
+ // For now, we cast the length from long to int as a result.
+ // If we ever decide to increase these limits for lets say blobs,
+ // in that case the cast to int would not be appropriate.
+ if ( length > Integer.MAX_VALUE ) {
+ throw newSQLException(SQLState.LANG_OUTSIDE_RANGE_FOR_DATATYPE,
+ getEmbedParameterSetMetaData().getParameterTypeName(
+ parameterIndex));
+ }
- try {
+ try {
- getParms().getParameterForSet(parameterIndex - 1).setValue(new RawToBinaryFormatStream(x, length), length);
+ getParms().getParameterForSet(parameterIndex - 1).setValue(new RawToBinaryFormatStream(x, (int)length), (int)length);
} catch (StandardException t) {
throw EmbedResultSet.noStateChangeException(t);
@@ -1165,7 +1179,14 @@
if (x == null)
setNull(i, Types.BLOB);
else
- setBinaryStreamInternal(i, x.getBinaryStream(), -1);
+ {
+ // Note, x.length() needs to be called before retrieving the
+ // stream using x.getBinaryStream() because EmbedBlob.length()
+ // will read from the stream and drain some part of the stream
+ // Hence the need to declare this local variable - streamLength
+ long streamLength = x.length();
+ setBinaryStreamInternal(i, x.getBinaryStream(), streamLength);
+ }
}
/**
Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/LobLimits.out
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/LobLimits.out?rev=373356&r1=373355&r2=373356&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/LobLimits.out (original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/LobLimits.out Sun Jan 29 12:52:34 2006
@@ -92,6 +92,14 @@
Matched rows selected with blob of size(2147483647) =1
========================================
========================================
+START BlobTest #4 - select and then update blob of size= 2147483647 - Uses getBlob api
+Rows Updated = 1
+========================================
+========================================
+START BlobTest #4.1 - select and then insert blob of size= 2147483647 - Uses getBlob api to do select and setBlob for insert
+Rows Updated = 1
+========================================
+========================================
START BlobTest #5.1 insert Blob of size = 104857600
Rows inserted with blob of size (104857600) =1
========================================
@@ -99,5 +107,15 @@
START BlobTest #5.2 - SELECT BLOB of size = 104857600
Matched rows selected with blob of size(104857600) =1
========================================
+========================================
+START BlobTest #6 - select and then update blob of size= 104857600 - Uses getBlob and setBlob api
+Rows Updated = 1
+========================================
Rows deleted =1
-Rows deleted =2
+Rows deleted =3
+========================================
+START BlobTest #7 (setBlob with 4Gb blobinsertBlob of size = 4294967296
+ Rows inserted with blob of size (4294967296) =0
+========================================
+DERBY DOES NOT SUPPORT INSERT OF 4GB BLOB
+EXPECTED SQL Exception: (22003) The resulting value is outside the range for the data type BLOB.
Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/LobLimits.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/LobLimits.java?rev=373356&r1=373355&r2=373356&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/LobLimits.java (original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/LobLimits.java Sun Jan 29 12:52:34 2006
@@ -229,10 +229,13 @@
// now do a select of one of the 2gb rows and update another 2g row
// using the setBlob api, updated blob is of length 2gb
- // setBlob,materializes, so disable testfor now.
- // Bug entry -DERBY-599
- //selectUpdateBlob("BlobTest #4",conn,selectBlob,_2GB,0,1,1);
-
+ // Fix for Bug entry -DERBY-599[setBlob should not materialize blob
+ // into memory]
+ selectUpdateBlob("BlobTest #4",conn,selectBlob,_2GB,0,1,1);
+ // select row from blobtbl and then do insert into the blobtbl
+ // using setBlob
+ selectInsertBlob("BlobTest #4.1",conn,selectBlob,insertBlob,_2GB,0,3,1);
+
// Test - generate random data, write to a file, use it to insert
// data into blob and then read back and compare if all is ok
// currently in fvt ( derbyall), tests check for substrings etc and
@@ -255,10 +258,11 @@
_100MB, DATAFILE);
selectBlob2("BlobTest #5.2 ", conn, selectBlob2, _100MB, 0, 1,
DATAFILE);
+
+
// update the 2gb row in blobtbl with the 100mb data and compare if the update
- // went ok. wont work now, test disabled currently
- // till DERBY599 is fixed
- //selectUpdateBlob2("BlobTest #6",conn,selectBlob2,selectBlob,_100MB,0,1,1,DATAFILE);
+ // went ok.
+ selectUpdateBlob2("BlobTest #6",conn,selectBlob2,selectBlob,_100MB,0,1,1,DATAFILE);
deleteTable(conn, deleteBlob2, 1);
@@ -268,9 +272,23 @@
}
conn.commit();
+
+ deleteTable(conn, deleteBlob, 3);
- deleteTable(conn, deleteBlob, 2);
+ // Negative Test, use setBlob api to insert a 4GB blob.
+ long _4GB = 4*1024*1024*(1024L);
+ BlobImpl _4GbBlob = new BlobImpl(new RandomByteStream(new java.util.Random(),_4GB),_4GB);
+ try
+ {
+ insertBlob_SetBlob("BlobTest #7 (setBlob with 4Gb blob",conn,insertBlob,_4GbBlob,
+ _4GB,0,1,0);
+ }
+ catch(SQLException sqle)
+ {
+ System.out.println("DERBY DOES NOT SUPPORT INSERT OF 4GB BLOB ");
+ expectedException(sqle);
+ }
// ADD NEW TESTS HERE
}
@@ -457,6 +475,60 @@
}
/**
+ * insert blob, using a setBlob api.
+ * @param bloblen
+ * length of blob to insert
+ * @param blob
+ * blob to insert
+ * @param start
+ * start id value for insert
+ * @param rows
+ * insert rows number of rows
+ * @param expectedRows
+ * rows expected to be inserted
+ */
+ private static void insertBlob_SetBlob(String testId, Connection conn,
+ PreparedStatement ps, java.sql.Blob blob, long bloblen, int start,
+ int rows, int expectedRows) throws SQLException {
+ System.out.println("========================================");
+ System.out.println("START " + testId + "insertBlob of size = "
+ + bloblen);
+ long ST = 0;
+ if (trace)
+ ST = System.currentTimeMillis();
+ int count = 0;
+
+ try {
+
+ for (int i = start; i < start + rows; i++) {
+ ps.setInt(1, i);
+ ps.setInt(2, 0);
+ ps.setLong(3, bloblen);
+ ps.setBlob(4, blob);
+ count += ps.executeUpdate();
+ }
+ conn.commit();
+ if (trace) {
+ System.out.println("Insert Blob (" + bloblen + ")" + " rows= "
+ + count + " = "
+ + (long) (System.currentTimeMillis() - ST));
+
+ }
+ } catch (SQLException e) {
+ verifyTest(count, expectedRows,
+ " Rows inserted with blob of size (" + bloblen + ") =");
+ System.out.println("========================================");
+ throw e;
+ }
+
+ verifyTest(count, expectedRows,
+ " Rows inserted with blob of size (" + bloblen + ") =");
+ System.out.println("========================================");
+
+ }
+
+
+ /**
* select from blob table (BLOBTBL)
* @param bloblen select expects to retrieve a blob of this length
* @param id id of the row to retrieve
@@ -656,6 +728,63 @@
}
/**
+ * Basically this test will do an insert using setBlob api -
+ * select row from blobtbl and then insert a row in blobtbl
+ * and verify updated data in blobtbl
+ * @param ps select statement from which blob is retrieved
+ * @param bloblen updating value is of length bloblen
+ * @param id id of the row retrieved, for the update
+ * @param insertId id of the row that is inserted
+ * @param expectedRows to be updated
+ */
+ private static void selectInsertBlob(String testId, Connection conn,
+ PreparedStatement ps,PreparedStatement ins, int bloblen, int id, int insertId,
+ int expectedRows) throws Exception {
+ System.out.println("========================================");
+ System.out.println("START " + testId + " - select and then insert blob of size= "
+ + bloblen + " - Uses getBlob api to do select and setBlob for insert");
+
+ ResultSet rs = null;
+
+ ps.setInt(1, id);
+ rs = ps.executeQuery();
+ rs.next();
+ Blob value = rs.getBlob(1);
+ long l = value.length();
+ long dlen = rs.getLong(2);
+ if (dlen != l) {
+ System.out
+ .println("FAIL - MISMATCH LENGTHS GOT " + l + " expected "
+ + dlen + " for row in BLOBTBL with ID=" + id);
+ }
+
+ ins.setInt(1,insertId);
+ ins.setInt(2,0);
+ ins.setLong(3,l);
+ ins.setBlob(4,value);
+
+ System.out.println("Rows Updated = " + ins.executeUpdate());
+ conn.commit();
+
+ // now select and verify that update went through ok.
+ ps.setInt(1, insertId);
+ ResultSet rs2 = ps.executeQuery();
+ rs2.next();
+ Blob insertedValue = rs2.getBlob(1);
+
+ if(insertedValue.length() != l)
+ System.out.println("FAIL - Retrieving the updated blob length does not match "+
+ "expected length = "+l +" found = "+ insertedValue.length());
+
+ // close resultsets
+ conn.commit();
+ rs.close();
+ rs2.close();
+ System.out.println("========================================");
+ }
+
+
+ /**
* Basically this test will do an update using setBinaryStream api and verifies the
* updated data. select row from blobtbl2 and then update a row in blobtbl
* and verify updated data in blobtbl
@@ -1171,11 +1300,11 @@
* Class to generate random byte data
*/
class RandomByteStream extends java.io.InputStream {
- private int length;
+ private long length;
private java.util.Random dpr;
- RandomByteStream(java.util.Random dpr, int length) {
+ RandomByteStream(java.util.Random dpr, long length) {
this.length = length;
this.dpr = dpr;
@@ -1195,7 +1324,7 @@
return -1;
if (len > length)
- len = length;
+ len = (int)length;
for (int i = 0; i < len; i++) {
// chop off bits and return a +ve byte value.
@@ -1344,3 +1473,84 @@
}
}
+
+/***
+ * Class to simulate a 4Gb blob impl in order to test if Derby
+ * handles such large blobs correctly. The main methods here are
+ * only the length() and the getBinaryStream(). Rest are just
+ * placeholders/dummy methods in order to implement the java.sql.Blob
+ * interface
+ * ----
+ * Derby throws an error if the blob length exceeds the max range of
+ * int.
+ */
+class BlobImpl implements java.sql.Blob
+{
+ long length;
+ InputStream myStream;
+
+ public BlobImpl(InputStream is, long length)
+ {
+ this.myStream = is;
+ this.length = length;
+ }
+ public InputStream getBinaryStream()
+ throws SQLException
+ {
+ return myStream;
+ }
+
+ public byte[] getBytes()
+ throws SQLException
+ {
+ throw new SQLException("Not implemented");
+ }
+ public long length()
+ throws SQLException
+ {
+ return length;
+ }
+
+ public long position(Blob pattern,long start)
+ throws SQLException
+ {
+ throw new SQLException("Not implemented");
+ }
+
+ public long position(byte[] pattern,long start)
+ throws SQLException
+ {
+ throw new SQLException("Not implemented");
+ }
+ public OutputStream setBinaryStream(long pos)
+ throws SQLException
+
+ {
+ throw new SQLException("Not implemented");
+ }
+
+ public int setBytes(long pos,byte[] bytes)
+ throws SQLException
+ {
+ throw new SQLException("Not implemented");
+ }
+ public int setBytes(long pos,byte[] bytes,int offset,int len)
+ throws SQLException
+ {
+ throw new SQLException("Not implemented");
+ }
+
+ public void truncate(long len)
+ throws SQLException
+ {
+ throw new SQLException("Not implemented");
+ }
+
+ public byte[] getBytes(long pos, int length)
+ throws SQLException
+ {
+ throw new SQLException("Not implemented");
+ }
+
+}
+