You are viewing a plain text version of this content. The canonical link for it is here.
Posted to derby-dev@db.apache.org by "Grégoire Dubois (JIRA)" <de...@db.apache.org> on 2005/09/01 16:35:19 UTC
[jira] Created: (DERBY-550) BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
-----------------------------------------------------------------------------------------------
Key: DERBY-550
URL: http://issues.apache.org/jira/browse/DERBY-550
Project: Derby
Type: Bug
Components: JDBC
Versions: 10.1.1.0
Environment: Any environment.
Reporter: Grégoire Dubois
Using the org.apache.derby.jdbc.ClientDriver driver to access the
Derby database through network, the driver is writting all the file into memory (RAM) before sending
it to the database.
Writting small files (smaller than 5Mo) into the database works fine,
but it is impossible to write big files (40Mo for example, or more), without getting the
exception java.lang.OutOfMemoryError.
The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
Here follows some code that create a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
import NetNoLedge.Configuration.Configs;
import org.apache.derby.drda.NetworkServerControl;
import java.net.InetAddress;
import java.io.*;
import java.sql.*;
/**
*
* @author greg
*/
public class DerbyServer_JDBC_BLOB_test {
// The unique instance of DerbyServer in the application.
private static DerbyServer_JDBC_BLOB_test derbyServer;
private NetworkServerControl server;
private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
private static final String DERBY_DATABASE_NAME = "Test";
// ###############################################################
// ############### SET HERE THE EXISTING PATH YOU WANT ################
// ###############################################################
private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
// ###############################################################
// ###############################################################
private static int derbyPort = 9157;
private static String userName = "user";
private static String userPassword = "password";
// ###################################################################################
// ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
// ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
// ###################################################################################
private static final File FILE = new File("/home/greg/01.jpg");
// ###################################################################################
// ###################################################################################
/**
* <p>Used to test the server.
*/
public static void main(String args[]) {
try {
DerbyServer_JDBC_BLOB_test.launchServer();
DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
server.start();
System.out.println("Server started");
// After the server has been started, launch a first connection to the database to
// 1) Create the database if it doesn't exist already,
// 2) Create the tables if they don't exist already.
Class.forName(DERBY_JDBC_DRIVER).newInstance();
Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
Statement statement2;
// Create the table "file" if it doesn't already exist.
String [] tableNames={"file"};
boolean exist;
String currentTable;
ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
for (int i=0;i<tableNames.length;i++) {
exist=false;
while (result.next()){
if (tableNames[i].equalsIgnoreCase(result.getString(1)))
exist=true;
}
if (!exist) {
statement2 = connection.createStatement();
statement2.execute("CREATE TABLE file (" +
"file BLOB(2G) NOT NULL)");
connection.commit();
}
result.beforeFirst();
}
System.out.println("Table file created if not created already");
System.out.println("File insertion into BLOB");
FileInputStream inputStream = new FileInputStream(FILE);
PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
preparedStatement .execute();
connection.commit();
System.out.println("File inserted into BLOB");
}
catch (Exception e) {
e.printStackTrace();
}
}
/** Creates a new instance of MckoiServer
* Password is used at the database creation. It will be the database password once created.
*/
private DerbyServer_JDBC_BLOB_test() throws Exception {
System.setProperty("derby.system.home", DERBY_DBMS_PATH);
// Set the server to request an authentification.
System.setProperty("derby.authentication.provider", "BUILTIN");
System.setProperty("derby.connection.requireAuthentication", "true");
// Create a user that can connect to Derby.
System.setProperty("derby.user."+userName, userPassword);
// Set Derby to grant full access to the created user (to all the databases).
System.setProperty("derby.database.fullAccessUsers", userName);
//System.setProperty("derby.system.bootAll", "true");
// See if the 9157 port is already taken.
// Change it if necessary.
boolean isPortFree = false;
while ( !isPortFree ) {
try {
java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
serverTest.close();
serverTest = null;
isPortFree = true;
}
catch (Exception e) {
System.out.println("Port already in use : "+derbyPort);
derbyPort++;
System.out.println("Try with port "+derbyPort);
}
}
server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
}
public static void launchServer() throws Exception {
derbyServer = new DerbyServer_JDBC_BLOB_test();
}
public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
return derbyServer;
}
/**
* <p>Start the server.
*/
public void start() {
try {
server.start(null);
}
catch (Exception e) {
e.printStackTrace(System.err);
}
}
/**
* <p>Stop the server.
*/
public void stop() {
try {
server.shutdown();
}
catch (Exception e) {
e.printStackTrace(System.err);
}
}
}
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Closed: (DERBY-550) BLOB : java.lang.OutOfMemoryError with
network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Kristian Waagan (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/DERBY-550?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Kristian Waagan closed DERBY-550.
---------------------------------
Closing the issue.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: https://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Updated: (DERBY-550) BLOB : java.lang.OutOfMemoryError with
network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Andreas Korneliussen (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=all ]
Andreas Korneliussen updated DERBY-550:
---------------------------------------
Attachment: BlobOutOfMem.java
Attached is a repro.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC, Network Server
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Andreas Korneliussen <An...@Sun.COM>.
Kathey Marsden wrote:
> TomohitoNakayama wrote:
>
>>
>> I think it is needed to find where excessive expanding code exists ....
>>
>
>
> I agree and think a good step toward that is to make sure there are
> separate cases filed with reproducible cases for the issues that show
> different symptoms. I am getting a little confused. What different
> usage scenarios are at issue here?
>
> Kathey
>
Two scenarios:
1. Insert a BLOB using prepared statement. The network server will
expand the entire BLOB into memory when streaming it from the client,
and if the BLOB is big, you will get OutOfMemoryError
2. Select a BLOB using a result set. The network client will expand the
entire BLOB into memory when streaming it from the server. If the BLOB
is big, you will get OutOfMemoryError.
Both of these scenarios are related to the fact the the receiving end of
a stream, will expand all data of the stream into memory. I think, if
this cannot be fixed, at least it should be documented (maybe it already
is).
--Andreas
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Kathey Marsden <km...@sbcglobal.net>.
TomohitoNakayama wrote:
>
> I think it is needed to find where excessive expanding code exists ....
>
I agree and think a good step toward that is to make sure there are
separate cases filed with reproducible cases for the issues that show
different symptoms. I am getting a little confused. What different
usage scenarios are at issue here?
Kathey
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by TomohitoNakayama <to...@basil.ocn.ne.jp>.
Hello.
>Would it be possible, to create a server side InputStream which requests bytes from the client on demand, and that on the server side we could use ps.setBinaryStream(..) instead of ps.setBytes(..) ?
>The server would then stream the bytes when doing ps.execute(). Would this work if there are multiple parameters ?
>
It may be possible to implement the idea of "requests bytes from the
client on demand" almostly.
//Almostly means that I think there would be some restriction in server
side code, such as order of InputStream object to be read ....
However, I think priority is cutting down excessive expanding into
memory in the NetworkServer code.
Reading several comments from others, I feel some kind of unhealthy
code, which is worse than just expanding into memory at once.
// I think expanding whole image into memory is unavoidable in some
cases, and can be decided with careful consideration.
I think it is needed to find where excessive expanding code exists ....
Best regards.
Andreas Korneliussen (JIRA) wrote:
> [ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419715 ]
>
>Andreas Korneliussen commented on DERBY-550:
>--------------------------------------------
>
>I have done some tracing of this error on the server side which confirms that this is most likely a server side problem.
>When the DRDAConnThread reads parameter data, it will create a byte buffer and stream all the data of the blob into it. It happens when
>
>DRDAConnThread.readAndSetExtParam calls reader.getExtData(..).
>
>private void readAndSetExtParam( int i, DRDAStatement stmt,
> int drdaType, int extLen)
> throws DRDAProtocolException, SQLException
> {
> PreparedStatement ps = stmt.getPreparedStatement();
> drdaType = (drdaType & 0x000000ff); // need unsigned value
> boolean checkNullability = false;
> if (sqlamLevel >= MGRLVL_7 &&
> FdocaConstants.isNullable(drdaType))
> checkNullability = true;
>
> try {
> byte[] paramBytes = reader.getExtData(checkNullability); <---- here the entire LOB data gets populated in a byte array
>
>Later the paramer data is set using ps.setBytes(..).
>
>In addition, if running in debug mode, all the bytes are concatenated to the tracing string here:
>case DRDAConstants.DRDA_TYPE_LOBBYTES:
> case DRDAConstants.DRDA_TYPE_NLOBBYTES:
> if (SanityManager.DEBUG)
> trace("parameter value is: "+paramBytes); <---- more memory usage
> ps.setBytes(i+1, paramBytes);
> break;
>
>Would it be possible, to create a server side InputStream which requests bytes from the client on demand, and that on the server side we could use ps.setBinaryStream(..) instead of ps.setBytes(..) ?
>The server would then stream the bytes when doing ps.execute(). Would this work if there are multiple parameters ?
>
>
>
>>BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
>>-----------------------------------------------------------------------------------------------
>>
>> Key: DERBY-550
>> URL: http://issues.apache.org/jira/browse/DERBY-550
>> Project: Derby
>> Type: Bug
>>
>>
>
>
>
>> Components: JDBC
>> Versions: 10.1.1.0
>> Environment: Any environment.
>> Reporter: Grégoire Dubois
>> Assignee: Tomohito Nakayama
>>
>>
>
>
>
>>Using the org.apache.derby.jdbc.ClientDriver driver to access the
>>Derby database through network, the driver is writting all the file into memory (RAM) before sending
>>it to the database.
>>Writting small files (smaller than 5Mo) into the database works fine,
>>but it is impossible to write big files (40Mo for example, or more), without getting the
>>exception java.lang.OutOfMemoryError.
>>The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
>>Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
>>import NetNoLedge.Configuration.Configs;
>>import org.apache.derby.drda.NetworkServerControl;
>>import java.net.InetAddress;
>>import java.io.*;
>>import java.sql.*;
>>/**
>> *
>> * @author greg
>> */
>>public class DerbyServer_JDBC_BLOB_test {
>>
>> // The unique instance of DerbyServer in the application.
>> private static DerbyServer_JDBC_BLOB_test derbyServer;
>>
>> private NetworkServerControl server;
>>
>> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
>> private static final String DERBY_DATABASE_NAME = "Test";
>>
>> // ###############################################################
>> // ############### SET HERE THE EXISTING PATH YOU WANT ################
>> // ###############################################################
>> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
>> // ###############################################################
>> // ###############################################################
>>
>>
>> private static int derbyPort = 9157;
>> private static String userName = "user";
>> private static String userPassword = "password";
>>
>> // ###################################################################################
>> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
>> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
>> // ###################################################################################
>> private static final File FILE = new File("/home/greg/01.jpg");
>> // ###################################################################################
>> // ###################################################################################
>>
>> /**
>> * <p>Used to test the server.
>> */
>> public static void main(String args[]) {
>> try {
>> DerbyServer_JDBC_BLOB_test.launchServer();
>> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
>> server.start();
>> System.out.println("Server started");
>>
>> // After the server has been started, launch a first connection to the database to
>> // 1) Create the database if it doesn't exist already,
>> // 2) Create the tables if they don't exist already.
>> Class.forName(DERBY_JDBC_DRIVER).newInstance();
>> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
>> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>>
>> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
>> Statement statement2;
>> // Create the table "file" if it doesn't already exist.
>> String [] tableNames={"file"};
>> boolean exist;
>> String currentTable;
>> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
>> for (int i=0;i<tableNames.length;i++) {
>> exist=false;
>> while (result.next()){
>> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
>> exist=true;
>> }
>>
>> if (!exist) {
>> statement2 = connection.createStatement();
>> statement2.execute("CREATE TABLE file (" +
>> "file BLOB(2G) NOT NULL)");
>> connection.commit();
>> }
>> result.beforeFirst();
>> }
>> System.out.println("Table file created if not created already");
>>
>> System.out.println("File insertion into BLOB");
>> FileInputStream inputStream = new FileInputStream(FILE);
>> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
>> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
>> preparedStatement .execute();
>> connection.commit();
>> System.out.println("File inserted into BLOB");
>> }
>> catch (Exception e) {
>> e.printStackTrace();
>> }
>> }
>>
>> /** Creates a new instance of MckoiServer
>> * Password is used at the database creation. It will be the database password once created.
>> */
>> private DerbyServer_JDBC_BLOB_test() throws Exception {
>> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>>
>> // Set the server to request an authentification.
>> System.setProperty("derby.authentication.provider", "BUILTIN");
>> System.setProperty("derby.connection.requireAuthentication", "true");
>>
>> // Create a user that can connect to Derby.
>> System.setProperty("derby.user."+userName, userPassword);
>>
>> // Set Derby to grant full access to the created user (to all the databases).
>> System.setProperty("derby.database.fullAccessUsers", userName);
>>
>> //System.setProperty("derby.system.bootAll", "true");
>>
>> // See if the 9157 port is already taken.
>> // Change it if necessary.
>> boolean isPortFree = false;
>> while ( !isPortFree ) {
>> try {
>> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
>> serverTest.close();
>> serverTest = null;
>>
>> isPortFree = true;
>> }
>> catch (Exception e) {
>> System.out.println("Port already in use : "+derbyPort);
>> derbyPort++;
>> System.out.println("Try with port "+derbyPort);
>> }
>> }
>>
>> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
>> }
>>
>> public static void launchServer() throws Exception {
>> derbyServer = new DerbyServer_JDBC_BLOB_test();
>> }
>>
>> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
>> return derbyServer;
>> }
>>
>> /**
>> * <p>Start the server.
>> */
>> public void start() {
>> try {
>> server.start(null);
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>>
>> /**
>> * <p>Stop the server.
>> */
>> public void stop() {
>> try {
>> server.shutdown();
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>>}
>>
>>
>
>
>
--
/*
Tomohito Nakayama
tomonaka@basil.ocn.ne.jp
tomohito@rose.zero.ad.jp
tmnk@apache.org
Naka
http://www5.ocn.ne.jp/~tomohito/TopPage.html
*/
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Andreas Korneliussen (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419715 ]
Andreas Korneliussen commented on DERBY-550:
--------------------------------------------
I have done some tracing of this error on the server side which confirms that this is most likely a server side problem.
When the DRDAConnThread reads parameter data, it will create a byte buffer and stream all the data of the blob into it. It happens when
DRDAConnThread.readAndSetExtParam calls reader.getExtData(..).
private void readAndSetExtParam( int i, DRDAStatement stmt,
int drdaType, int extLen)
throws DRDAProtocolException, SQLException
{
PreparedStatement ps = stmt.getPreparedStatement();
drdaType = (drdaType & 0x000000ff); // need unsigned value
boolean checkNullability = false;
if (sqlamLevel >= MGRLVL_7 &&
FdocaConstants.isNullable(drdaType))
checkNullability = true;
try {
byte[] paramBytes = reader.getExtData(checkNullability); <---- here the entire LOB data gets populated in a byte array
Later the paramer data is set using ps.setBytes(..).
In addition, if running in debug mode, all the bytes are concatenated to the tracing string here:
case DRDAConstants.DRDA_TYPE_LOBBYTES:
case DRDAConstants.DRDA_TYPE_NLOBBYTES:
if (SanityManager.DEBUG)
trace("parameter value is: "+paramBytes); <---- more memory usage
ps.setBytes(i+1, paramBytes);
break;
Would it be possible, to create a server side InputStream which requests bytes from the client on demand, and that on the server side we could use ps.setBinaryStream(..) instead of ps.setBytes(..) ?
The server would then stream the bytes when doing ps.execute(). Would this work if there are multiple parameters ?
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12421281 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
Wait a moment ....
ByteArrayOutputStream have writeTo method....
I wonder wheter we can use this method and not use byte[] type data....
Well....
I will try.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assigned To: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators: http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] Updated: (DERBY-550) BLOB : java.lang.OutOfMemoryError with
network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=all ]
Tomohito Nakayama updated DERBY-550:
------------------------------------
Component: Network Server
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC, Network Server
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12421270 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
I tested with -Xmx192m -Xmx128m -Xmx64m also.
The results was as next
With -Xmx192m :
Exception occurred: java.lang.OutOfMemoryError (to be caught at: org.apache.derby.impl.store.raw.data.StoredPage.logRow(), line=4,209 bci=1,370)"thread=DRDAConnThread_3", org.apache.derby.impl.store.raw.data.MemByteHolder.<init>(), line=68 bci=26
68 this.curBuf = new byte[bufSize];
DRDAConnThread_3[1] where
[1] org.apache.derby.impl.store.raw.data.MemByteHolder.<init> (MemByteHolder.java:68)
[2] org.apache.derby.impl.store.raw.data.StoredPage.logColumn (StoredPage.java:6,351)
[3] org.apache.derby.impl.store.raw.data.StoredPage.logRow (StoredPage.java:3,965)
[4] org.apache.derby.impl.store.raw.data.InsertOperation.writeOptionalDataToBuffer (InsertOperation.java:369)
[5] org.apache.derby.impl.store.raw.data.InsertOperation.<init> (InsertOperation.java:114)
[6] org.apache.derby.impl.store.raw.data.LoggableActions.actionInsert (LoggableActions.java:138)
[7] org.apache.derby.impl.store.raw.data.BasePage.insertNoOverflow (BasePage.java:664)
[8] org.apache.derby.impl.store.raw.data.BasePage.insertAtSlot (BasePage.java:585)
[9] org.apache.derby.impl.store.raw.data.StoredPage.insertAtSlot (StoredPage.java:6,735)
[10] org.apache.derby.impl.store.raw.data.BasePage.insert (BasePage.java:691)
[11] org.apache.derby.impl.store.access.heap.HeapController.doInsert (HeapController.java:254)
[12] org.apache.derby.impl.store.access.heap.HeapController.insert (HeapController.java:573)
[13] org.apache.derby.impl.sql.execute.RowChangerImpl.insertRow (RowChangerImpl.java:447)
[14] org.apache.derby.impl.sql.execute.InsertResultSet.normalInsertCore (InsertResultSet.java:995)
[15] org.apache.derby.impl.sql.execute.InsertResultSet.open (InsertResultSet.java:522)
[16] org.apache.derby.impl.sql.GenericPreparedStatement.execute (GenericPreparedStatement.java:357)
[17] org.apache.derby.impl.jdbc.EmbedStatement.executeStatement (EmbedStatement.java:1,181)
[18] org.apache.derby.impl.jdbc.EmbedPreparedStatement.executeStatement (EmbedPreparedStatement.java:1,510)
[19] org.apache.derby.impl.jdbc.EmbedPreparedStatement.execute (EmbedPreparedStatement.java:1,188)
[20] org.apache.derby.impl.drda.DRDAStatement.execute (DRDAStatement.java:559)
[21] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTT (DRDAConnThread.java:3,655)
[22] org.apache.derby.impl.drda.DRDAConnThread.processCommands (DRDAConnThread.java:928)
[23] org.apache.derby.impl.drda.DRDAConnThread.run (DRDAConnThread.java:254)
DRDAConnThread_3[1] eval java.lang.Runtime.getRuntime().maxMemory()
java.lang.Runtime.getRuntime().maxMemory() = 199819264
With -Xmx128m :
Exception occurred: java.lang.OutOfMemoryError (uncaught)
Exception occurred: java.lang.OutOfMemoryError (uncaught)"thread=DRDAConnThread_3", java.io.ByteArrayOutputStream.toByteArray(), line=136 bci=4
DRDAConnThread_3[1] where
[1] java.io.ByteArrayOutputStream.toByteArray (ByteArrayOutputStream.java:136)
[2] org.apache.derby.impl.drda.DDMReader.getExtData (DDMReader.java:1,004)
[3] org.apache.derby.impl.drda.DDMReader.getExtData (DDMReader.java:944)
[4] org.apache.derby.impl.drda.DRDAConnThread.readAndSetExtParam (DRDAConnThread.java:4,355)
[5] org.apache.derby.impl.drda.DRDAConnThread.readAndSetAllExtParams (DRDAConnThread.java:4,320)
[6] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTTobjects (DRDAConnThread.java:3,811)
[7] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTT (DRDAConnThread.java:3,640)
[8] org.apache.derby.impl.drda.DRDAConnThread.processCommands (DRDAConnThread.java:928)
[9] org.apache.derby.impl.drda.DRDAConnThread.run (DRDAConnThread.java:254)
DRDAConnThread_3[1] eval java.lang.Runtime.getRuntime().maxMemory()
java.lang.Runtime.getRuntime().maxMemory() = 133234688
With -Xmx64m:
Exception occurred: java.lang.OutOfMemoryError (uncaught)
Exception occurred: java.lang.OutOfMemoryError (uncaught)"thread=DRDAConnThread_3", java.io.ByteArrayOutputStream.<init>(), line=59 bci=37
DRDAConnThread_3[1] where
[1] java.io.ByteArrayOutputStream.<init> (ByteArrayOutputStream.java:59)
[2] org.apache.derby.impl.drda.DDMReader.getExtData (DDMReader.java:958)
[3] org.apache.derby.impl.drda.DDMReader.getExtData (DDMReader.java:944)
[4] org.apache.derby.impl.drda.DRDAConnThread.readAndSetExtParam (DRDAConnThread.java:4,355)
[5] org.apache.derby.impl.drda.DRDAConnThread.readAndSetAllExtParams (DRDAConnThread.java:4,320)
[6] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTTobjects (DRDAConnThread.java:3,811)
[7] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTT (DRDAConnThread.java:3,640)
[8] org.apache.derby.impl.drda.DRDAConnThread.processCommands (DRDAConnThread.java:928)
[9] org.apache.derby.impl.drda.DRDAConnThread.run (DRDAConnThread.java:254)
DRDAConnThread_3[1] eval java.lang.Runtime.getRuntime().maxMemory()
java.lang.Runtime.getRuntime().maxMemory() = 66650112
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assigned To: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators: http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Bryan Pendleton (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419634 ]
Bryan Pendleton commented on DERBY-550:
---------------------------------------
I think the problem is actually on the server side, not on the client side. When I tried writing a test program to investigate this (see DERBY-1472 for the program), I discovered that the client does get an OutOfMemoryError, but this exception appears to be actually occuring on the server side, not on the client side. In my experimentation, client-side memory usage was low. But there is clearly a server-side memory problem to be solved.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by TomohitoNakayama <to...@basil.ocn.ne.jp>.
Hello Andreas.
>
>> //Even if OutOfMemoryError happens, it would be not a problem in an
>> environment with very small amount of memory ....
>
>
> I am not sure I understand why it would not be a problem in an
> environment with very small amount of memory. The side-effects of
> outofmemoryerror would be the same:
>
> * they may occur anywhere in the user application
> * client would hang if the DRDAConnThread stops
Here, I think there exists difference of opinion between us.
I think expanding object into memory is not completely wrong,
if there are some reason.
// And I think there may exist the reason in the case of from server to
client,
// such as that program works in both of client and server separatedly,
though not sure yet....
//// I suspect whether it is practical to stream fron server to client
on demand.
//// Now I'm reading spec of DRDA and try to find answer, though not
found it yet ...
Then, in this my opinion, OutOfMemoryError in an environment with very
small memory is not always problem.
It may be just a natural consequence.
I think you don't agree this my opinion,
it is natural because we are not copied person.
However, current implementation of network server side seems to expand
object into memory more than once.
I think this behavior is very problematic,
because we can escape this problem just sharing memory where object was
expanded to in the network server.
I think this behavior should be fixed.
I think you would agree my latter opinion.
Best regards.
Andreas Korneliussen wrote:
> TomohitoNakayama wrote:
>
>> Hello.
>>
>> Regretfully I'm not sure whether I can fix this issue until 10.2 release
>> because it is not clear where memory is used wastefully and
>> how we can judge whether we could solve this issue.
>>
>
> Hi.
> I am not sure I could fix this for 10.2 either, therefore I suggested
> the other approach, which I would hope gives a less severe problem for
> the user to handle.
>
> On the client side the problem occurs when
> NetStatementReply.copyEXTDTA(..) streams all the data over from the
> server and puts it into a byte array which is put into NetCursor. This
> array is later used to create LOBs. To fix it on the client side, the
> LOB implemtation classes would need to be reimplemented, so that they
> will do the streaming.
>
> On the server side, the problem occurs in
> DRDAConnThread.readAndSetExtParam calls reader.getExtData(..). See my
> previous comments, where I suggested a solution.
>
> It may of course be other problems also, however this is what I found
> when tracing this problem with a debugger.
>
>> //Even if OutOfMemoryError happens, it would be not a problem in an
>> environment with very small amount of memory ....
>
>
> I am not sure I understand why it would not be a problem in an
> environment with very small amount of memory. The side-effects of
> outofmemoryerror would be the same:
>
> * they may occur anywhere in the user application
> * client would hang if the DRDAConnThread stops
>
>
>> //I think criterion to judge resolve of this issue is needed ...
>>
>> However, I think throw Exception judging from amount of spare memory
>> would be too cunning behavor ......
>
>
> I think it would be good to use a conservative approach to avoid
> consuming all heap space in the VM.
>
>>
>> Now ... I'm in dilemma.....
>> At least, I won't veto it.
>>
>
> Well, I am not sure I will do it then, the best is to fix the real
> problem.
>
> Regards
>
> Andreas
>
>> Best regards.
>>
>>
>> Andreas Korneliussen (JIRA) wrote:
>>
>>> [
>>> http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420591
>>> ]
>>> Andreas Korneliussen commented on DERBY-550:
>>> --------------------------------------------
>>>
>>> Unless the streaming could be fixed for 10.2 so that we avoid
>>> OutOfMemoryError on the receiver side, I would propose the following:
>>>
>>> We know the size of the LOB, and can check if it can go into memory
>>> (using the Runtime class). If it cannot go into memory, we can throw
>>> an SQLException, instead of consuming all memory in the VM until we
>>> get OutOfMemoryError.
>>>
>>> By using this approach, we achieve the following:
>>> * Side-effects on other connections in the VM: Although it is the
>>> LOB which is taking almost all the memory in the VM, the
>>> OutOfMemoryError may be thrown in another thread in the VM, causing
>>> side-effects on other connections or on the application itself.
>>> * Currently, if the Network server goes out of memory when streaming
>>> data, the DRDAConnThread will stop. This causes hangs in the user
>>> applications.
>>> If the streaming is fixed, there is not need to do this. Does anyone
>>> plan to fix the streaming issues for 10.2 ? If not, I will make a
>>> couple of JIRA issues to do the work of avoiding OutOfMemoryError by
>>> checking size before allocating the byte arrays.
>>>
>>>
>>>
>>>
>>>> BLOB : java.lang.OutOfMemoryError with network JDBC driver
>>>> (org.apache.derby.jdbc.ClientDriver)
>>>> -----------------------------------------------------------------------------------------------
>>>>
>>>>
>>>> Key: DERBY-550
>>>> URL: http://issues.apache.org/jira/browse/DERBY-550
>>>> Project: Derby
>>>> Type: Bug
>>>>
>>>
>>>
>>>
>>>
>>>> Components: JDBC, Network Server
>>>> Versions: 10.1.1.0
>>>> Environment: Any environment.
>>>> Reporter: Grégoire Dubois
>>>> Assignee: Tomohito Nakayama
>>>> Attachments: BlobOutOfMem.java
>>>>
>>>> Using the org.apache.derby.jdbc.ClientDriver driver to access the
>>>> Derby database through network, the driver is writting all the file
>>>> into memory (RAM) before sending
>>>> it to the database.
>>>> Writting small files (smaller than 5Mo) into the database works fine,
>>>> but it is impossible to write big files (40Mo for example, or
>>>> more), without getting the
>>>> exception java.lang.OutOfMemoryError.
>>>> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
>>>> Here follows some code that creates a database, a table, and trys
>>>> to write a BLOB. 2 parameters are to be changed for the code to
>>>> work for you : DERBY_DBMS_PATH and FILE
>>>> import NetNoLedge.Configuration.Configs;
>>>> import org.apache.derby.drda.NetworkServerControl;
>>>> import java.net.InetAddress;
>>>> import java.io.*;
>>>> import java.sql.*;
>>>> /**
>>>> *
>>>> * @author greg
>>>> */
>>>> public class DerbyServer_JDBC_BLOB_test {
>>>> // The unique instance of DerbyServer in the application.
>>>> private static DerbyServer_JDBC_BLOB_test derbyServer;
>>>> private NetworkServerControl server;
>>>> private static final String DERBY_JDBC_DRIVER =
>>>> "org.apache.derby.jdbc.ClientDriver";
>>>> private static final String DERBY_DATABASE_NAME = "Test";
>>>> //
>>>> ###############################################################
>>>> // ############### SET HERE THE EXISTING PATH YOU WANT
>>>> ################
>>>> // ###############################################################
>>>> private static final String DERBY_DBMS_PATH =
>>>> "/home/greg/DatabaseTest";
>>>> // ###############################################################
>>>> // ###############################################################
>>>> private static int derbyPort = 9157;
>>>> private static String userName = "user";
>>>> private static String userPassword = "password";
>>>> //
>>>> ###################################################################################
>>>>
>>>> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO
>>>> WRITE INTO THE DATABASE ###########
>>>> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR
>>>> BIGGER FILE #########################
>>>> //
>>>> ###################################################################################
>>>>
>>>> private static final File FILE = new File("/home/greg/01.jpg");
>>>> //
>>>> ###################################################################################
>>>>
>>>> //
>>>> ###################################################################################
>>>>
>>>> /**
>>>> * <p>Used to test the server.
>>>> */
>>>> public static void main(String args[]) {
>>>> try {
>>>> DerbyServer_JDBC_BLOB_test.launchServer();
>>>> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
>>>> server.start();
>>>> System.out.println("Server started");
>>>> // After the server has been started, launch a
>>>> first connection to the database to
>>>> // 1) Create the database if it doesn't exist already,
>>>> // 2) Create the tables if they don't exist
>>>> already.
>>>> Class.forName(DERBY_JDBC_DRIVER).newInstance();
>>>> Connection connection = DriverManager.getConnection
>>>> ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true",
>>>> userName, userPassword);
>>>> System.out.println("Network JDBC connection to Derby
>>>> succeded. Database created if not created already.");
>>>> Statement statement =
>>>> connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
>>>> ResultSet.CONCUR_READ_ONLY);
>>>> Statement statement2;
>>>> // Create the table "file" if it doesn't already exist.
>>>> String [] tableNames={"file"};
>>>> boolean exist;
>>>> String currentTable;
>>>> ResultSet result = statement.executeQuery("SELECT
>>>> TABLENAME FROM SYS.SYSTABLES");
>>>> for (int i=0;i<tableNames.length;i++) {
>>>> exist=false;
>>>> while (result.next()){
>>>> if
>>>> (tableNames[i].equalsIgnoreCase(result.getString(1)))
>>>> exist=true;
>>>> }
>>>> if (!exist) {
>>>> statement2 = connection.createStatement();
>>>> statement2.execute("CREATE TABLE file (" +
>>>> "file BLOB(2G) NOT NULL)");
>>>> connection.commit();
>>>> }
>>>> result.beforeFirst();
>>>> }
>>>> System.out.println("Table file created if not created
>>>> already");
>>>> System.out.println("File insertion into BLOB");
>>>> FileInputStream inputStream = new FileInputStream(FILE);
>>>> PreparedStatement preparedStatement =
>>>> connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
>>>> preparedStatement.setBinaryStream(1,inputStream,(int)
>>>> FILE.length());
>>>> preparedStatement .execute();
>>>> connection.commit();
>>>> System.out.println("File inserted into BLOB");
>>>> }
>>>> catch (Exception e) {
>>>> e.printStackTrace();
>>>> }
>>>> }
>>>> /** Creates a new instance of MckoiServer
>>>> * Password is used at the database creation. It will be the
>>>> database password once created.
>>>> */ private DerbyServer_JDBC_BLOB_test() throws Exception {
>>>> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>>>> // Set the server to request an authentification.
>>>> System.setProperty("derby.authentication.provider", "BUILTIN");
>>>> System.setProperty("derby.connection.requireAuthentication",
>>>> "true");
>>>> // Create a user that can connect to Derby.
>>>> System.setProperty("derby.user."+userName, userPassword);
>>>> // Set Derby to grant full access to the created user
>>>> (to all the databases).
>>>> System.setProperty("derby.database.fullAccessUsers", userName);
>>>> //System.setProperty("derby.system.bootAll", "true");
>>>> // See if the 9157 port is already taken.
>>>> // Change it if necessary.
>>>> boolean isPortFree = false;
>>>> while ( !isPortFree ) {
>>>> try {
>>>> java.net.ServerSocket serverTest = new
>>>> java.net.ServerSocket(derbyPort);
>>>> serverTest.close();
>>>> serverTest = null;
>>>> isPortFree = true;
>>>> }
>>>> catch (Exception e) {
>>>> System.out.println("Port already in use : "+derbyPort);
>>>> derbyPort++;
>>>> System.out.println("Try with port "+derbyPort);
>>>> }
>>>> } server = new
>>>> NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
>>>> }
>>>> public static void launchServer() throws Exception {
>>>> derbyServer = new DerbyServer_JDBC_BLOB_test();
>>>> }
>>>> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
>>>> return derbyServer;
>>>> }
>>>> /**
>>>> * <p>Start the server.
>>>> */
>>>> public void start() {
>>>> try {
>>>> server.start(null);
>>>> }
>>>> catch (Exception e) {
>>>> e.printStackTrace(System.err);
>>>> }
>>>> }
>>>> /**
>>>> * <p>Stop the server.
>>>> */
>>>> public void stop() {
>>>> try {
>>>> server.shutdown();
>>>> }
>>>> catch (Exception e) {
>>>> e.printStackTrace(System.err);
>>>> }
>>>> }
>>>> }
>>>>
>>>
>>>
>>>
>>>
>>
>
>
>
--
/*
Tomohito Nakayama
tomonaka@basil.ocn.ne.jp
tomohito@rose.zero.ad.jp
tmnk@apache.org
Naka
http://www5.ocn.ne.jp/~tomohito/TopPage.html
*/
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Andreas Korneliussen <An...@Sun.COM>.
TomohitoNakayama wrote:
> Hello.
>
> Regretfully I'm not sure whether I can fix this issue until 10.2 release
> because it is not clear where memory is used wastefully and
> how we can judge whether we could solve this issue.
>
Hi.
I am not sure I could fix this for 10.2 either, therefore I suggested
the other approach, which I would hope gives a less severe problem for
the user to handle.
On the client side the problem occurs when
NetStatementReply.copyEXTDTA(..) streams all the data over from the
server and puts it into a byte array which is put into NetCursor. This
array is later used to create LOBs. To fix it on the client side, the
LOB implemtation classes would need to be reimplemented, so that they
will do the streaming.
On the server side, the problem occurs in
DRDAConnThread.readAndSetExtParam calls reader.getExtData(..). See my
previous comments, where I suggested a solution.
It may of course be other problems also, however this is what I found
when tracing this problem with a debugger.
> //Even if OutOfMemoryError happens, it would be not a problem in an
> environment with very small amount of memory ....
I am not sure I understand why it would not be a problem in an
environment with very small amount of memory. The side-effects of
outofmemoryerror would be the same:
* they may occur anywhere in the user application
* client would hang if the DRDAConnThread stops
> //I think criterion to judge resolve of this issue is needed ...
>
> However, I think throw Exception judging from amount of spare memory
> would be too cunning behavor ......
I think it would be good to use a conservative approach to avoid
consuming all heap space in the VM.
>
> Now ... I'm in dilemma.....
> At least, I won't veto it.
>
Well, I am not sure I will do it then, the best is to fix the real problem.
Regards
Andreas
> Best regards.
>
>
> Andreas Korneliussen (JIRA) wrote:
>
>> [
>> http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420591
>> ]
>> Andreas Korneliussen commented on DERBY-550:
>> --------------------------------------------
>>
>> Unless the streaming could be fixed for 10.2 so that we avoid
>> OutOfMemoryError on the receiver side, I would propose the following:
>>
>> We know the size of the LOB, and can check if it can go into memory
>> (using the Runtime class). If it cannot go into memory, we can throw
>> an SQLException, instead of consuming all memory in the VM until we
>> get OutOfMemoryError.
>>
>> By using this approach, we achieve the following:
>> * Side-effects on other connections in the VM: Although it is the LOB
>> which is taking almost all the memory in the VM, the OutOfMemoryError
>> may be thrown in another thread in the VM, causing side-effects on
>> other connections or on the application itself.
>> * Currently, if the Network server goes out of memory when streaming
>> data, the DRDAConnThread will stop. This causes hangs in the user
>> applications.
>> If the streaming is fixed, there is not need to do this. Does anyone
>> plan to fix the streaming issues for 10.2 ? If not, I will make a
>> couple of JIRA issues to do the work of avoiding OutOfMemoryError by
>> checking size before allocating the byte arrays.
>>
>>
>>
>>
>>> BLOB : java.lang.OutOfMemoryError with network JDBC driver
>>> (org.apache.derby.jdbc.ClientDriver)
>>> -----------------------------------------------------------------------------------------------
>>>
>>>
>>> Key: DERBY-550
>>> URL: http://issues.apache.org/jira/browse/DERBY-550
>>> Project: Derby
>>> Type: Bug
>>>
>>
>>
>>
>>> Components: JDBC, Network Server
>>> Versions: 10.1.1.0
>>> Environment: Any environment.
>>> Reporter: Grégoire Dubois
>>> Assignee: Tomohito Nakayama
>>> Attachments: BlobOutOfMem.java
>>>
>>> Using the org.apache.derby.jdbc.ClientDriver driver to access the
>>> Derby database through network, the driver is writting all the file
>>> into memory (RAM) before sending
>>> it to the database.
>>> Writting small files (smaller than 5Mo) into the database works fine,
>>> but it is impossible to write big files (40Mo for example, or more),
>>> without getting the
>>> exception java.lang.OutOfMemoryError.
>>> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
>>> Here follows some code that creates a database, a table, and trys to
>>> write a BLOB. 2 parameters are to be changed for the code to work for
>>> you : DERBY_DBMS_PATH and FILE
>>> import NetNoLedge.Configuration.Configs;
>>> import org.apache.derby.drda.NetworkServerControl;
>>> import java.net.InetAddress;
>>> import java.io.*;
>>> import java.sql.*;
>>> /**
>>> *
>>> * @author greg
>>> */
>>> public class DerbyServer_JDBC_BLOB_test {
>>> // The unique instance of DerbyServer in the application.
>>> private static DerbyServer_JDBC_BLOB_test derbyServer;
>>> private NetworkServerControl server;
>>> private static final String DERBY_JDBC_DRIVER =
>>> "org.apache.derby.jdbc.ClientDriver";
>>> private static final String DERBY_DATABASE_NAME = "Test";
>>> // ###############################################################
>>> // ############### SET HERE THE EXISTING PATH YOU WANT
>>> ################
>>> // ###############################################################
>>> private static final String DERBY_DBMS_PATH =
>>> "/home/greg/DatabaseTest";
>>> // ###############################################################
>>> // ###############################################################
>>> private static int derbyPort = 9157;
>>> private static String userName = "user";
>>> private static String userPassword = "password";
>>> //
>>> ###################################################################################
>>>
>>> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO
>>> WRITE INTO THE DATABASE ###########
>>> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER
>>> FILE #########################
>>> //
>>> ###################################################################################
>>>
>>> private static final File FILE = new File("/home/greg/01.jpg");
>>> //
>>> ###################################################################################
>>>
>>> //
>>> ###################################################################################
>>>
>>> /**
>>> * <p>Used to test the server.
>>> */
>>> public static void main(String args[]) {
>>> try {
>>> DerbyServer_JDBC_BLOB_test.launchServer();
>>> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
>>> server.start();
>>> System.out.println("Server started");
>>> // After the server has been started, launch a
>>> first connection to the database to
>>> // 1) Create the database if it doesn't exist already,
>>> // 2) Create the tables if they don't exist
>>> already.
>>> Class.forName(DERBY_JDBC_DRIVER).newInstance();
>>> Connection connection = DriverManager.getConnection
>>> ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true",
>>> userName, userPassword);
>>> System.out.println("Network JDBC connection to Derby
>>> succeded. Database created if not created already.");
>>> Statement statement =
>>> connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
>>> ResultSet.CONCUR_READ_ONLY);
>>> Statement statement2;
>>> // Create the table "file" if it doesn't already exist.
>>> String [] tableNames={"file"};
>>> boolean exist;
>>> String currentTable;
>>> ResultSet result = statement.executeQuery("SELECT
>>> TABLENAME FROM SYS.SYSTABLES");
>>> for (int i=0;i<tableNames.length;i++) {
>>> exist=false;
>>> while (result.next()){
>>> if
>>> (tableNames[i].equalsIgnoreCase(result.getString(1)))
>>> exist=true;
>>> }
>>> if (!exist) {
>>> statement2 = connection.createStatement();
>>> statement2.execute("CREATE TABLE file (" +
>>> "file BLOB(2G) NOT NULL)");
>>> connection.commit();
>>> }
>>> result.beforeFirst();
>>> }
>>> System.out.println("Table file created if not created
>>> already");
>>> System.out.println("File insertion into BLOB");
>>> FileInputStream inputStream = new FileInputStream(FILE);
>>> PreparedStatement preparedStatement =
>>> connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
>>> preparedStatement.setBinaryStream(1,inputStream,(int)
>>> FILE.length());
>>> preparedStatement .execute();
>>> connection.commit();
>>> System.out.println("File inserted into BLOB");
>>> }
>>> catch (Exception e) {
>>> e.printStackTrace();
>>> }
>>> }
>>> /** Creates a new instance of MckoiServer
>>> * Password is used at the database creation. It will be the
>>> database password once created.
>>> */ private DerbyServer_JDBC_BLOB_test() throws Exception {
>>> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>>> // Set the server to request an authentification.
>>> System.setProperty("derby.authentication.provider", "BUILTIN");
>>> System.setProperty("derby.connection.requireAuthentication",
>>> "true");
>>> // Create a user that can connect to Derby.
>>> System.setProperty("derby.user."+userName, userPassword);
>>> // Set Derby to grant full access to the created user
>>> (to all the databases).
>>> System.setProperty("derby.database.fullAccessUsers", userName);
>>> //System.setProperty("derby.system.bootAll", "true");
>>> // See if the 9157 port is already taken.
>>> // Change it if necessary.
>>> boolean isPortFree = false;
>>> while ( !isPortFree ) {
>>> try {
>>> java.net.ServerSocket serverTest = new
>>> java.net.ServerSocket(derbyPort);
>>> serverTest.close();
>>> serverTest = null;
>>> isPortFree = true;
>>> }
>>> catch (Exception e) {
>>> System.out.println("Port already in use : "+derbyPort);
>>> derbyPort++;
>>> System.out.println("Try with port "+derbyPort);
>>> }
>>> } server = new
>>> NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
>>> }
>>> public static void launchServer() throws Exception {
>>> derbyServer = new DerbyServer_JDBC_BLOB_test();
>>> }
>>> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
>>> return derbyServer;
>>> }
>>> /**
>>> * <p>Start the server.
>>> */
>>> public void start() {
>>> try {
>>> server.start(null);
>>> }
>>> catch (Exception e) {
>>> e.printStackTrace(System.err);
>>> }
>>> }
>>> /**
>>> * <p>Stop the server.
>>> */
>>> public void stop() {
>>> try {
>>> server.shutdown();
>>> }
>>> catch (Exception e) {
>>> e.printStackTrace(System.err);
>>> }
>>> }
>>> }
>>>
>>
>>
>>
>
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by TomohitoNakayama <to...@basil.ocn.ne.jp>.
Hello.
Regretfully I'm not sure whether I can fix this issue until 10.2 release
because it is not clear where memory is used wastefully and
how we can judge whether we could solve this issue.
//Even if OutOfMemoryError happens, it would be not a problem in an
environment with very small amount of memory ....
//I think criterion to judge resolve of this issue is needed ...
However, I think throw Exception judging from amount of spare memory
would be too cunning behavor ......
Now ... I'm in dilemma.....
At least, I won't veto it.
Best regards.
Andreas Korneliussen (JIRA) wrote:
> [ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420591 ]
>
>Andreas Korneliussen commented on DERBY-550:
>--------------------------------------------
>
>Unless the streaming could be fixed for 10.2 so that we avoid OutOfMemoryError on the receiver side, I would propose the following:
>
>We know the size of the LOB, and can check if it can go into memory (using the Runtime class). If it cannot go into memory, we can throw an SQLException, instead of consuming all memory in the VM until we get OutOfMemoryError.
>
>By using this approach, we achieve the following:
>* Side-effects on other connections in the VM: Although it is the LOB which is taking almost all the memory in the VM, the OutOfMemoryError may be thrown in another thread in the VM, causing side-effects on other connections or on the application itself.
>* Currently, if the Network server goes out of memory when streaming data, the DRDAConnThread will stop. This causes hangs in the user applications.
>
>If the streaming is fixed, there is not need to do this. Does anyone plan to fix the streaming issues for 10.2 ? If not, I will make a couple of JIRA issues to do the work of avoiding OutOfMemoryError by checking size before allocating the byte arrays.
>
>
>
>
>>BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
>>-----------------------------------------------------------------------------------------------
>>
>> Key: DERBY-550
>> URL: http://issues.apache.org/jira/browse/DERBY-550
>> Project: Derby
>> Type: Bug
>>
>>
>
>
>
>> Components: JDBC, Network Server
>> Versions: 10.1.1.0
>> Environment: Any environment.
>> Reporter: Grégoire Dubois
>> Assignee: Tomohito Nakayama
>> Attachments: BlobOutOfMem.java
>>
>>Using the org.apache.derby.jdbc.ClientDriver driver to access the
>>Derby database through network, the driver is writting all the file into memory (RAM) before sending
>>it to the database.
>>Writting small files (smaller than 5Mo) into the database works fine,
>>but it is impossible to write big files (40Mo for example, or more), without getting the
>>exception java.lang.OutOfMemoryError.
>>The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
>>Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
>>import NetNoLedge.Configuration.Configs;
>>import org.apache.derby.drda.NetworkServerControl;
>>import java.net.InetAddress;
>>import java.io.*;
>>import java.sql.*;
>>/**
>> *
>> * @author greg
>> */
>>public class DerbyServer_JDBC_BLOB_test {
>>
>> // The unique instance of DerbyServer in the application.
>> private static DerbyServer_JDBC_BLOB_test derbyServer;
>>
>> private NetworkServerControl server;
>>
>> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
>> private static final String DERBY_DATABASE_NAME = "Test";
>>
>> // ###############################################################
>> // ############### SET HERE THE EXISTING PATH YOU WANT ################
>> // ###############################################################
>> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
>> // ###############################################################
>> // ###############################################################
>>
>>
>> private static int derbyPort = 9157;
>> private static String userName = "user";
>> private static String userPassword = "password";
>>
>> // ###################################################################################
>> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
>> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
>> // ###################################################################################
>> private static final File FILE = new File("/home/greg/01.jpg");
>> // ###################################################################################
>> // ###################################################################################
>>
>> /**
>> * <p>Used to test the server.
>> */
>> public static void main(String args[]) {
>> try {
>> DerbyServer_JDBC_BLOB_test.launchServer();
>> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
>> server.start();
>> System.out.println("Server started");
>>
>> // After the server has been started, launch a first connection to the database to
>> // 1) Create the database if it doesn't exist already,
>> // 2) Create the tables if they don't exist already.
>> Class.forName(DERBY_JDBC_DRIVER).newInstance();
>> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
>> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>>
>> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
>> Statement statement2;
>> // Create the table "file" if it doesn't already exist.
>> String [] tableNames={"file"};
>> boolean exist;
>> String currentTable;
>> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
>> for (int i=0;i<tableNames.length;i++) {
>> exist=false;
>> while (result.next()){
>> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
>> exist=true;
>> }
>>
>> if (!exist) {
>> statement2 = connection.createStatement();
>> statement2.execute("CREATE TABLE file (" +
>> "file BLOB(2G) NOT NULL)");
>> connection.commit();
>> }
>> result.beforeFirst();
>> }
>> System.out.println("Table file created if not created already");
>>
>> System.out.println("File insertion into BLOB");
>> FileInputStream inputStream = new FileInputStream(FILE);
>> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
>> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
>> preparedStatement .execute();
>> connection.commit();
>> System.out.println("File inserted into BLOB");
>> }
>> catch (Exception e) {
>> e.printStackTrace();
>> }
>> }
>>
>> /** Creates a new instance of MckoiServer
>> * Password is used at the database creation. It will be the database password once created.
>> */
>> private DerbyServer_JDBC_BLOB_test() throws Exception {
>> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>>
>> // Set the server to request an authentification.
>> System.setProperty("derby.authentication.provider", "BUILTIN");
>> System.setProperty("derby.connection.requireAuthentication", "true");
>>
>> // Create a user that can connect to Derby.
>> System.setProperty("derby.user."+userName, userPassword);
>>
>> // Set Derby to grant full access to the created user (to all the databases).
>> System.setProperty("derby.database.fullAccessUsers", userName);
>>
>> //System.setProperty("derby.system.bootAll", "true");
>>
>> // See if the 9157 port is already taken.
>> // Change it if necessary.
>> boolean isPortFree = false;
>> while ( !isPortFree ) {
>> try {
>> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
>> serverTest.close();
>> serverTest = null;
>>
>> isPortFree = true;
>> }
>> catch (Exception e) {
>> System.out.println("Port already in use : "+derbyPort);
>> derbyPort++;
>> System.out.println("Try with port "+derbyPort);
>> }
>> }
>>
>> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
>> }
>>
>> public static void launchServer() throws Exception {
>> derbyServer = new DerbyServer_JDBC_BLOB_test();
>> }
>>
>> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
>> return derbyServer;
>> }
>>
>> /**
>> * <p>Start the server.
>> */
>> public void start() {
>> try {
>> server.start(null);
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>>
>> /**
>> * <p>Stop the server.
>> */
>> public void stop() {
>> try {
>> server.shutdown();
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>>}
>>
>>
>
>
>
--
/*
Tomohito Nakayama
tomonaka@basil.ocn.ne.jp
tomohito@rose.zero.ad.jp
tmnk@apache.org
Naka
http://www5.ocn.ne.jp/~tomohito/TopPage.html
*/
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Andreas Korneliussen (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420591 ]
Andreas Korneliussen commented on DERBY-550:
--------------------------------------------
Unless the streaming could be fixed for 10.2 so that we avoid OutOfMemoryError on the receiver side, I would propose the following:
We know the size of the LOB, and can check if it can go into memory (using the Runtime class). If it cannot go into memory, we can throw an SQLException, instead of consuming all memory in the VM until we get OutOfMemoryError.
By using this approach, we achieve the following:
* Side-effects on other connections in the VM: Although it is the LOB which is taking almost all the memory in the VM, the OutOfMemoryError may be thrown in another thread in the VM, causing side-effects on other connections or on the application itself.
* Currently, if the Network server goes out of memory when streaming data, the DRDAConnThread will stop. This causes hangs in the user applications.
If the streaming is fixed, there is not need to do this. Does anyone plan to fix the streaming issues for 10.2 ? If not, I will make a couple of JIRA issues to do the work of avoiding OutOfMemoryError by checking size before allocating the byte arrays.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC, Network Server
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by David Van Couvering <Da...@Sun.COM>.
Thanks, Bryan, I was not aware you could do that in JDBC -- I never
programmed BLOBs in JDBC. Definitely puts some constraints on the
network logic. If you can't cache it in memory, the network layer has
to do something with that data, and know how to get back to it when you
ask for it, and have protocol support for "can I have that BLOB again?"
That does get pretty tricky.
David
Bryan Pendleton wrote:
> David Van Couvering wrote:
>> I guess what I was assuming was, if the application goes off and does
>> something else, we can notice that and either raise an exception
>> ("you're not done with that BLOB column yet") or flush the rest of the
>> BLOB data, since it's obvious they won't be getting back to it (e.g.
>> if they send another query or do ResultSet.next(), it's clear they're
>> done with the BLOB column).
>
> Are you sure that's acceptable JDBC behavior? My (very old) copy of the
> JDBC spec says things like:
>
> The standard behavior for a Blob instance is to remain valid until the
> transaction in which it was created is either committed or rolled back.
>
> So if I do something like:
>
> ResultSet rs = stmt.executeQuery("SELECT DATA FROM TABLE1");
> rs.first();
> Blob data = rs.getBlob("DATA");
> InputStream blobStream = blob.getBinaryStream();
>
> I think I am supposed to be allowed to access blobStream quite some time
> later,
> even if I do other things on the connection in the meantime.
>
> But I confess I don't do a lot of BLOB programming in JDBC, so maybe I'm
> manufacturing bogeymen that don't actually exist.
>
> thanks,
>
> bryan
>
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by David Van Couvering <Da...@Sun.COM>.
This sounds like a reasonable short-term compromise for me. If we can
figure out how to make DRDA support the more "flexible" application,
that would be great, too, but I recognize that's a lot of effort and if
we can put it on the back burner and make some progress, that would be
great.
David
Andreas Korneliussen wrote:
> Bryan Pendleton wrote:
>> David Van Couvering wrote:
>>> I guess what I was assuming was, if the application goes off and does
>>> something else, we can notice that and either raise an exception
>>> ("you're not done with that BLOB column yet") or flush the rest of
>>> the BLOB data, since it's obvious they won't be getting back to it
>>> (e.g. if they send another query or do ResultSet.next(), it's clear
>>> they're done with the BLOB column).
>>
>> Are you sure that's acceptable JDBC behavior? My (very old) copy of the
>> JDBC spec says things like:
>>
>> The standard behavior for a Blob instance is to remain valid until the
>> transaction in which it was created is either committed or rolled back.
>>
>> So if I do something like:
>>
>> ResultSet rs = stmt.executeQuery("SELECT DATA FROM TABLE1");
>> rs.first();
>> Blob data = rs.getBlob("DATA");
>> InputStream blobStream = blob.getBinaryStream();
>>
>> I think I am supposed to be allowed to access blobStream quite some
>> time later,
>> even if I do other things on the connection in the meantime.
>>
>
> There is a similar discussion in
> http://www.nabble.com/-jira--Updated%3A-%28DERBY-721%29-State-of-InputStream-retrieved-from-resultset-is-not-clean-%2C-if-there-exists-previous-InputStream-.-tf664829.html#a1805521
>
>
> I understand that to preserve a BLOB throughout the lifetime of the
> transaction may be complicated.
>
> I would suggest a simplification:
> * We could optimize on the case where the user handles one column in the
> row at the time. If the user moves away from the row, or to another
> column, we could flush the Blob into memory, otherwise, if there is a
> nice user who handles one row / one column at the time, the user will
> avoid outofmemory issues. If there is a user who makes a tablescan and
> collects all the n Blob objects in memory, the user may risk OutOfMemory
> problems.
>
> Additionally, I think we should be conservative and check available
> memory before allocating memory for BLOBs, due to all the side-effects
> to the applications when the VM goes out of memory.
>
> Andreas
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Andreas Korneliussen <An...@Sun.COM>.
Bryan Pendleton wrote:
> David Van Couvering wrote:
>> I guess what I was assuming was, if the application goes off and does
>> something else, we can notice that and either raise an exception
>> ("you're not done with that BLOB column yet") or flush the rest of the
>> BLOB data, since it's obvious they won't be getting back to it (e.g.
>> if they send another query or do ResultSet.next(), it's clear they're
>> done with the BLOB column).
>
> Are you sure that's acceptable JDBC behavior? My (very old) copy of the
> JDBC spec says things like:
>
> The standard behavior for a Blob instance is to remain valid until the
> transaction in which it was created is either committed or rolled back.
>
> So if I do something like:
>
> ResultSet rs = stmt.executeQuery("SELECT DATA FROM TABLE1");
> rs.first();
> Blob data = rs.getBlob("DATA");
> InputStream blobStream = blob.getBinaryStream();
>
> I think I am supposed to be allowed to access blobStream quite some time
> later,
> even if I do other things on the connection in the meantime.
>
There is a similar discussion in
http://www.nabble.com/-jira--Updated%3A-%28DERBY-721%29-State-of-InputStream-retrieved-from-resultset-is-not-clean-%2C-if-there-exists-previous-InputStream-.-tf664829.html#a1805521
I understand that to preserve a BLOB throughout the lifetime of the
transaction may be complicated.
I would suggest a simplification:
* We could optimize on the case where the user handles one column in the
row at the time. If the user moves away from the row, or to another
column, we could flush the Blob into memory, otherwise, if there is a
nice user who handles one row / one column at the time, the user will
avoid outofmemory issues. If there is a user who makes a tablescan and
collects all the n Blob objects in memory, the user may risk OutOfMemory
problems.
Additionally, I think we should be conservative and check available
memory before allocating memory for BLOBs, due to all the side-effects
to the applications when the VM goes out of memory.
Andreas
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Bryan Pendleton <bp...@amberpoint.com>.
David Van Couvering wrote:
> I guess what I was assuming was, if the application goes off and does
> something else, we can notice that and either raise an exception
> ("you're not done with that BLOB column yet") or flush the rest of the
> BLOB data, since it's obvious they won't be getting back to it (e.g. if
> they send another query or do ResultSet.next(), it's clear they're done
> with the BLOB column).
Are you sure that's acceptable JDBC behavior? My (very old) copy of the
JDBC spec says things like:
The standard behavior for a Blob instance is to remain valid until the
transaction in which it was created is either committed or rolled back.
So if I do something like:
ResultSet rs = stmt.executeQuery("SELECT DATA FROM TABLE1");
rs.first();
Blob data = rs.getBlob("DATA");
InputStream blobStream = blob.getBinaryStream();
I think I am supposed to be allowed to access blobStream quite some time later,
even if I do other things on the connection in the meantime.
But I confess I don't do a lot of BLOB programming in JDBC, so maybe I'm
manufacturing bogeymen that don't actually exist.
thanks,
bryan
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by David Van Couvering <Da...@Sun.COM>.
I guess what I was assuming was, if the application goes off and does
something else, we can notice that and either raise an exception
("you're not done with that BLOB column yet") or flush the rest of the
BLOB data, since it's obvious they won't be getting back to it (e.g. if
they send another query or do ResultSet.next(), it's clear they're done
with the BLOB column). This is something that could be caught the next
time the am code hits the network layer for something. The network
layer would notice there was a half-processed BLOB and Do The Right Thing.
Lots of hand-waving, but it seems to me that this is solvable, and it
seems quite problematic to me that as it stands we just shoot the user
in the foot if they happen to have a BLOB column that's larger than
available memory, and there's nothing they can do to mitigate that.
David
Bryan Pendleton wrote:
> David Van Couvering wrote:
>> ... We should not allow any other requests to be sent to the server
>> over this connection until the BLOB data is processed or cancelled
>
> I'm not quite sure how we would do this. What is preventing the client
> from,
> say, calling ResultSet.next(), or going off to some other Statement object
> and running some other query, etc.?
>
> The DRDA protocol has a whole lot of logic regarding exactly how
> the requestor and the server communicate about the state of the
> conversation,
> what statements are in play and how to request or respond with actions
> on them,
> what results have already been communicated, how to pick up and resume a
> partially-fetched set of query results, etc.
>
> DRDA may well have all the protocol mechanisms in place for suspending an
> externalized data object partway through, picking it up later, etc., and we
> may already have all that support in place in our network client libraries.
>
> But I think there's some DRDA research waiting to be done here, that was
> the
> main point I was trying to raise.
>
> thanks,
>
> bryan
>
>
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Bryan Pendleton <bp...@amberpoint.com>.
David Van Couvering wrote:
> ... We should not
> allow any other requests to be sent to the server over this connection
> until the BLOB data is processed or cancelled
I'm not quite sure how we would do this. What is preventing the client from,
say, calling ResultSet.next(), or going off to some other Statement object
and running some other query, etc.?
The DRDA protocol has a whole lot of logic regarding exactly how
the requestor and the server communicate about the state of the conversation,
what statements are in play and how to request or respond with actions on them,
what results have already been communicated, how to pick up and resume a
partially-fetched set of query results, etc.
DRDA may well have all the protocol mechanisms in place for suspending an
externalized data object partway through, picking it up later, etc., and we
may already have all that support in place in our network client libraries.
But I think there's some DRDA research waiting to be done here, that was the
main point I was trying to raise.
thanks,
bryan
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by David Van Couvering <Da...@Sun.COM>.
Hi, Bryan, perhaps I'm missing your point, isn't it the case that the
server's thread simply blocks when its network buffer fills up, and that
the only effect is that this particular thread is put to sleep? Then
the possible scenarios occur:
- The client pulls the next batch data into its local buffer, and the
server thread can then send more data; this continues until all data is
processed.
- The client maintains the connection but never gets around to
processing the data. This seems unusual, but it's fine. We should not
allow any other requests to be sent to the server over this connection
until the BLOB data is processed or cancelled (and this raises the
question: *can* you cancel? I don't know JDBC well enough to answer this).
- The client connection goes away, at which point TCP-IP notifies the
server socket, and the server-side thread can be cleaned up and put back
on the free queue or whatever it is we do with it.
What deadlock situation are you concerned about?
David
Bryan Pendleton (JIRA) wrote:
> [ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420877 ]
>
> Bryan Pendleton commented on DERBY-550:
> ---------------------------------------
>
>
> RE: Andreas's observation that LOB implemtation classes would need to be reimplemented, so that they will do the streaming.
>
> I'm wondering whether this might accidentally introduce other new and undesirable behaviors.
>
> The net effect of a change like this is that the LOB data will remain in the network pipe, or be queued on the server side, until the blob is read by the client.
>
> But what if that takes a long time, or in fact never happens?
>
> We might need a way to "cancel" a partially-sent BLOB object which was returned by the server but which the client for whatever reason never decided to read.
>
> The current "greedy" algorithm seems to ensure that we minimize the risk of producer-consumer deadlocks of various sorts, at the expense of accumulating the entire data into memory.
>
> I hope this makes sense. I don't know of an actual problem here; I just have a funny feeling that this change is going to be rather tricky to accomplish.
>
>
>
>
>> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
>> -----------------------------------------------------------------------------------------------
>>
>> Key: DERBY-550
>> URL: http://issues.apache.org/jira/browse/DERBY-550
>> Project: Derby
>> Type: Bug
>
>> Components: JDBC, Network Server
>> Versions: 10.1.1.0
>> Environment: Any environment.
>> Reporter: Grégoire Dubois
>> Assignee: Tomohito Nakayama
>> Attachments: BlobOutOfMem.java
>>
>> Using the org.apache.derby.jdbc.ClientDriver driver to access the
>> Derby database through network, the driver is writting all the file into memory (RAM) before sending
>> it to the database.
>> Writting small files (smaller than 5Mo) into the database works fine,
>> but it is impossible to write big files (40Mo for example, or more), without getting the
>> exception java.lang.OutOfMemoryError.
>> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
>> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
>> import NetNoLedge.Configuration.Configs;
>> import org.apache.derby.drda.NetworkServerControl;
>> import java.net.InetAddress;
>> import java.io.*;
>> import java.sql.*;
>> /**
>> *
>> * @author greg
>> */
>> public class DerbyServer_JDBC_BLOB_test {
>>
>> // The unique instance of DerbyServer in the application.
>> private static DerbyServer_JDBC_BLOB_test derbyServer;
>>
>> private NetworkServerControl server;
>>
>> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
>> private static final String DERBY_DATABASE_NAME = "Test";
>>
>> // ###############################################################
>> // ############### SET HERE THE EXISTING PATH YOU WANT ################
>> // ###############################################################
>> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
>> // ###############################################################
>> // ###############################################################
>>
>>
>> private static int derbyPort = 9157;
>> private static String userName = "user";
>> private static String userPassword = "password";
>>
>> // ###################################################################################
>> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
>> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
>> // ###################################################################################
>> private static final File FILE = new File("/home/greg/01.jpg");
>> // ###################################################################################
>> // ###################################################################################
>>
>> /**
>> * <p>Used to test the server.
>> */
>> public static void main(String args[]) {
>> try {
>> DerbyServer_JDBC_BLOB_test.launchServer();
>> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
>> server.start();
>> System.out.println("Server started");
>>
>> // After the server has been started, launch a first connection to the database to
>> // 1) Create the database if it doesn't exist already,
>> // 2) Create the tables if they don't exist already.
>> Class.forName(DERBY_JDBC_DRIVER).newInstance();
>> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
>> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>>
>> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
>> Statement statement2;
>> // Create the table "file" if it doesn't already exist.
>> String [] tableNames={"file"};
>> boolean exist;
>> String currentTable;
>> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
>> for (int i=0;i<tableNames.length;i++) {
>> exist=false;
>> while (result.next()){
>> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
>> exist=true;
>> }
>>
>> if (!exist) {
>> statement2 = connection.createStatement();
>> statement2.execute("CREATE TABLE file (" +
>> "file BLOB(2G) NOT NULL)");
>> connection.commit();
>> }
>> result.beforeFirst();
>> }
>> System.out.println("Table file created if not created already");
>>
>> System.out.println("File insertion into BLOB");
>> FileInputStream inputStream = new FileInputStream(FILE);
>> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
>> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
>> preparedStatement .execute();
>> connection.commit();
>> System.out.println("File inserted into BLOB");
>> }
>> catch (Exception e) {
>> e.printStackTrace();
>> }
>> }
>>
>> /** Creates a new instance of MckoiServer
>> * Password is used at the database creation. It will be the database password once created.
>> */
>> private DerbyServer_JDBC_BLOB_test() throws Exception {
>> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>>
>> // Set the server to request an authentification.
>> System.setProperty("derby.authentication.provider", "BUILTIN");
>> System.setProperty("derby.connection.requireAuthentication", "true");
>>
>> // Create a user that can connect to Derby.
>> System.setProperty("derby.user."+userName, userPassword);
>>
>> // Set Derby to grant full access to the created user (to all the databases).
>> System.setProperty("derby.database.fullAccessUsers", userName);
>>
>> //System.setProperty("derby.system.bootAll", "true");
>>
>> // See if the 9157 port is already taken.
>> // Change it if necessary.
>> boolean isPortFree = false;
>> while ( !isPortFree ) {
>> try {
>> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
>> serverTest.close();
>> serverTest = null;
>>
>> isPortFree = true;
>> }
>> catch (Exception e) {
>> System.out.println("Port already in use : "+derbyPort);
>> derbyPort++;
>> System.out.println("Try with port "+derbyPort);
>> }
>> }
>>
>> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
>> }
>>
>> public static void launchServer() throws Exception {
>> derbyServer = new DerbyServer_JDBC_BLOB_test();
>> }
>>
>> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
>> return derbyServer;
>> }
>>
>> /**
>> * <p>Start the server.
>> */
>> public void start() {
>> try {
>> server.start(null);
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>>
>> /**
>> * <p>Stop the server.
>> */
>> public void stop() {
>> try {
>> server.shutdown();
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>> }
>
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Bryan Pendleton (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420877 ]
Bryan Pendleton commented on DERBY-550:
---------------------------------------
RE: Andreas's observation that LOB implemtation classes would need to be reimplemented, so that they will do the streaming.
I'm wondering whether this might accidentally introduce other new and undesirable behaviors.
The net effect of a change like this is that the LOB data will remain in the network pipe, or be queued on the server side, until the blob is read by the client.
But what if that takes a long time, or in fact never happens?
We might need a way to "cancel" a partially-sent BLOB object which was returned by the server but which the client for whatever reason never decided to read.
The current "greedy" algorithm seems to ensure that we minimize the risk of producer-consumer deadlocks of various sorts, at the expense of accumulating the entire data into memory.
I hope this makes sense. I don't know of an actual problem here; I just have a funny feeling that this change is going to be rather tricky to accomplish.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC, Network Server
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12421274 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
I noticed that there exists difference of phenomena between under -Xmx64m and under -Xmx128m.
Former, under -Xmx64m, OutOfMemory happens when java.io.ByteArrayOutputStream was constructed.
However , latter , -Xmx128m, OutOfMemory happens when java.io.ByteArrayOutputStream.toByteArray() was called.
I think this may have something to do with insufficient use of memory ....
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assigned To: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators: http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by TomohitoNakayama <to...@basil.ocn.ne.jp>.
Or .....
Did you use words "sending a large amount of " as just a meaning of
"when Network Server receives a large amount of data" ?
Excuse me for the difficulty of comprehension ...
Best regards.
TomohitoNakayama wrote:
> Hello Kathey.
>
>> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
>> Derby database through network, sending a large amount of parameter
>> data with setBinaryStream will cause Network Server to throw an
>> OutOfMemoryException.
>
>
> I'm not sure why you used the words "sending a large amount of " in
> upper text ....
> This words makes me think that there are problem of inefficient
> communication between server and client ...
>
> Is this just a misunderstanding ?
> Or there exists that kind of problem in communication between server
> and client ?
>
> Best regards.
>
>
> Kathey Marsden wrote:
>
>> Tomohito Nakayama (JIRA) wrote:
>>
>>> [
>>> http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419714
>>> ]
>>> Tomohito Nakayama commented on DERBY-550:
>>> -----------------------------------------
>>>
>>> I intended to resolve this issue as "Cannot Reproduce" as next url.
>>> http://issues.apache.org/jira/browse/DERBY-1472#action_12419712
>>>
>>> However, I reconsidered.
>>>
>>> I won't resolve DERBY-550, "BLOB : java.lang.OutOfMemoryError with
>>> network JDBC driver (org.apache.derby.jdbc.ClientDriver)",
>>> because it seems to be true that OutOfMemoryError happens "in server
>>> side".
>>>
>>> Title of DERBY-550 does not say that Error happens in client side.
>>> Then, this issue is not mistaken completely.
>>>
>>> Description of the issue "writting all the file into memory (RAM)
>>> before sending ", is mistaken.
>>>
>>>
>>>
>> Sorry for my own propagation of confusion over this issue. When
>> filing DERBY-326 and seeing similar code in the client I suspected
>> there might be streaming issues in the client as well. I then
>> took DERBY-550 as confirmation of that when I saw it filed against
>> the client and described in this way.
>>
>> In addition to changing the component to Network Server, changing
>> the DERBY-550 description as follows might clarify things sufficiently.
>> from
>>
>> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
>> Derby database through network, the driver is writting all the file
>> into memory (RAM) before sending
>> it to the database. "
>>
>> to more of a a description of the functional problem:
>>
>> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
>> Derby database through network, sending a large amount of parameter
>> data with setBinaryStream will cause Network Server to throw an
>> OutOfMemoryException.
>>
>> I am not sure if setCharacterStream is also at issue. If so it could
>> be filed as a separate issue.
>>
>> Kathey
>>
>>
>>
>
--
/*
Tomohito Nakayama
tomonaka@basil.ocn.ne.jp
tomohito@rose.zero.ad.jp
tmnk@apache.org
Naka
http://www5.ocn.ne.jp/~tomohito/TopPage.html
*/
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by TomohitoNakayama <to...@basil.ocn.ne.jp>.
Hello Kathey.
> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, sending a large amount of parameter
> data with setBinaryStream will cause Network Server to throw an
> OutOfMemoryException.
I'm not sure why you used the words "sending a large amount of " in
upper text ....
This words makes me think that there are problem of inefficient
communication between server and client ...
Is this just a misunderstanding ?
Or there exists that kind of problem in communication between server and
client ?
Best regards.
Kathey Marsden wrote:
> Tomohito Nakayama (JIRA) wrote:
>
>> [
>> http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419714
>> ]
>> Tomohito Nakayama commented on DERBY-550:
>> -----------------------------------------
>>
>> I intended to resolve this issue as "Cannot Reproduce" as next url.
>> http://issues.apache.org/jira/browse/DERBY-1472#action_12419712
>>
>> However, I reconsidered.
>>
>> I won't resolve DERBY-550, "BLOB : java.lang.OutOfMemoryError with
>> network JDBC driver (org.apache.derby.jdbc.ClientDriver)",
>> because it seems to be true that OutOfMemoryError happens "in server
>> side".
>>
>> Title of DERBY-550 does not say that Error happens in client side.
>> Then, this issue is not mistaken completely.
>>
>> Description of the issue "writting all the file into memory (RAM)
>> before sending ", is mistaken.
>>
>>
>>
> Sorry for my own propagation of confusion over this issue. When
> filing DERBY-326 and seeing similar code in the client I suspected
> there might be streaming issues in the client as well. I then took
> DERBY-550 as confirmation of that when I saw it filed against the
> client and described in this way.
>
> In addition to changing the component to Network Server, changing the
> DERBY-550 description as follows might clarify things sufficiently.
> from
>
> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file
> into memory (RAM) before sending
> it to the database. "
>
> to more of a a description of the functional problem:
>
> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, sending a large amount of parameter
> data with setBinaryStream will cause Network Server to throw an
> OutOfMemoryException.
>
> I am not sure if setCharacterStream is also at issue. If so it could
> be filed as a separate issue.
>
> Kathey
>
>
>
--
/*
Tomohito Nakayama
tomonaka@basil.ocn.ne.jp
tomohito@rose.zero.ad.jp
tmnk@apache.org
Naka
http://www5.ocn.ne.jp/~tomohito/TopPage.html
*/
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Andreas Korneliussen <An...@Sun.COM>.
> Do we support streaming in the embedded driver?
>
The embedded driver supports streaming. If you i.e call
PreparedStatement.setBinaryStream(int parameterIndex, InpuStream x, int
length), all the data of the InpuStream will be inserted into the row
when doing executeUpdate(), *without* expanding all the data of the
stream into memory. The same applies to updatable resultsets
(ResultSet.updateBinaryStream(..) and ResultSet.updateRow()).
Andreas
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Kristian Waagan <Kr...@Sun.COM>.
Andreas Korneliussen wrote:
> Kathey Marsden wrote:
>> Tomohito Nakayama (JIRA) wrote:
>>
>>> [
>>> http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419714
>>> ]
>>> Tomohito Nakayama commented on DERBY-550:
>>> -----------------------------------------
>>>
>>> I intended to resolve this issue as "Cannot Reproduce" as next url.
>>> http://issues.apache.org/jira/browse/DERBY-1472#action_12419712
>>>
>>> However, I reconsidered.
>>>
>>> I won't resolve DERBY-550, "BLOB : java.lang.OutOfMemoryError with
>>> network JDBC driver (org.apache.derby.jdbc.ClientDriver)",
>>> because it seems to be true that OutOfMemoryError happens "in server
>>> side".
>>>
>>> Title of DERBY-550 does not say that Error happens in client side.
>>> Then, this issue is not mistaken completely.
>>>
>>> Description of the issue "writting all the file into memory (RAM)
>>> before sending ", is mistaken.
>>>
>>>
>>>
>> Sorry for my own propagation of confusion over this issue. When
>> filing DERBY-326 and seeing similar code in the client I suspected
>> there might be streaming issues in the client as well. I then took
>> DERBY-550 as confirmation of that when I saw it filed against the
>> client and described in this way.
>>
>> In addition to changing the component to Network Server, changing the
>> DERBY-550 description as follows might clarify things sufficiently.
>> from
>>
>> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
>> Derby database through network, the driver is writting all the file
>> into memory (RAM) before sending
>> it to the database. "
>>
>> to more of a a description of the functional problem:
>>
>> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
>> Derby database through network, sending a large amount of parameter
>> data with setBinaryStream will cause Network Server to throw an
>> OutOfMemoryException.
>>
>> I am not sure if setCharacterStream is also at issue. If so it could
>> be filed as a separate issue.
>>
>
> I think additionally one could make an issue for the client:
>
> "Using the org.apache.derby.jdbc.ClientDriver driver to access the Derby
> database through network, receiving a large amount of resultset data
> with rs.next() may cause Network Client to go out of memory"
>
> I think derby works fine in the "sending" end of the streams, however in
> the receiving end, there are memory problems.
I can second that, as I had a look at the code for ResultSet when
looking at another issue. For instance, the
NetCursor.getBlobColumn-method seems to materialize the whole Blob:
(data is a byte array)
if (data != null) {
// data found
// set data offset based on the presence of a null indicator
if (!nullable_[index]) {
dataOffset = 0;
} else {
dataOffset = 1;
}
blob = new Blob(data, agent, dataOffset);
} else {
The contents of 'data' come from the copyEXTDTA-method in
NetStatementReply.java.
protected void copyEXTDTA(NetCursor netCursor) throws
DisconnectException {
try {
parseLengthAndMatchCodePoint(CodePoint.EXTDTA);
byte[] data = null;
if (longValueForDecryption_ == null) {
data = (getData(null)).toByteArray();
} else {
data = longValueForDecryption_;
dssLength_ = 0;
longValueForDecryption_ = null;
}
netCursor.extdtaData_.add(data);
} catch (java.lang.OutOfMemoryError e) {
agent_.accumulateChainBreakingReadExceptionAndThrow(new
DisconnectException(agent_,
new
ClientMessageId(SQLState.NET_LOB_DATA_TOO_LARGE_FOR_JVM),
e));
}
}
I know layerB streaming in DRDA is not yet implemented (DERBY-1471) for
streaming from the client to the server (to be used for streams with
unknown length).
Do we have any support for streaming between server and client, which
could be used to implement LOB streaming?
Do we support streaming in the embedded driver?
--
Kristian
>
> Andreas
>
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Andreas Korneliussen <An...@Sun.COM>.
Kathey Marsden wrote:
> Tomohito Nakayama (JIRA) wrote:
>
>> [
>> http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419714
>> ]
>> Tomohito Nakayama commented on DERBY-550:
>> -----------------------------------------
>>
>> I intended to resolve this issue as "Cannot Reproduce" as next url.
>> http://issues.apache.org/jira/browse/DERBY-1472#action_12419712
>>
>> However, I reconsidered.
>>
>> I won't resolve DERBY-550, "BLOB : java.lang.OutOfMemoryError with
>> network JDBC driver (org.apache.derby.jdbc.ClientDriver)",
>> because it seems to be true that OutOfMemoryError happens "in server
>> side".
>>
>> Title of DERBY-550 does not say that Error happens in client side.
>> Then, this issue is not mistaken completely.
>>
>> Description of the issue "writting all the file into memory (RAM)
>> before sending ", is mistaken.
>>
>>
>>
> Sorry for my own propagation of confusion over this issue. When
> filing DERBY-326 and seeing similar code in the client I suspected
> there might be streaming issues in the client as well. I then took
> DERBY-550 as confirmation of that when I saw it filed against the
> client and described in this way.
>
> In addition to changing the component to Network Server, changing the
> DERBY-550 description as follows might clarify things sufficiently.
> from
>
> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into
> memory (RAM) before sending
> it to the database. "
>
> to more of a a description of the functional problem:
>
> "Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, sending a large amount of parameter data
> with setBinaryStream will cause Network Server to throw an
> OutOfMemoryException.
>
> I am not sure if setCharacterStream is also at issue. If so it could be
> filed as a separate issue.
>
I think additionally one could make an issue for the client:
"Using the org.apache.derby.jdbc.ClientDriver driver to access the Derby
database through network, receiving a large amount of resultset data
with rs.next() may cause Network Client to go out of memory"
I think derby works fine in the "sending" end of the streams, however in
the receiving end, there are memory problems.
Andreas
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Kathey Marsden <km...@sbcglobal.net>.
Tomohito Nakayama (JIRA) wrote:
> [ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419714 ]
>
>Tomohito Nakayama commented on DERBY-550:
>-----------------------------------------
>
>I intended to resolve this issue as "Cannot Reproduce" as next url.
>http://issues.apache.org/jira/browse/DERBY-1472#action_12419712
>
>However, I reconsidered.
>
>I won't resolve DERBY-550, "BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)",
>because it seems to be true that OutOfMemoryError happens "in server side".
>
>Title of DERBY-550 does not say that Error happens in client side.
>Then, this issue is not mistaken completely.
>
>Description of the issue "writting all the file into memory (RAM) before sending ", is mistaken.
>
>
>
Sorry for my own propagation of confusion over this issue. When
filing DERBY-326 and seeing similar code in the client I suspected
there might be streaming issues in the client as well. I then took
DERBY-550 as confirmation of that when I saw it filed against the
client and described in this way.
In addition to changing the component to Network Server, changing the
DERBY-550 description as follows might clarify things sufficiently.
from
"Using the org.apache.derby.jdbc.ClientDriver driver to access the
Derby database through network, the driver is writting all the file into
memory (RAM) before sending
it to the database. "
to more of a a description of the functional problem:
"Using the org.apache.derby.jdbc.ClientDriver driver to access the
Derby database through network, sending a large amount of parameter data
with setBinaryStream will cause Network Server to throw an
OutOfMemoryException.
I am not sure if setCharacterStream is also at issue. If so it could be
filed as a separate issue.
Kathey
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419714 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
I intended to resolve this issue as "Cannot Reproduce" as next url.
http://issues.apache.org/jira/browse/DERBY-1472#action_12419712
However, I reconsidered.
I won't resolve DERBY-550, "BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)",
because it seems to be true that OutOfMemoryError happens "in server side".
Title of DERBY-550 does not say that Error happens in client side.
Then, this issue is not mistaken completely.
Description of the issue "writting all the file into memory (RAM) before sending ", is mistaken.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Andreas Korneliussen <An...@Sun.COM>.
TomohitoNakayama wrote:
> Hello Andreas.
>
> I see ...
> H mm. Complex circumstances ....
>
> The program failed in creating error message of OutOfMemoryError ...
>
> Please upload your application program.
> I want to make it possible to see the phenomena by myself.
>
> Best regards.
>
Hi,
Yes I will upload the program.
Note: the VM throws a OutOfMemoryException, which is caught in:
NetStatementReply.copyEXTDTA:
protected void copyEXTDTA(NetCursor netCursor) throws
DisconnectException {
try {
parseLengthAndMatchCodePoint(CodePoint.EXTDTA);
byte[] data = null;
if (longValueForDecryption_ == null) {
data = (getData(null)).toByteArray();
} else {
data = longValueForDecryption_;
dssLength_ = 0;
longValueForDecryption_ = null;
}
netCursor.extdtaData_.add(data);
} catch (java.lang.OutOfMemoryError e) { <--- outofmemory
agent_.accumulateChainBreakingReadExceptionAndThrow(new
DisconnectException(agent_,
new
ClientMessageId(SQLState.NET_LOB_DATA_TOO_LARGE_FOR_JVM),
e)); <----- message does not take parameters, causing
assert failure
}
}
When it calls new ClientMessageId(...), it fails with the assert, since
the message does not take any parameters (here e is added as parameter
to the message)
Andreas
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by TomohitoNakayama <to...@basil.ocn.ne.jp>.
Hello Andreas.
I see ...
H mm. Complex circumstances ....
The program failed in creating error message of OutOfMemoryError ...
Please upload your application program.
I want to make it possible to see the phenomena by myself.
Best regards.
Andreas Korneliussen wrote:
> TomohitoNakayama wrote:
>
>> Hello Andreas.
>>
>> I have some question for your comment....
>>
>> Reading beggining part, I take your reported phenomena as problem
>> when scrollable resultset is used.
>> However,reading last part, I found your report tells that problem
>> was found also when forward-only resultset was used.
>>
>> I feel some inconsistency and cannot be sure I understand your
>> comment ....
>>
>> My question is whether your report is limited to scrollable
>> resultset, or not
>> Please give me your answer..
>>
>> Best regards.
>>
> Hi, Tomohito
>
> To answer your question: the problem applies to both scrollable and
> forward-only result sets.
>
> When I first tested, I used scrollable resultsets. The second time I
> used forward-only resultsets, and got another problem when doing
> connect. I then restarted the network-server, and retried with
> forward-only resultset, and then I got the exactly same problem.
>
> Below is the stack trace:
>
> Exception in thread "main"
> org.apache.derby.shared.common.sanity.AssertFailure: ASSERT FAILED
> Number of parameters expected for message id 58009.C.6 (0) does not
> match number of arguments received (1)
> at
> org.apache.derby.shared.common.sanity.SanityManager.ASSERT(SanityManager.java:119)
>
> at
> org.apache.derby.shared.common.i18n.MessageUtil.formatMessage(MessageUtil.java:233)
>
> at
> org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:142)
>
> at
> org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:182)
>
> at
> org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:98)
>
> at
> org.apache.derby.client.am.SqlException.<init>(SqlException.java:168)
> at
> org.apache.derby.client.am.SqlException.<init>(SqlException.java:182)
> at
> org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:28)
>
> at
> org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:39)
>
> at
> org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:49)
>
> at
> org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:67)
>
> at
> org.apache.derby.client.net.NetStatementReply.copyEXTDTA(NetStatementReply.java:1486)
>
> at
> org.apache.derby.client.net.NetResultSetReply.parseCNTQRYreply(NetResultSetReply.java:139)
>
> at
> org.apache.derby.client.net.NetResultSetReply.readFetch(NetResultSetReply.java:41)
>
> at
> org.apache.derby.client.net.ResultSetReply.readFetch(ResultSetReply.java:40)
>
> at
> org.apache.derby.client.net.NetResultSet.readFetch_(NetResultSet.java:205)
>
> at
> org.apache.derby.client.am.ResultSet.flowFetch(ResultSet.java:4160)
> at
> org.apache.derby.client.net.NetCursor.getMoreData_(NetCursor.java:1182)
> at org.apache.derby.client.am.Cursor.stepNext(Cursor.java:176)
> at org.apache.derby.client.am.Cursor.next(Cursor.java:195)
> at org.apache.derby.client.am.ResultSet.nextX(ResultSet.java:299)
> at org.apache.derby.client.am.ResultSet.next(ResultSet.java:269)
> at derbytest.BlobOutOfMem.main(BlobOutOfMem.java:95)
>
> My application has a BLOB column of size 64 MB, which I inserted in
> embedded mode. If you like, I can upload it to the JIRA.
>
> Regards
>
> --Andreas
>
>
--
/*
Tomohito Nakayama
tomonaka@basil.ocn.ne.jp
tomohito@rose.zero.ad.jp
tmnk@apache.org
Naka
http://www5.ocn.ne.jp/~tomohito/TopPage.html
*/
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by Andreas Korneliussen <An...@Sun.COM>.
TomohitoNakayama wrote:
> Hello Andreas.
>
> I have some question for your comment....
>
> Reading beggining part, I take your reported phenomena as problem when
> scrollable resultset is used.
> However,reading last part, I found your report tells that problem was
> found also when forward-only resultset was used.
>
> I feel some inconsistency and cannot be sure I understand your comment ....
>
> My question is whether your report is limited to scrollable resultset,
> or not
> Please give me your answer..
>
> Best regards.
>
Hi, Tomohito
To answer your question: the problem applies to both scrollable and
forward-only result sets.
When I first tested, I used scrollable resultsets. The second time I
used forward-only resultsets, and got another problem when doing
connect. I then restarted the network-server, and retried with
forward-only resultset, and then I got the exactly same problem.
Below is the stack trace:
Exception in thread "main"
org.apache.derby.shared.common.sanity.AssertFailure: ASSERT FAILED
Number of parameters expected for message id 58009.C.6 (0) does not
match number of arguments received (1)
at
org.apache.derby.shared.common.sanity.SanityManager.ASSERT(SanityManager.java:119)
at
org.apache.derby.shared.common.i18n.MessageUtil.formatMessage(MessageUtil.java:233)
at
org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:142)
at
org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:182)
at
org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:98)
at
org.apache.derby.client.am.SqlException.<init>(SqlException.java:168)
at
org.apache.derby.client.am.SqlException.<init>(SqlException.java:182)
at
org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:28)
at
org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:39)
at
org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:49)
at
org.apache.derby.client.am.DisconnectException.<init>(DisconnectException.java:67)
at
org.apache.derby.client.net.NetStatementReply.copyEXTDTA(NetStatementReply.java:1486)
at
org.apache.derby.client.net.NetResultSetReply.parseCNTQRYreply(NetResultSetReply.java:139)
at
org.apache.derby.client.net.NetResultSetReply.readFetch(NetResultSetReply.java:41)
at
org.apache.derby.client.net.ResultSetReply.readFetch(ResultSetReply.java:40)
at
org.apache.derby.client.net.NetResultSet.readFetch_(NetResultSet.java:205)
at
org.apache.derby.client.am.ResultSet.flowFetch(ResultSet.java:4160)
at
org.apache.derby.client.net.NetCursor.getMoreData_(NetCursor.java:1182)
at org.apache.derby.client.am.Cursor.stepNext(Cursor.java:176)
at org.apache.derby.client.am.Cursor.next(Cursor.java:195)
at org.apache.derby.client.am.ResultSet.nextX(ResultSet.java:299)
at org.apache.derby.client.am.ResultSet.next(ResultSet.java:269)
at derbytest.BlobOutOfMem.main(BlobOutOfMem.java:95)
My application has a BLOB column of size 64 MB, which I inserted in
embedded mode. If you like, I can upload it to the JIRA.
Regards
--Andreas
Re: [jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by TomohitoNakayama <to...@basil.ocn.ne.jp>.
Hello Andreas.
I have some question for your comment....
Reading beggining part, I take your reported phenomena as problem when
scrollable resultset is used.
However,reading last part, I found your report tells that problem was
found also when forward-only resultset was used.
I feel some inconsistency and cannot be sure I understand your comment ....
My question is whether your report is limited to scrollable resultset,
or not
Please give me your answer..
Best regards.
Andreas Korneliussen (JIRA) wrote:
> [ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419724 ]
>
>Andreas Korneliussen commented on DERBY-550:
>--------------------------------------------
>
>One more comment: there seems to be a client-side problem when streaming data to the client.
>I have created a table with a 64 MB blob in embedded mode, and if I try to get the data with a scrollable resultset on the client side, it fails with OutOfMemoryError in NetStatementReply.copyEXTDTA, which is caught in an exception handler:
>
> protected void copyEXTDTA(NetCursor netCursor) throws DisconnectException {
> try {
> parseLengthAndMatchCodePoint(CodePoint.EXTDTA);
> byte[] data = null;
> if (longValueForDecryption_ == null) {
> data = (getData(null)).toByteArray();
> } else {
> data = longValueForDecryption_;
> dssLength_ = 0;
> longValueForDecryption_ = null;
> }
> netCursor.extdtaData_.add(data);
> } catch (java.lang.OutOfMemoryError e) {
> agent_.accumulateChainBreakingReadExceptionAndThrow(new DisconnectException(agent_,
> new ClientMessageId(SQLState.NET_LOB_DATA_TOO_LARGE_FOR_JVM),
> e));
> }
>This again causes an assert in MessageUtil:
>Exception in thread "main" org.apache.derby.shared.common.sanity.AssertFailure: ASSERT FAILED Number of parameters expected for message id 58009.C.6 (0) does not match number of arguments received (1)
> at org.apache.derby.shared.common.sanity.SanityManager.ASSERT(SanityManager.java:119)
> at org.apache.derby.shared.common.i18n.MessageUtil.formatMessage(MessageUtil.java:233)
> at org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:142)
> at org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:182)
> at org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:98)
>
>Using forward-only resultset:
>Get this error:
>
>java.sql.SQLException: Network protocol exception: DSS chained with same id at end of same id chain parse. The connection has been terminated.
> at org.apache.derby.client.am.SQLExceptionFactory.getSQLException(SQLExceptionFactory.java:45)
> at org.apache.derby.client.am.SqlException.getSQLException(SqlException.java:344)
> at org.apache.derby.jdbc.ClientDriver.connect(ClientDriver.java:148)
> at java.sql.DriverManager.getConnection(DriverManager.java:525)
> at java.sql.DriverManager.getConnection(DriverManager.java:171)
> at derbytest.BlobOutOfMem.main(BlobOutOfMem.java:59)
>
>
>
>>BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
>>-----------------------------------------------------------------------------------------------
>>
>> Key: DERBY-550
>> URL: http://issues.apache.org/jira/browse/DERBY-550
>> Project: Derby
>> Type: Bug
>>
>>
>
>
>
>> Components: JDBC, Network Server
>> Versions: 10.1.1.0
>> Environment: Any environment.
>> Reporter: Grégoire Dubois
>> Assignee: Tomohito Nakayama
>>
>>
>
>
>
>>Using the org.apache.derby.jdbc.ClientDriver driver to access the
>>Derby database through network, the driver is writting all the file into memory (RAM) before sending
>>it to the database.
>>Writting small files (smaller than 5Mo) into the database works fine,
>>but it is impossible to write big files (40Mo for example, or more), without getting the
>>exception java.lang.OutOfMemoryError.
>>The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
>>Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
>>import NetNoLedge.Configuration.Configs;
>>import org.apache.derby.drda.NetworkServerControl;
>>import java.net.InetAddress;
>>import java.io.*;
>>import java.sql.*;
>>/**
>> *
>> * @author greg
>> */
>>public class DerbyServer_JDBC_BLOB_test {
>>
>> // The unique instance of DerbyServer in the application.
>> private static DerbyServer_JDBC_BLOB_test derbyServer;
>>
>> private NetworkServerControl server;
>>
>> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
>> private static final String DERBY_DATABASE_NAME = "Test";
>>
>> // ###############################################################
>> // ############### SET HERE THE EXISTING PATH YOU WANT ################
>> // ###############################################################
>> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
>> // ###############################################################
>> // ###############################################################
>>
>>
>> private static int derbyPort = 9157;
>> private static String userName = "user";
>> private static String userPassword = "password";
>>
>> // ###################################################################################
>> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
>> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
>> // ###################################################################################
>> private static final File FILE = new File("/home/greg/01.jpg");
>> // ###################################################################################
>> // ###################################################################################
>>
>> /**
>> * <p>Used to test the server.
>> */
>> public static void main(String args[]) {
>> try {
>> DerbyServer_JDBC_BLOB_test.launchServer();
>> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
>> server.start();
>> System.out.println("Server started");
>>
>> // After the server has been started, launch a first connection to the database to
>> // 1) Create the database if it doesn't exist already,
>> // 2) Create the tables if they don't exist already.
>> Class.forName(DERBY_JDBC_DRIVER).newInstance();
>> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
>> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>>
>> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
>> Statement statement2;
>> // Create the table "file" if it doesn't already exist.
>> String [] tableNames={"file"};
>> boolean exist;
>> String currentTable;
>> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
>> for (int i=0;i<tableNames.length;i++) {
>> exist=false;
>> while (result.next()){
>> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
>> exist=true;
>> }
>>
>> if (!exist) {
>> statement2 = connection.createStatement();
>> statement2.execute("CREATE TABLE file (" +
>> "file BLOB(2G) NOT NULL)");
>> connection.commit();
>> }
>> result.beforeFirst();
>> }
>> System.out.println("Table file created if not created already");
>>
>> System.out.println("File insertion into BLOB");
>> FileInputStream inputStream = new FileInputStream(FILE);
>> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
>> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
>> preparedStatement .execute();
>> connection.commit();
>> System.out.println("File inserted into BLOB");
>> }
>> catch (Exception e) {
>> e.printStackTrace();
>> }
>> }
>>
>> /** Creates a new instance of MckoiServer
>> * Password is used at the database creation. It will be the database password once created.
>> */
>> private DerbyServer_JDBC_BLOB_test() throws Exception {
>> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>>
>> // Set the server to request an authentification.
>> System.setProperty("derby.authentication.provider", "BUILTIN");
>> System.setProperty("derby.connection.requireAuthentication", "true");
>>
>> // Create a user that can connect to Derby.
>> System.setProperty("derby.user."+userName, userPassword);
>>
>> // Set Derby to grant full access to the created user (to all the databases).
>> System.setProperty("derby.database.fullAccessUsers", userName);
>>
>> //System.setProperty("derby.system.bootAll", "true");
>>
>> // See if the 9157 port is already taken.
>> // Change it if necessary.
>> boolean isPortFree = false;
>> while ( !isPortFree ) {
>> try {
>> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
>> serverTest.close();
>> serverTest = null;
>>
>> isPortFree = true;
>> }
>> catch (Exception e) {
>> System.out.println("Port already in use : "+derbyPort);
>> derbyPort++;
>> System.out.println("Try with port "+derbyPort);
>> }
>> }
>>
>> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
>> }
>>
>> public static void launchServer() throws Exception {
>> derbyServer = new DerbyServer_JDBC_BLOB_test();
>> }
>>
>> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
>> return derbyServer;
>> }
>>
>> /**
>> * <p>Start the server.
>> */
>> public void start() {
>> try {
>> server.start(null);
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>>
>> /**
>> * <p>Stop the server.
>> */
>> public void stop() {
>> try {
>> server.shutdown();
>> }
>> catch (Exception e) {
>> e.printStackTrace(System.err);
>> }
>> }
>>}
>>
>>
>
>
>
--
/*
Tomohito Nakayama
tomonaka@basil.ocn.ne.jp
tomohito@rose.zero.ad.jp
tmnk@apache.org
Naka
http://www5.ocn.ne.jp/~tomohito/TopPage.html
*/
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Andreas Korneliussen (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419724 ]
Andreas Korneliussen commented on DERBY-550:
--------------------------------------------
One more comment: there seems to be a client-side problem when streaming data to the client.
I have created a table with a 64 MB blob in embedded mode, and if I try to get the data with a scrollable resultset on the client side, it fails with OutOfMemoryError in NetStatementReply.copyEXTDTA, which is caught in an exception handler:
protected void copyEXTDTA(NetCursor netCursor) throws DisconnectException {
try {
parseLengthAndMatchCodePoint(CodePoint.EXTDTA);
byte[] data = null;
if (longValueForDecryption_ == null) {
data = (getData(null)).toByteArray();
} else {
data = longValueForDecryption_;
dssLength_ = 0;
longValueForDecryption_ = null;
}
netCursor.extdtaData_.add(data);
} catch (java.lang.OutOfMemoryError e) {
agent_.accumulateChainBreakingReadExceptionAndThrow(new DisconnectException(agent_,
new ClientMessageId(SQLState.NET_LOB_DATA_TOO_LARGE_FOR_JVM),
e));
}
This again causes an assert in MessageUtil:
Exception in thread "main" org.apache.derby.shared.common.sanity.AssertFailure: ASSERT FAILED Number of parameters expected for message id 58009.C.6 (0) does not match number of arguments received (1)
at org.apache.derby.shared.common.sanity.SanityManager.ASSERT(SanityManager.java:119)
at org.apache.derby.shared.common.i18n.MessageUtil.formatMessage(MessageUtil.java:233)
at org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:142)
at org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:182)
at org.apache.derby.shared.common.i18n.MessageUtil.getCompleteMessage(MessageUtil.java:98)
Using forward-only resultset:
Get this error:
java.sql.SQLException: Network protocol exception: DSS chained with same id at end of same id chain parse. The connection has been terminated.
at org.apache.derby.client.am.SQLExceptionFactory.getSQLException(SQLExceptionFactory.java:45)
at org.apache.derby.client.am.SqlException.getSQLException(SqlException.java:344)
at org.apache.derby.jdbc.ClientDriver.connect(ClientDriver.java:148)
at java.sql.DriverManager.getConnection(DriverManager.java:525)
at java.sql.DriverManager.getConnection(DriverManager.java:171)
at derbytest.BlobOutOfMem.main(BlobOutOfMem.java:59)
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC, Network Server
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/DERBY-550?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12611367#action_12611367 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
My understanding is that performance of memory usage was improved issues including DERBY-1513, DERBY-1535 and other issues worked at that time ....
I think this issue should be closed.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: https://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12423067 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
I confirmed that the BlobOutOfMem.java ran without error in next configuration, after the patch of DERBY-1513 and DERBY-1535.
ServerSide:
$JAVA_HOME/bin/java -Xmx256m org.apache.derby.drda.NetworkServerControl start -h $DERBY_SERVER_HOST -p $DERBY_SERVER_PORT
ClientSide:
java -Xms32m -Xmx256m derbytest.BlobOutOfMem
I think it has been improved to some extent.
I hope opinion from others...
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assigned To: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators: http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Kathey Marsden (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/DERBY-550?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12611350#action_12611350 ]
Kathey Marsden commented on DERBY-550:
--------------------------------------
I wonder if this issue can be closed now that the repro passes without error according to Tomohito
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: https://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Assigned: (DERBY-550) BLOB : java.lang.OutOfMemoryError with
network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=all ]
Tomohito Nakayama reassigned DERBY-550:
---------------------------------------
Assign To: Tomohito Nakayama
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420894 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
Configuring -Xmx256m option to jdb under NetworkServer, I tested again.
Then,I found different stack for the error.
DRDAConnThread_3[1] where
[1] org.apache.derby.iapi.services.io.DynamicByteArrayOutputStream.<init> (DynamicByteArrayOutputStream.java:63)
[2] org.apache.derby.impl.store.raw.data.BasePage.insertAllowOverflow (BasePage.java:821)
[3] org.apache.derby.impl.store.raw.data.BasePage.insert (BasePage.java:694)
[4] org.apache.derby.impl.store.access.heap.HeapController.doInsert (HeapController.java:306)
[5] org.apache.derby.impl.store.access.heap.HeapController.insert (HeapController.java:573)
[6] org.apache.derby.impl.sql.execute.RowChangerImpl.insertRow (RowChangerImpl.java:447)
[7] org.apache.derby.impl.sql.execute.InsertResultSet.normalInsertCore (InsertResultSet.java:995)
[8] org.apache.derby.impl.sql.execute.InsertResultSet.open (InsertResultSet.java:522)
[9] org.apache.derby.impl.sql.GenericPreparedStatement.execute (GenericPreparedStatement.java:357)
[10] org.apache.derby.impl.jdbc.EmbedStatement.executeStatement (EmbedStatement.java:1,181)
[11] org.apache.derby.impl.jdbc.EmbedPreparedStatement.executeStatement (EmbedPreparedStatement.java:1,510)
[12] org.apache.derby.impl.jdbc.EmbedPreparedStatement.execute (EmbedPreparedStatement.java:1,188)
[13] org.apache.derby.impl.drda.DRDAStatement.execute (DRDAStatement.java:559)
[14] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTT (DRDAConnThread.java:3,655)
[15] org.apache.derby.impl.drda.DRDAConnThread.processCommands (DRDAConnThread.java:928)
[16] org.apache.derby.impl.drda.DRDAConnThread.run (DRDAConnThread.java:254)
DRDAConnThread_3[1] eval java.lang.Runtime.getRuntime().maxMemory()
java.lang.Runtime.getRuntime().maxMemory() = 266403840
DRDAConnThread_3[1]
I think it is remarkable that error happend at code of Engine in this case.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC, Network Server
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12420891 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
I executed BlobOutOfMem.java with Network Server running on jdb and
tried to recognize circumstance where OutOfMemoryError happens.
The result was as next.
naka@rufelza:~/derby/test/20060714$ startDebugNetworkServer.ksh
Initializing jdb ...
> catch java.lang.OutOfMemoryError
Deferring all java.lang.OutOfMemoryError.
It will be set after the class is loaded.
> run
run org.apache.derby.drda.NetworkServerControl start -h localhost -p 1527
Set uncaught java.lang.Throwable
Set deferred all java.lang.OutOfMemoryError
Set deferred uncaught java.lang.Throwable
>
VM Started: Apache Derby Network Server - 10.2.0.4 alpha started and ready to accept connections on port 1527 at 2006-07-13 15:40:31.278 GMT
Exception occurred: java.lang.OutOfMemoryError (uncaught)
Exception occurred: java.lang.OutOfMemoryError (uncaught)"thread=DRDAConnThread_3", java.io.ByteArrayOutputStream.<init>(), line=59 bci=37
DRDAConnThread_3[1] where
[1] java.io.ByteArrayOutputStream.<init> (ByteArrayOutputStream.java:59)
[2] org.apache.derby.impl.drda.DDMReader.getExtData (DDMReader.java:958)
[3] org.apache.derby.impl.drda.DDMReader.getExtData (DDMReader.java:944)
[4] org.apache.derby.impl.drda.DRDAConnThread.readAndSetExtParam (DRDAConnThread.java:4,355)
[5] org.apache.derby.impl.drda.DRDAConnThread.readAndSetAllExtParams (DRDAConnThread.java:4,320)
[6] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTTobjects (DRDAConnThread.java:3,811)
[7] org.apache.derby.impl.drda.DRDAConnThread.parseEXCSQLSTT (DRDAConnThread.java:3,640)
[8] org.apache.derby.impl.drda.DRDAConnThread.processCommands (DRDAConnThread.java:928)
[9] org.apache.derby.impl.drda.DRDAConnThread.run (DRDAConnThread.java:254)
DRDAConnThread_3[1] up
DRDAConnThread_3[2] list
954
955
956 if (desiredLength != -1) {
957 // allocate a stream based on a known amount of data
958 => baos = new ByteArrayOutputStream ((int) desiredLength);
959 }
960 else {
961 // allocate a stream to hold an unknown amount of data
962 baos = new ByteArrayOutputStream ();
963 //isLengthAndNullabilityUnknown = true;
DRDAConnThread_3[2] up
DRDAConnThread_3[3] list
940
941
942 byte[] getExtData (boolean checkNullability) throws DRDAProtocolException
943 {
944 => return getExtData(ddmScalarLen, checkNullability);
945 }
946
947
948 byte[] getExtData (long desiredLength, boolean checkNullability) throws DRDAProtocolException
949 {
DRDAConnThread_3[3] up
DRDAConnThread_3[4] list
4,351 FdocaConstants.isNullable(drdaType))
4,352 checkNullability = true;
4,353
4,354 try {
4,355 => byte[] paramBytes = reader.getExtData(checkNullability);
4,356 String paramString = null;
4,357 switch (drdaType)
4,358 {
4,359 case DRDAConstants.DRDA_TYPE_LOBBYTES:
4,360 case DRDAConstants.DRDA_TYPE_NLOBBYTES:
DRDAConnThread_3[4]
I confirm this part is where Andreas mentioned in previous comment.
I think this is one of very suspective places.
However I think there may be another place where memory was used in waste.
Because reports from others for memory usage told that amount of used memory was much more larger than actual size LOB to be streamed.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC, Network Server
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Kristian Waagan (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/DERBY-550?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12611641#action_12611641 ]
Kristian Waagan commented on DERBY-550:
---------------------------------------
I also tested this, and I agree with Tomohito.
With regards to memory usage, the problems seem to be resolved for the scenario mentioned.
However, I discovered multiple performance issues, some of which are rather significant. I'll create Jiras for these and start working on them.
One of my test runs with the 64 MB Blob showed an improvement from around 270 to 4 seconds (localhost)!
There are issues both in the embedded driver and the client driver.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: https://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Resolved: (DERBY-550) BLOB : java.lang.OutOfMemoryError with
network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Kathey Marsden (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/DERBY-550?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Kathey Marsden resolved DERBY-550.
----------------------------------
Resolution: Fixed
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: https://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12421282 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
It seems to be have much amount of impact on whole code of NetworkServer to use writeTo method and not to use byte[] type data............
Well ......
First, I will work to the idea of http://issues.apache.org/jira/browse/DERBY-550#action_12421280 , and see the result.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assigned To: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators: http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12419716 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
BTW, I just tried to resolve this issue and
assigned myself to resolve as "Cannot Reproduce".
Well.. This may be some kind of destiny ....
I will work for this issue.
// Fortunately I decided to wait DERBY-1417 in DERBY-1471 :)
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assignee: Tomohito Nakayama
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Updated: (DERBY-550) BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Grégoire Dubois (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=all ]
Grégoire Dubois updated DERBY-550:
----------------------------------
Description:
Using the org.apache.derby.jdbc.ClientDriver driver to access the
Derby database through network, the driver is writting all the file into memory (RAM) before sending
it to the database.
Writting small files (smaller than 5Mo) into the database works fine,
but it is impossible to write big files (40Mo for example, or more), without getting the
exception java.lang.OutOfMemoryError.
The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
import NetNoLedge.Configuration.Configs;
import org.apache.derby.drda.NetworkServerControl;
import java.net.InetAddress;
import java.io.*;
import java.sql.*;
/**
*
* @author greg
*/
public class DerbyServer_JDBC_BLOB_test {
// The unique instance of DerbyServer in the application.
private static DerbyServer_JDBC_BLOB_test derbyServer;
private NetworkServerControl server;
private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
private static final String DERBY_DATABASE_NAME = "Test";
// ###############################################################
// ############### SET HERE THE EXISTING PATH YOU WANT ################
// ###############################################################
private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
// ###############################################################
// ###############################################################
private static int derbyPort = 9157;
private static String userName = "user";
private static String userPassword = "password";
// ###################################################################################
// ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
// ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
// ###################################################################################
private static final File FILE = new File("/home/greg/01.jpg");
// ###################################################################################
// ###################################################################################
/**
* <p>Used to test the server.
*/
public static void main(String args[]) {
try {
DerbyServer_JDBC_BLOB_test.launchServer();
DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
server.start();
System.out.println("Server started");
// After the server has been started, launch a first connection to the database to
// 1) Create the database if it doesn't exist already,
// 2) Create the tables if they don't exist already.
Class.forName(DERBY_JDBC_DRIVER).newInstance();
Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
Statement statement2;
// Create the table "file" if it doesn't already exist.
String [] tableNames={"file"};
boolean exist;
String currentTable;
ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
for (int i=0;i<tableNames.length;i++) {
exist=false;
while (result.next()){
if (tableNames[i].equalsIgnoreCase(result.getString(1)))
exist=true;
}
if (!exist) {
statement2 = connection.createStatement();
statement2.execute("CREATE TABLE file (" +
"file BLOB(2G) NOT NULL)");
connection.commit();
}
result.beforeFirst();
}
System.out.println("Table file created if not created already");
System.out.println("File insertion into BLOB");
FileInputStream inputStream = new FileInputStream(FILE);
PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
preparedStatement .execute();
connection.commit();
System.out.println("File inserted into BLOB");
}
catch (Exception e) {
e.printStackTrace();
}
}
/** Creates a new instance of MckoiServer
* Password is used at the database creation. It will be the database password once created.
*/
private DerbyServer_JDBC_BLOB_test() throws Exception {
System.setProperty("derby.system.home", DERBY_DBMS_PATH);
// Set the server to request an authentification.
System.setProperty("derby.authentication.provider", "BUILTIN");
System.setProperty("derby.connection.requireAuthentication", "true");
// Create a user that can connect to Derby.
System.setProperty("derby.user."+userName, userPassword);
// Set Derby to grant full access to the created user (to all the databases).
System.setProperty("derby.database.fullAccessUsers", userName);
//System.setProperty("derby.system.bootAll", "true");
// See if the 9157 port is already taken.
// Change it if necessary.
boolean isPortFree = false;
while ( !isPortFree ) {
try {
java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
serverTest.close();
serverTest = null;
isPortFree = true;
}
catch (Exception e) {
System.out.println("Port already in use : "+derbyPort);
derbyPort++;
System.out.println("Try with port "+derbyPort);
}
}
server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
}
public static void launchServer() throws Exception {
derbyServer = new DerbyServer_JDBC_BLOB_test();
}
public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
return derbyServer;
}
/**
* <p>Start the server.
*/
public void start() {
try {
server.start(null);
}
catch (Exception e) {
e.printStackTrace(System.err);
}
}
/**
* <p>Stop the server.
*/
public void stop() {
try {
server.shutdown();
}
catch (Exception e) {
e.printStackTrace(System.err);
}
}
}
was:
Using the org.apache.derby.jdbc.ClientDriver driver to access the
Derby database through network, the driver is writting all the file into memory (RAM) before sending
it to the database.
Writting small files (smaller than 5Mo) into the database works fine,
but it is impossible to write big files (40Mo for example, or more), without getting the
exception java.lang.OutOfMemoryError.
The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
Here follows some code that create a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
import NetNoLedge.Configuration.Configs;
import org.apache.derby.drda.NetworkServerControl;
import java.net.InetAddress;
import java.io.*;
import java.sql.*;
/**
*
* @author greg
*/
public class DerbyServer_JDBC_BLOB_test {
// The unique instance of DerbyServer in the application.
private static DerbyServer_JDBC_BLOB_test derbyServer;
private NetworkServerControl server;
private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
private static final String DERBY_DATABASE_NAME = "Test";
// ###############################################################
// ############### SET HERE THE EXISTING PATH YOU WANT ################
// ###############################################################
private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
// ###############################################################
// ###############################################################
private static int derbyPort = 9157;
private static String userName = "user";
private static String userPassword = "password";
// ###################################################################################
// ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
// ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
// ###################################################################################
private static final File FILE = new File("/home/greg/01.jpg");
// ###################################################################################
// ###################################################################################
/**
* <p>Used to test the server.
*/
public static void main(String args[]) {
try {
DerbyServer_JDBC_BLOB_test.launchServer();
DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
server.start();
System.out.println("Server started");
// After the server has been started, launch a first connection to the database to
// 1) Create the database if it doesn't exist already,
// 2) Create the tables if they don't exist already.
Class.forName(DERBY_JDBC_DRIVER).newInstance();
Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
Statement statement2;
// Create the table "file" if it doesn't already exist.
String [] tableNames={"file"};
boolean exist;
String currentTable;
ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
for (int i=0;i<tableNames.length;i++) {
exist=false;
while (result.next()){
if (tableNames[i].equalsIgnoreCase(result.getString(1)))
exist=true;
}
if (!exist) {
statement2 = connection.createStatement();
statement2.execute("CREATE TABLE file (" +
"file BLOB(2G) NOT NULL)");
connection.commit();
}
result.beforeFirst();
}
System.out.println("Table file created if not created already");
System.out.println("File insertion into BLOB");
FileInputStream inputStream = new FileInputStream(FILE);
PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
preparedStatement .execute();
connection.commit();
System.out.println("File inserted into BLOB");
}
catch (Exception e) {
e.printStackTrace();
}
}
/** Creates a new instance of MckoiServer
* Password is used at the database creation. It will be the database password once created.
*/
private DerbyServer_JDBC_BLOB_test() throws Exception {
System.setProperty("derby.system.home", DERBY_DBMS_PATH);
// Set the server to request an authentification.
System.setProperty("derby.authentication.provider", "BUILTIN");
System.setProperty("derby.connection.requireAuthentication", "true");
// Create a user that can connect to Derby.
System.setProperty("derby.user."+userName, userPassword);
// Set Derby to grant full access to the created user (to all the databases).
System.setProperty("derby.database.fullAccessUsers", userName);
//System.setProperty("derby.system.bootAll", "true");
// See if the 9157 port is already taken.
// Change it if necessary.
boolean isPortFree = false;
while ( !isPortFree ) {
try {
java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
serverTest.close();
serverTest = null;
isPortFree = true;
}
catch (Exception e) {
System.out.println("Port already in use : "+derbyPort);
derbyPort++;
System.out.println("Try with port "+derbyPort);
}
}
server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
}
public static void launchServer() throws Exception {
derbyServer = new DerbyServer_JDBC_BLOB_test();
}
public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
return derbyServer;
}
/**
* <p>Start the server.
*/
public void start() {
try {
server.start(null);
}
catch (Exception e) {
e.printStackTrace(System.err);
}
}
/**
* <p>Stop the server.
*/
public void stop() {
try {
server.shutdown();
}
catch (Exception e) {
e.printStackTrace(System.err);
}
}
}
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Type: Bug
> Components: JDBC
> Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12421280 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
However, I think our program may know the length of data in this case,
because our program fail to error at construction time of ByteArrayOutputStream ...
Well...
I think I can create a patch to escape allocating memory again ,
though it is valid only in the case received length of the data is known and correct....
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assigned To: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators: http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] Commented: (DERBY-550) BLOB : java.lang.OutOfMemoryError
with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
Posted by "Tomohito Nakayama (JIRA)" <de...@db.apache.org>.
[ http://issues.apache.org/jira/browse/DERBY-550?page=comments#action_12421279 ]
Tomohito Nakayama commented on DERBY-550:
-----------------------------------------
My consideration is that ByteArrayOutputStream should not be used for large data, especially the size is unknown.
When program retrieve data from ByteArrayOutputStream object, it is needed to allocate another memory other than instance variable byte[]buf of ByteArrayOutputStream.
Because ByteArrayOutputStream.buf does not have same length to the data of whch program passed to ByteArrayOutputStream object.
When the size of data is known, we can escape this problem
because we can fix the size of the ByteArrayOutputStream.buf to the size of data at construction time of ByteArrayOutputStream instance, and we can escape allocating another memory just using ByteArrayOutputStream.buf directly.
> BLOB : java.lang.OutOfMemoryError with network JDBC driver (org.apache.derby.jdbc.ClientDriver)
> -----------------------------------------------------------------------------------------------
>
> Key: DERBY-550
> URL: http://issues.apache.org/jira/browse/DERBY-550
> Project: Derby
> Issue Type: Bug
> Components: JDBC, Network Server
> Affects Versions: 10.1.1.0
> Environment: Any environment.
> Reporter: Grégoire Dubois
> Assigned To: Tomohito Nakayama
> Attachments: BlobOutOfMem.java
>
>
> Using the org.apache.derby.jdbc.ClientDriver driver to access the
> Derby database through network, the driver is writting all the file into memory (RAM) before sending
> it to the database.
> Writting small files (smaller than 5Mo) into the database works fine,
> but it is impossible to write big files (40Mo for example, or more), without getting the
> exception java.lang.OutOfMemoryError.
> The org.apache.derby.jdbc.EmbeddedDriver doesn't have this problem.
> Here follows some code that creates a database, a table, and trys to write a BLOB. 2 parameters are to be changed for the code to work for you : DERBY_DBMS_PATH and FILE
> import NetNoLedge.Configuration.Configs;
> import org.apache.derby.drda.NetworkServerControl;
> import java.net.InetAddress;
> import java.io.*;
> import java.sql.*;
> /**
> *
> * @author greg
> */
> public class DerbyServer_JDBC_BLOB_test {
>
> // The unique instance of DerbyServer in the application.
> private static DerbyServer_JDBC_BLOB_test derbyServer;
>
> private NetworkServerControl server;
>
> private static final String DERBY_JDBC_DRIVER = "org.apache.derby.jdbc.ClientDriver";
> private static final String DERBY_DATABASE_NAME = "Test";
>
> // ###############################################################
> // ############### SET HERE THE EXISTING PATH YOU WANT ################
> // ###############################################################
> private static final String DERBY_DBMS_PATH = "/home/greg/DatabaseTest";
> // ###############################################################
> // ###############################################################
>
>
> private static int derbyPort = 9157;
> private static String userName = "user";
> private static String userPassword = "password";
>
> // ###################################################################################
> // ############# DEFINE HERE THE PATH TO THE FILE YOU WANT TO WRITE INTO THE DATABASE ###########
> // ############# TRY A 100kb-3Mb FILE, AND AFTER A 40Mb OR BIGGER FILE #########################
> // ###################################################################################
> private static final File FILE = new File("/home/greg/01.jpg");
> // ###################################################################################
> // ###################################################################################
>
> /**
> * <p>Used to test the server.
> */
> public static void main(String args[]) {
> try {
> DerbyServer_JDBC_BLOB_test.launchServer();
> DerbyServer_JDBC_BLOB_test server = getUniqueInstance();
> server.start();
> System.out.println("Server started");
>
> // After the server has been started, launch a first connection to the database to
> // 1) Create the database if it doesn't exist already,
> // 2) Create the tables if they don't exist already.
> Class.forName(DERBY_JDBC_DRIVER).newInstance();
> Connection connection = DriverManager.getConnection ("jdbc:derby://localhost:"+derbyPort+"/"+DERBY_DATABASE_NAME+";create=true", userName, userPassword);
> System.out.println("Network JDBC connection to Derby succeded. Database created if not created already.");
>
> Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
> Statement statement2;
> // Create the table "file" if it doesn't already exist.
> String [] tableNames={"file"};
> boolean exist;
> String currentTable;
> ResultSet result = statement.executeQuery("SELECT TABLENAME FROM SYS.SYSTABLES");
> for (int i=0;i<tableNames.length;i++) {
> exist=false;
> while (result.next()){
> if (tableNames[i].equalsIgnoreCase(result.getString(1)))
> exist=true;
> }
>
> if (!exist) {
> statement2 = connection.createStatement();
> statement2.execute("CREATE TABLE file (" +
> "file BLOB(2G) NOT NULL)");
> connection.commit();
> }
> result.beforeFirst();
> }
> System.out.println("Table file created if not created already");
>
> System.out.println("File insertion into BLOB");
> FileInputStream inputStream = new FileInputStream(FILE);
> PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO file(file) VALUES (?)");
> preparedStatement.setBinaryStream(1,inputStream,(int) FILE.length());
> preparedStatement .execute();
> connection.commit();
> System.out.println("File inserted into BLOB");
> }
> catch (Exception e) {
> e.printStackTrace();
> }
> }
>
> /** Creates a new instance of MckoiServer
> * Password is used at the database creation. It will be the database password once created.
> */
> private DerbyServer_JDBC_BLOB_test() throws Exception {
> System.setProperty("derby.system.home", DERBY_DBMS_PATH);
>
> // Set the server to request an authentification.
> System.setProperty("derby.authentication.provider", "BUILTIN");
> System.setProperty("derby.connection.requireAuthentication", "true");
>
> // Create a user that can connect to Derby.
> System.setProperty("derby.user."+userName, userPassword);
>
> // Set Derby to grant full access to the created user (to all the databases).
> System.setProperty("derby.database.fullAccessUsers", userName);
>
> //System.setProperty("derby.system.bootAll", "true");
>
> // See if the 9157 port is already taken.
> // Change it if necessary.
> boolean isPortFree = false;
> while ( !isPortFree ) {
> try {
> java.net.ServerSocket serverTest = new java.net.ServerSocket(derbyPort);
> serverTest.close();
> serverTest = null;
>
> isPortFree = true;
> }
> catch (Exception e) {
> System.out.println("Port already in use : "+derbyPort);
> derbyPort++;
> System.out.println("Try with port "+derbyPort);
> }
> }
>
> server = new NetworkServerControl(InetAddress.getByName("localhost"),derbyPort);
> }
>
> public static void launchServer() throws Exception {
> derbyServer = new DerbyServer_JDBC_BLOB_test();
> }
>
> public static DerbyServer_JDBC_BLOB_test getUniqueInstance() {
> return derbyServer;
> }
>
> /**
> * <p>Start the server.
> */
> public void start() {
> try {
> server.start(null);
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
>
> /**
> * <p>Stop the server.
> */
> public void stop() {
> try {
> server.shutdown();
> }
> catch (Exception e) {
> e.printStackTrace(System.err);
> }
> }
> }
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators: http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see: http://www.atlassian.com/software/jira