You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ofbiz.apache.org by do...@apache.org on 2011/06/26 20:07:19 UTC
svn commit: r1139851 -
/ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Author: doogie
Date: Sun Jun 26 18:07:18 2011
New Revision: 1139851
URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
Log:
FEATURE: When doing bulk creation of tables, foreign key indices,
and declared indices, do the work in parallel. Also, when the database
does not support wildcard looking of primary key info, do the per-table
lookup in parallel.
Modified:
ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Modified: ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java?rev=1139851&r1=1139850&r2=1139851&view=diff
==============================================================================
--- ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java (original)
+++ ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java Sun Jun 26 18:07:18 2011
@@ -42,6 +42,7 @@ import javolution.util.FastMap;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
+import org.ofbiz.base.concurrent.ExecutionPool;
import org.ofbiz.base.util.Debug;
import org.ofbiz.base.util.UtilTimer;
import org.ofbiz.base.util.UtilValidate;
@@ -232,6 +233,7 @@ public class DatabaseUtil {
Debug.logError(message, module);
return;
}
+ List<Future<CreateTableCallable>> tableFutures = FastList.newInstance();
for (ModelEntity entity: modelEntityList) {
curEnt++;
@@ -420,20 +422,13 @@ public class DatabaseUtil {
if (addMissing) {
// create the table
- String errMsg = createTable(entity, modelEntities, false);
- if (UtilValidate.isNotEmpty(errMsg)) {
- message = "Could not create table [" + tableName + "]: " + errMsg;
- Debug.logError(message, module);
- if (messages != null) messages.add(message);
- } else {
- entitiesAdded.add(entity);
- message = "Created table [" + tableName + "]";
- Debug.logImportant(message, module);
- if (messages != null) messages.add(message);
- }
+ tableFutures.add(submitWork(new CreateTableCallable(entity, modelEntities, tableName)));
}
}
}
+ for (CreateTableCallable tableCallable: ExecutionPool.getAllFutures(tableFutures)) {
+ tableCallable.updateData(messages, entitiesAdded);
+ }
timer.timerString("After Individual Table/Column Check");
@@ -447,11 +442,20 @@ public class DatabaseUtil {
// for each newly added table, add fk indices
if (datasourceInfo.useFkIndices) {
int totalFkIndices = 0;
+ List<Future<AbstractCountingCallable>> fkIndicesFutures = FastList.newInstance();
for (ModelEntity curEntity: entitiesAdded) {
if (curEntity.getRelationsOneSize() > 0) {
- totalFkIndices += this.createForeignKeyIndices(curEntity, datasourceInfo.constraintNameClipLength, messages);
+ fkIndicesFutures.add(submitWork(new AbstractCountingCallable(curEntity, modelEntities) {
+ public AbstractCountingCallable call() throws Exception {
+ count = createForeignKeyIndices(entity, datasourceInfo.constraintNameClipLength, messages);
+ return this;
+ }
+ }));
}
}
+ for (AbstractCountingCallable fkIndicesCallable: ExecutionPool.getAllFutures(fkIndicesFutures)) {
+ totalFkIndices += fkIndicesCallable.updateData(messages);
+ }
if (totalFkIndices > 0) Debug.logImportant("==== TOTAL Foreign Key Indices Created: " + totalFkIndices, module);
}
@@ -467,11 +471,21 @@ public class DatabaseUtil {
// for each newly added table, add declared indexes
if (datasourceInfo.useIndices) {
int totalDis = 0;
+ List<Future<AbstractCountingCallable>> disFutures = FastList.newInstance();
for (ModelEntity curEntity: entitiesAdded) {
if (curEntity.getIndexesSize() > 0) {
- totalDis += this.createDeclaredIndices(curEntity, messages);
+ disFutures.add(submitWork(new AbstractCountingCallable(curEntity, modelEntities) {
+ public AbstractCountingCallable call() throws Exception {
+ count = createDeclaredIndices(entity, messages);
+ return this;
+ }
+ }));
+
}
}
+ for (AbstractCountingCallable disCallable: ExecutionPool.getAllFutures(disFutures)) {
+ totalDis += disCallable.updateData(messages);
+ }
if (totalDis > 0) Debug.logImportant("==== TOTAL Declared Indices Created: " + totalDis, module);
}
@@ -1188,6 +1202,17 @@ public class DatabaseUtil {
return tableNames;
}
+ private AbstractCountingCallable createPrimaryKeyFetcher(final DatabaseMetaData dbData, final String lookupSchemaName, final boolean needsUpperCase, final Map<String, Map<String, ColumnCheckInfo>> colInfo, final Collection<String> messages, final String curTable) {
+ return new AbstractCountingCallable(null, null) {
+ public AbstractCountingCallable call() throws Exception {
+ Debug.logInfo("Fetching primary keys for " + curTable, module);
+ ResultSet rsPks = dbData.getPrimaryKeys(null, lookupSchemaName, curTable);
+ count = checkPrimaryKeyInfo(rsPks, lookupSchemaName, needsUpperCase, colInfo, messages);
+ return this;
+ }
+ };
+ }
+
public Map<String, Map<String, ColumnCheckInfo>> getColumnInfo(Set<String> tableNames, boolean getPks, Collection<String> messages) {
// if there are no tableNames, don't even try to get the columns
if (tableNames.size() == 0) {
@@ -1317,10 +1342,13 @@ public class DatabaseUtil {
}
if (pkCount == 0) {
Debug.logInfo("Searching in " + tableNames.size() + " tables for primary key fields ...", module);
+ List<Future<AbstractCountingCallable>> pkFetcherFutures = FastList.newInstance();
for (String curTable: tableNames) {
curTable = curTable.substring(curTable.indexOf('.') + 1); //cut off schema name
- ResultSet rsPks = dbData.getPrimaryKeys(null, lookupSchemaName, curTable);
- pkCount += checkPrimaryKeyInfo(rsPks, lookupSchemaName, needsUpperCase, colInfo, messages);
+ pkFetcherFutures.add(submitWork(createPrimaryKeyFetcher(dbData, lookupSchemaName, needsUpperCase, colInfo, messages, curTable)));
+ }
+ for (AbstractCountingCallable pkFetcherCallable: ExecutionPool.getAllFutures(pkFetcherFutures)) {
+ pkCount += pkFetcherCallable.updateData(messages);
}
}
@@ -1662,6 +1690,66 @@ public class DatabaseUtil {
return indexInfo;
}
+ private class CreateTableCallable implements Callable<CreateTableCallable> {
+ private final ModelEntity entity;
+ private final Map<String, ModelEntity> modelEntities;
+ private final String tableName;
+ private String message;
+ private boolean success;
+
+ protected CreateTableCallable(ModelEntity entity, Map<String, ModelEntity> modelEntities, String tableName) {
+ this.entity = entity;
+ this.modelEntities = modelEntities;
+ this.tableName = tableName;
+ }
+
+ public CreateTableCallable call() throws Exception {
+ String errMsg = createTable(entity, modelEntities, false);
+ if (UtilValidate.isNotEmpty(errMsg)) {
+ this.success = false;
+ this.message = "Could not create table [" + tableName + "]: " + errMsg;
+ Debug.logError(this.message, module);
+ } else {
+ this.success = true;
+ this.message = "Created table [" + tableName + "]";
+ Debug.logImportant(this.message, module);
+ }
+ return this;
+ }
+
+ protected void updateData(Collection<String> messages, List<ModelEntity> entitiesAdded) {
+ if (this.success) {
+ entitiesAdded.add(entity);
+ if (messages != null) {
+ messages.add(this.message);
+ }
+ } else {
+ if (messages != null) {
+ messages.add(this.message);
+ }
+ }
+ }
+ }
+
+ private abstract class AbstractCountingCallable<T extends AbstractCountingCallable<T>> implements Callable<T> {
+ protected final ModelEntity entity;
+ protected final Map<String, ModelEntity> modelEntities;
+ protected final List<String> messages = FastList.newInstance();
+ protected int count;
+
+ protected AbstractCountingCallable(ModelEntity entity, Map<String, ModelEntity> modelEntities) {
+ this.entity = entity;
+ this.modelEntities = modelEntities;
+ }
+
+ protected int updateData(Collection<String> messages) {
+ if (messages != null && UtilValidate.isNotEmpty(this.messages)) {
+ messages.addAll(messages);
+ }
+ return count;
+ }
+ }
+
/* ====================================================================== */
/* ====================================================================== */
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adam Heath <do...@brainfood.com>.
On 06/26/2011 03:14 PM, Adrian Crum wrote:
> Some advice on using this feature (based on my previous experience with
> multi-threaded table creation): Start off with small values - like 1 or
> 2 on a multi-CPU machine. Creating more threads does not equal better
> performance - there is a threshold where more threads will slow things
> down.
The astute will notice that this feature is not the same as the one I
mistakenly committed ages ago; this one is simpler.
>
> -Adrian
>
> On 6/26/2011 7:22 PM, Adam Heath wrote:
>> On 06/26/2011 01:07 PM, doogie@apache.org wrote:
>>> Author: doogie
>>> Date: Sun Jun 26 18:07:18 2011
>>> New Revision: 1139851
>>>
>>> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
>>> Log:
>>> FEATURE: When doing bulk creation of tables, foreign key indices,
>>> and declared indices, do the work in parallel. Also, when the database
>>> does not support wildcard looking of primary key info, do the per-table
>>> lookup in parallel.
>>>
>>> Modified:
>>> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
>>
>> This is a major feature. By default, it's not actually going to do
>> anything in parallel. You need to set a 'max-worker-pool-size' to some
>> value(positive, or negative, which will then be multiplied by the
>> number of cpus), to take advantage of this.
>>
>> Eventually, this per-datasource thread pool could be used to limit
>> concurrent queries, but that is not something I've done yet.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adam Heath <do...@brainfood.com>.
On 06/26/2011 04:19 PM, Adrian Crum wrote:
> I'm not sure changing the value when using a RAM disk would make much
> difference. It seems to me multi-threading would benefit a system that
> has I/O bottlenecks.
By running on a ram disk, IO contention is much reduced(of course).
Then using a high-core machine, allows me to test that the actual code
has no issues.
> Also, the JDBC driver itself could be a bottleneck. That's why I had a
> separate thread reading the entity model XML files - so one thread could
> be parsing XML while another one is waiting for disk I/O.
All that is good. I can verify that running these things in parallel
speeds up both derby and oracle. It was initially developed on
oracle, which doesn't have the aformentioned deadlock issue with
foreign-keys.
We can come back to your staged idea. I have a helper class
locally(without test cases yet), for batched staging, and shared
pooling(hard to explain this latter part).
I also want to speed up service def parsing, ecas can run in
parallel(they have to anyways, considering that there could have been
multiple calling threads initially).
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adrian Crum <ad...@sandglass-software.com>.
I'm not sure changing the value when using a RAM disk would make much
difference. It seems to me multi-threading would benefit a system that
has I/O bottlenecks.
Also, the JDBC driver itself could be a bottleneck. That's why I had a
separate thread reading the entity model XML files - so one thread could
be parsing XML while another one is waiting for disk I/O.
-Adrian
On 6/26/2011 9:24 PM, Adam Heath wrote:
> On 06/26/2011 03:14 PM, Adrian Crum wrote:
>> Some advice on using this feature (based on my previous experience with
>> multi-threaded table creation): Start off with small values - like 1 or
>> 2 on a multi-CPU machine. Creating more threads does not equal better
>> performance - there is a threshold where more threads will slow things
>> down.
>
> I tested this on a 6-core machine, with the entire ofbiz checkout
> running on a ram disk. Speed was nice, but I didn't do any checking
> to see which values were faster. I didn't have any problems setting
> the value to -1(which caused a pool size of 6), except for the
> aformentioned foreign-key issue; as such, that change is not in this
> commit.
>
>>
>> -Adrian
>>
>> On 6/26/2011 7:22 PM, Adam Heath wrote:
>>> On 06/26/2011 01:07 PM, doogie@apache.org wrote:
>>>> Author: doogie
>>>> Date: Sun Jun 26 18:07:18 2011
>>>> New Revision: 1139851
>>>>
>>>> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
>>>> Log:
>>>> FEATURE: When doing bulk creation of tables, foreign key indices,
>>>> and declared indices, do the work in parallel. Also, when the database
>>>> does not support wildcard looking of primary key info, do the
>>>> per-table
>>>> lookup in parallel.
>>>>
>>>> Modified:
>>>> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
>>>>
>>>
>>> This is a major feature. By default, it's not actually going to do
>>> anything in parallel. You need to set a 'max-worker-pool-size' to some
>>> value(positive, or negative, which will then be multiplied by the
>>> number of cpus), to take advantage of this.
>>>
>>> Eventually, this per-datasource thread pool could be used to limit
>>> concurrent queries, but that is not something I've done yet.
>
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adrian Crum <ad...@sandglass-software.com>.
Okay, that makes sense. Thanks!
-Adrian
On 6/26/2011 9:43 PM, Adam Heath wrote:
> On 06/26/2011 03:41 PM, Adrian Crum wrote:
>> What would +2 do? Or is the multiplier an absolute value?
>
> 0=1
> 1=1
> 2=2
> 3=3
> -1=abs(-1)*num-cpus
> -2=abs(-2)*num-cpus
> -5=abs(-5)*num-cpus
>
> and so on.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by "doogie@brainfood.com" <do...@brainfood.com>.
Adrian Crum <ad...@sandglass-software.com> wrote:
>A setting of -2 on my dual-CPU, high speed RAID array development
>machine resulted in a measurable reduction in table creation time. Good
>job!
>
>-Adrian
>
>On 6/26/2011 9:43 PM, Adam Heath wrote:
>> On 06/26/2011 03:41 PM, Adrian Crum wrote:
>>> What would +2 do? Or is the multiplier an absolute value?
>>
>> 0=1
>> 1=1
>> 2=2
>> 3=3
>> -1=abs(-1)*num-cpus
>> -2=abs(-2)*num-cpus
>> -5=abs(-5)*num-cpus
>>
>> and so on.
That is not what the final value should be; eventually the per-datasource thread pool will be used to reduce database overloading. In these cases, -1 is the right value, because other threads will be doing page rendering.
--
Sent from my Android phone with K-9 Mail. Please excuse my brevity.
Re: Multi-threaded DB creation
Posted by Adam Heath <do...@brainfood.com>.
On 06/30/2011 08:49 AM, Adrian Crum wrote:
> On the other hand, I tried -1 and 2 settings on an 4 CPU system with a
> single hard disk and they made no difference.
How much L2-cache on the cpu(s) was there? Java quite often is larger
than the L2-cache, which then means it's waiting on slower memory.
> On 6/26/2011 10:32 PM, Adrian Crum wrote:
>> A setting of -2 on my dual-CPU, high speed RAID array development
>> machine resulted in a measurable reduction in table creation time.
>> Good job!
>>
>> -Adrian
>>
>> On 6/26/2011 9:43 PM, Adam Heath wrote:
>>> On 06/26/2011 03:41 PM, Adrian Crum wrote:
>>>> What would +2 do? Or is the multiplier an absolute value?
>>>
>>> 0=1
>>> 1=1
>>> 2=2
>>> 3=3
>>> -1=abs(-1)*num-cpus
>>> -2=abs(-2)*num-cpus
>>> -5=abs(-5)*num-cpus
>>>
>>> and so on.
Re: Multi-threaded DB creation (was: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java)
Posted by Adrian Crum <ad...@sandglass-software.com>.
On the other hand, I tried -1 and 2 settings on an 4 CPU system with a
single hard disk and they made no difference.
-Adrian
On 6/26/2011 10:32 PM, Adrian Crum wrote:
> A setting of -2 on my dual-CPU, high speed RAID array development
> machine resulted in a measurable reduction in table creation time.
> Good job!
>
> -Adrian
>
> On 6/26/2011 9:43 PM, Adam Heath wrote:
>> On 06/26/2011 03:41 PM, Adrian Crum wrote:
>>> What would +2 do? Or is the multiplier an absolute value?
>>
>> 0=1
>> 1=1
>> 2=2
>> 3=3
>> -1=abs(-1)*num-cpus
>> -2=abs(-2)*num-cpus
>> -5=abs(-5)*num-cpus
>>
>> and so on.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by "doogie@brainfood.com" <do...@brainfood.com>.
Adrian Crum <ad...@sandglass-software.com> wrote:
>A setting of -2 on my dual-CPU, high speed RAID array development
>machine resulted in a measurable reduction in table creation time. Good
>job!
>
>-Adrian
>
>On 6/26/2011 9:43 PM, Adam Heath wrote:
>> On 06/26/2011 03:41 PM, Adrian Crum wrote:
>>> What would +2 do? Or is the multiplier an absolute value?
>>
>> 0=1
>> 1=1
>> 2=2
>> 3=3
>> -1=abs(-1)*num-cpus
>> -2=abs(-2)*num-cpus
>> -5=abs(-5)*num-cpus
>>
>> and so on.
As I mentioned before, I have tested this on oracle(initial development) and Derby, and both are faster. Haven't done postgres or mysql. I have further plans to load multiple delegators in parallel.
--
Sent from my Android phone with K-9 Mail. Please excuse my brevity.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adrian Crum <ad...@sandglass-software.com>.
A setting of -2 on my dual-CPU, high speed RAID array development
machine resulted in a measurable reduction in table creation time. Good job!
-Adrian
On 6/26/2011 9:43 PM, Adam Heath wrote:
> On 06/26/2011 03:41 PM, Adrian Crum wrote:
>> What would +2 do? Or is the multiplier an absolute value?
>
> 0=1
> 1=1
> 2=2
> 3=3
> -1=abs(-1)*num-cpus
> -2=abs(-2)*num-cpus
> -5=abs(-5)*num-cpus
>
> and so on.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adam Heath <do...@brainfood.com>.
On 06/26/2011 03:41 PM, Adrian Crum wrote:
> What would +2 do? Or is the multiplier an absolute value?
0=1
1=1
2=2
3=3
-1=abs(-1)*num-cpus
-2=abs(-2)*num-cpus
-5=abs(-5)*num-cpus
and so on.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adrian Crum <ad...@sandglass-software.com>.
What would +2 do? Or is the multiplier an absolute value?
-Adrian
On 6/26/2011 9:38 PM, Adam Heath wrote:
> On 06/26/2011 03:32 PM, Adrian Crum wrote:
>> Okay, that -1 setting seems confusing. Maybe we need a better
>> explanation of how the setting works.
>
> You could specify -2, to mean twice the number of cpus, -5 to mean 5
> times the number of cpus. I couldn't think of a better way to do that.
>
>>
>> -Adrian
>>
>> On 6/26/2011 9:24 PM, Adam Heath wrote:
>>> On 06/26/2011 03:14 PM, Adrian Crum wrote:
>>>> Some advice on using this feature (based on my previous experience
>>>> with
>>>> multi-threaded table creation): Start off with small values - like
>>>> 1 or
>>>> 2 on a multi-CPU machine. Creating more threads does not equal better
>>>> performance - there is a threshold where more threads will slow things
>>>> down.
>>>
>>> I tested this on a 6-core machine, with the entire ofbiz checkout
>>> running on a ram disk. Speed was nice, but I didn't do any checking to
>>> see which values were faster. I didn't have any problems setting the
>>> value to -1(which caused a pool size of 6), except for the
>>> aformentioned foreign-key issue; as such, that change is not in this
>>> commit.
>>>
>>>>
>>>> -Adrian
>>>>
>>>> On 6/26/2011 7:22 PM, Adam Heath wrote:
>>>>> On 06/26/2011 01:07 PM, doogie@apache.org wrote:
>>>>>> Author: doogie
>>>>>> Date: Sun Jun 26 18:07:18 2011
>>>>>> New Revision: 1139851
>>>>>>
>>>>>> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
>>>>>> Log:
>>>>>> FEATURE: When doing bulk creation of tables, foreign key indices,
>>>>>> and declared indices, do the work in parallel. Also, when the
>>>>>> database
>>>>>> does not support wildcard looking of primary key info, do the
>>>>>> per-table
>>>>>> lookup in parallel.
>>>>>>
>>>>>> Modified:
>>>>>> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
>>>>>>
>>>>>>
>>>>>
>>>>> This is a major feature. By default, it's not actually going to do
>>>>> anything in parallel. You need to set a 'max-worker-pool-size' to
>>>>> some
>>>>> value(positive, or negative, which will then be multiplied by the
>>>>> number of cpus), to take advantage of this.
>>>>>
>>>>> Eventually, this per-datasource thread pool could be used to limit
>>>>> concurrent queries, but that is not something I've done yet.
>>>
>
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adam Heath <do...@brainfood.com>.
On 06/26/2011 03:32 PM, Adrian Crum wrote:
> Okay, that -1 setting seems confusing. Maybe we need a better
> explanation of how the setting works.
You could specify -2, to mean twice the number of cpus, -5 to mean 5
times the number of cpus. I couldn't think of a better way to do that.
>
> -Adrian
>
> On 6/26/2011 9:24 PM, Adam Heath wrote:
>> On 06/26/2011 03:14 PM, Adrian Crum wrote:
>>> Some advice on using this feature (based on my previous experience with
>>> multi-threaded table creation): Start off with small values - like 1 or
>>> 2 on a multi-CPU machine. Creating more threads does not equal better
>>> performance - there is a threshold where more threads will slow things
>>> down.
>>
>> I tested this on a 6-core machine, with the entire ofbiz checkout
>> running on a ram disk. Speed was nice, but I didn't do any checking to
>> see which values were faster. I didn't have any problems setting the
>> value to -1(which caused a pool size of 6), except for the
>> aformentioned foreign-key issue; as such, that change is not in this
>> commit.
>>
>>>
>>> -Adrian
>>>
>>> On 6/26/2011 7:22 PM, Adam Heath wrote:
>>>> On 06/26/2011 01:07 PM, doogie@apache.org wrote:
>>>>> Author: doogie
>>>>> Date: Sun Jun 26 18:07:18 2011
>>>>> New Revision: 1139851
>>>>>
>>>>> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
>>>>> Log:
>>>>> FEATURE: When doing bulk creation of tables, foreign key indices,
>>>>> and declared indices, do the work in parallel. Also, when the database
>>>>> does not support wildcard looking of primary key info, do the
>>>>> per-table
>>>>> lookup in parallel.
>>>>>
>>>>> Modified:
>>>>> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
>>>>>
>>>>
>>>> This is a major feature. By default, it's not actually going to do
>>>> anything in parallel. You need to set a 'max-worker-pool-size' to some
>>>> value(positive, or negative, which will then be multiplied by the
>>>> number of cpus), to take advantage of this.
>>>>
>>>> Eventually, this per-datasource thread pool could be used to limit
>>>> concurrent queries, but that is not something I've done yet.
>>
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adrian Crum <ad...@sandglass-software.com>.
Okay, that -1 setting seems confusing. Maybe we need a better
explanation of how the setting works.
-Adrian
On 6/26/2011 9:24 PM, Adam Heath wrote:
> On 06/26/2011 03:14 PM, Adrian Crum wrote:
>> Some advice on using this feature (based on my previous experience with
>> multi-threaded table creation): Start off with small values - like 1 or
>> 2 on a multi-CPU machine. Creating more threads does not equal better
>> performance - there is a threshold where more threads will slow things
>> down.
>
> I tested this on a 6-core machine, with the entire ofbiz checkout
> running on a ram disk. Speed was nice, but I didn't do any checking
> to see which values were faster. I didn't have any problems setting
> the value to -1(which caused a pool size of 6), except for the
> aformentioned foreign-key issue; as such, that change is not in this
> commit.
>
>>
>> -Adrian
>>
>> On 6/26/2011 7:22 PM, Adam Heath wrote:
>>> On 06/26/2011 01:07 PM, doogie@apache.org wrote:
>>>> Author: doogie
>>>> Date: Sun Jun 26 18:07:18 2011
>>>> New Revision: 1139851
>>>>
>>>> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
>>>> Log:
>>>> FEATURE: When doing bulk creation of tables, foreign key indices,
>>>> and declared indices, do the work in parallel. Also, when the database
>>>> does not support wildcard looking of primary key info, do the
>>>> per-table
>>>> lookup in parallel.
>>>>
>>>> Modified:
>>>> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
>>>>
>>>
>>> This is a major feature. By default, it's not actually going to do
>>> anything in parallel. You need to set a 'max-worker-pool-size' to some
>>> value(positive, or negative, which will then be multiplied by the
>>> number of cpus), to take advantage of this.
>>>
>>> Eventually, this per-datasource thread pool could be used to limit
>>> concurrent queries, but that is not something I've done yet.
>
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adam Heath <do...@brainfood.com>.
On 06/26/2011 03:14 PM, Adrian Crum wrote:
> Some advice on using this feature (based on my previous experience with
> multi-threaded table creation): Start off with small values - like 1 or
> 2 on a multi-CPU machine. Creating more threads does not equal better
> performance - there is a threshold where more threads will slow things
> down.
I tested this on a 6-core machine, with the entire ofbiz checkout
running on a ram disk. Speed was nice, but I didn't do any checking
to see which values were faster. I didn't have any problems setting
the value to -1(which caused a pool size of 6), except for the
aformentioned foreign-key issue; as such, that change is not in this
commit.
>
> -Adrian
>
> On 6/26/2011 7:22 PM, Adam Heath wrote:
>> On 06/26/2011 01:07 PM, doogie@apache.org wrote:
>>> Author: doogie
>>> Date: Sun Jun 26 18:07:18 2011
>>> New Revision: 1139851
>>>
>>> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
>>> Log:
>>> FEATURE: When doing bulk creation of tables, foreign key indices,
>>> and declared indices, do the work in parallel. Also, when the database
>>> does not support wildcard looking of primary key info, do the per-table
>>> lookup in parallel.
>>>
>>> Modified:
>>> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
>>
>> This is a major feature. By default, it's not actually going to do
>> anything in parallel. You need to set a 'max-worker-pool-size' to some
>> value(positive, or negative, which will then be multiplied by the
>> number of cpus), to take advantage of this.
>>
>> Eventually, this per-datasource thread pool could be used to limit
>> concurrent queries, but that is not something I've done yet.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adrian Crum <ad...@sandglass-software.com>.
Some advice on using this feature (based on my previous experience with
multi-threaded table creation): Start off with small values - like 1 or
2 on a multi-CPU machine. Creating more threads does not equal better
performance - there is a threshold where more threads will slow things down.
-Adrian
On 6/26/2011 7:22 PM, Adam Heath wrote:
> On 06/26/2011 01:07 PM, doogie@apache.org wrote:
>> Author: doogie
>> Date: Sun Jun 26 18:07:18 2011
>> New Revision: 1139851
>>
>> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
>> Log:
>> FEATURE: When doing bulk creation of tables, foreign key indices,
>> and declared indices, do the work in parallel. Also, when the database
>> does not support wildcard looking of primary key info, do the per-table
>> lookup in parallel.
>>
>> Modified:
>>
>> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
>
> This is a major feature. By default, it's not actually going to do
> anything in parallel. You need to set a 'max-worker-pool-size' to
> some value(positive, or negative, which will then be multiplied by the
> number of cpus), to take advantage of this.
>
> Eventually, this per-datasource thread pool could be used to limit
> concurrent queries, but that is not something I've done yet.
Re: svn commit: r1139851 - /ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Posted by Adam Heath <do...@brainfood.com>.
On 06/26/2011 01:07 PM, doogie@apache.org wrote:
> Author: doogie
> Date: Sun Jun 26 18:07:18 2011
> New Revision: 1139851
>
> URL: http://svn.apache.org/viewvc?rev=1139851&view=rev
> Log:
> FEATURE: When doing bulk creation of tables, foreign key indices,
> and declared indices, do the work in parallel. Also, when the database
> does not support wildcard looking of primary key info, do the per-table
> lookup in parallel.
>
> Modified:
> ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
This is a major feature. By default, it's not actually going to do
anything in parallel. You need to set a 'max-worker-pool-size' to
some value(positive, or negative, which will then be multiplied by the
number of cpus), to take advantage of this.
Eventually, this per-datasource thread pool could be used to limit
concurrent queries, but that is not something I've done yet.