You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ofbiz.apache.org by do...@apache.org on 2010/06/01 23:46:25 UTC
svn commit: r950255 -
/ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Author: doogie
Date: Tue Jun 1 21:46:24 2010
New Revision: 950255
URL: http://svn.apache.org/viewvc?rev=950255&view=rev
Log:
Multi-threaded creation of tables, foreign key indices, and declared
indices.
Modified:
ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
Modified: ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java?rev=950255&r1=950254&r2=950255&view=diff
==============================================================================
--- ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java (original)
+++ ofbiz/trunk/framework/entity/src/org/ofbiz/entity/jdbc/DatabaseUtil.java Tue Jun 1 21:46:24 2010
@@ -31,7 +31,15 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
import javolution.util.FastList;
import javolution.util.FastMap;
@@ -55,6 +63,7 @@ import org.ofbiz.entity.model.ModelKeyMa
import org.ofbiz.entity.model.ModelRelation;
import org.ofbiz.entity.model.ModelViewEntity;
import org.ofbiz.entity.transaction.TransactionUtil;
+import org.ofbiz.base.concurrent.ExecutionPool;
/**
* Utilities for Entity Database Maintenance
@@ -142,62 +151,58 @@ public class DatabaseUtil {
checkDb(modelEntities, null, messages, datasourceInfo.checkPrimaryKeysOnStart, (datasourceInfo.useFks && datasourceInfo.checkForeignKeysOnStart), (datasourceInfo.useFkIndices && datasourceInfo.checkFkIndicesOnStart), addMissing);
}
- public void checkDb(Map<String, ModelEntity> modelEntities, List<String> colWrongSize, List<String> messages, boolean checkPks, boolean checkFks, boolean checkFkIdx, boolean addMissing) {
- if (isLegacy) {
- throw new RuntimeException("Cannot run checkDb on a legacy database connection; configure a database helper (entityengine.xml)");
- }
- UtilTimer timer = new UtilTimer();
- timer.timerString("Start - Before Get Database Meta Data");
-
- // get ALL tables from this database
- TreeSet<String> tableNames = this.getTableNames(messages);
- TreeSet<String> fkTableNames = tableNames == null ? null : new TreeSet<String>(tableNames);
- TreeSet<String> indexTableNames = tableNames == null ? null : new TreeSet<String>(tableNames);
+ private abstract class DBFuture<F extends DBFuture> implements Callable<F> {
+ protected final Map<String, ModelEntity> modelEntities;
+ protected final Set<String> tableNames;
+ protected final UtilTimer timer;
+ protected final Map<String, Map<String, ColumnCheckInfo>> colInfo;
+ protected final List<String> messages = FastList.newInstance();
- if (tableNames == null) {
- String message = "Could not get table name information from the database, aborting.";
- if (messages != null) messages.add(message);
- Debug.logError(message, module);
- return;
+ protected DBFuture(Map<String, ModelEntity> modelEntities, UtilTimer timer, Set<String> tableNames, Map<String, Map<String, ColumnCheckInfo>> colInfo) {
+ this.modelEntities = modelEntities;
+ this.timer = timer;
+ this.tableNames = tableNames;
+ this.colInfo = colInfo;
}
- timer.timerString("After Get All Table Names");
- // get ALL column info, put into hashmap by table name
- Map<String, Map<String, ColumnCheckInfo>> colInfo = this.getColumnInfo(tableNames, checkPks, messages);
- if (colInfo == null) {
- String message = "Could not get column information from the database, aborting.";
- if (messages != null) messages.add(message);
- Debug.logError(message, module);
- return;
+ protected void updateData(List<String> messages) {
+ if (messages != null) messages.addAll(this.messages);
}
- timer.timerString("After Get All Column Info");
-
- // -make sure all entities have a corresponding table
- // -list all tables that do not have a corresponding entity
- // -display message if number of table columns does not match number of entity fields
- // -list all columns that do not have a corresponding field
- // -make sure each corresponding column is of the correct type
- // -list all fields that do not have a corresponding column
+ }
- timer.timerString("Before Individual Table/Column Check");
+ private static <F> Collection<F> getAllFutures(Collection<Future<F>> futureList) {
+ List<F> result = FastList.newInstance();
+ for (Future<F> future: futureList) {
+ try {
+ result.add(future.get());
+ } catch (ExecutionException e) {
+ Debug.logError(e, module);
+ } catch (InterruptedException e) {
+ Debug.logError(e, module);
+ }
+ }
+ return result;
+ }
- ArrayList<ModelEntity> modelEntityList = new ArrayList<ModelEntity>(modelEntities.values());
- // sort using compareTo method on ModelEntity
- Collections.sort(modelEntityList);
- int curEnt = 0;
- int totalEnt = modelEntityList.size();
- List<ModelEntity> entitiesAdded = FastList.newInstance();
- for (ModelEntity entity: modelEntityList) {
- curEnt++;
+ private class TableFuture extends DBFuture<TableFuture> {
+ private final int curEnt;
+ private final int totalEnt;
+ private final ModelEntity entity;
+ private final boolean checkPks;
+ private final boolean addMissing;
+ private final List<String> colWrongSize = FastList.newInstance();
+ private boolean entityAdded = false;
- // if this is a view entity, do not check it...
- if (entity instanceof ModelViewEntity) {
- String entMessage = "(" + timer.timeSinceLast() + "ms) NOT Checking #" + curEnt + "/" + totalEnt + " View Entity " + entity.getEntityName();
- Debug.logVerbose(entMessage, module);
- if (messages != null) messages.add(entMessage);
- continue;
- }
+ protected TableFuture(Map<String, ModelEntity> modelEntities, UtilTimer timer, Set<String> tableNames, Map<String, Map<String, ColumnCheckInfo>> colInfo, int curEnt, int totalEnt, ModelEntity entity, boolean checkPks, boolean addMissing) {
+ super(modelEntities, timer, tableNames, colInfo);
+ this.curEnt = curEnt;
+ this.totalEnt = totalEnt;
+ this.entity = entity;
+ this.checkPks = checkPks;
+ this.addMissing = addMissing;
+ }
+ public TableFuture call() {
String entMessage = "(" + timer.timeSinceLast() + "ms) Checking #" + curEnt + "/" + totalEnt +
" Entity " + entity.getEntityName() + " with table " + entity.getTableName(datasourceInfo);
@@ -236,7 +241,6 @@ public class DatabaseUtil {
int openParen = fullTypeStr.indexOf('(');
int closeParen = fullTypeStr.indexOf(')');
int comma = fullTypeStr.indexOf(',');
-
if (openParen > 0 && closeParen > 0 && closeParen > openParen) {
typeName = fullTypeStr.substring(0, openParen);
if (comma > 0 && comma > openParen && comma < closeParen) {
@@ -264,7 +268,6 @@ public class DatabaseUtil {
} else {
typeName = fullTypeStr;
}
-
// override the default typeName with the sqlTypeAlias if it is specified
if (UtilValidate.isNotEmpty(modelFieldType.getSqlTypeAlias())) {
typeName = modelFieldType.getSqlTypeAlias();
@@ -368,15 +371,132 @@ public class DatabaseUtil {
Debug.logError(message, module);
if (messages != null) messages.add(message);
} else {
- entitiesAdded.add(entity);
+ entityAdded = true;
message = "Created table [" + entity.getTableName(datasourceInfo) + "]";
Debug.logImportant(message, module);
if (messages != null) messages.add(message);
}
}
}
+ return this;
+ }
+
+ protected void updateData(List<String> messages, List<ModelEntity> entitiesAdded) {
+ updateData(messages);
+ if (entityAdded) {
+ entitiesAdded.add(entity);
+ }
+ }
+ }
+
+ private abstract class CountingFuture<F extends CountingFuture<F>> implements Callable<F> {
+ protected final AtomicInteger count;
+ protected final List<String> messages = FastList.newInstance();
+
+ protected CountingFuture(AtomicInteger count) {
+ this.count = count;
+ }
+
+ protected void updateData(List<String> messages) {
+ if (messages != null) messages.addAll(this.messages);
+ }
+ }
+
+ private class ForeignKeyIndexFuture extends CountingFuture<ForeignKeyIndexFuture> {
+ private final ModelEntity entity;
+ private final int constraintNameClipLength;
+
+ protected ForeignKeyIndexFuture(AtomicInteger count, ModelEntity entity, int constraintNameClipLength) {
+ super(count);
+ this.entity = entity;
+ this.constraintNameClipLength = constraintNameClipLength;
+ }
+
+ public ForeignKeyIndexFuture call() {
+ count.addAndGet(createForeignKeyIndices(entity, constraintNameClipLength, messages));
+ return this;
+ }
+ }
+
+ private class DeclaredIndexFuture extends CountingFuture<DeclaredIndexFuture> {
+ private final ModelEntity entity;
+
+ protected DeclaredIndexFuture(AtomicInteger count, ModelEntity entity) {
+ super(count);
+ this.entity = entity;
+ }
+
+ public DeclaredIndexFuture call() {
+ count.addAndGet(createDeclaredIndices(entity, messages));
+ return this;
+ }
+ }
+
+ public void checkDb(Map<String, ModelEntity> modelEntities, List<String> colWrongSize, List<String> messages, boolean checkPks, boolean checkFks, boolean checkFkIdx, boolean addMissing) {
+ if (isLegacy) {
+ throw new RuntimeException("Cannot run checkDb on a legacy database connection; configure a database helper (entityengine.xml)");
+ }
+ UtilTimer timer = new UtilTimer();
+ timer.timerString("Start - Before Get Database Meta Data");
+
+ // get ALL tables from this database
+ SortedSet<String> tableNames = this.getTableNames(messages);
+ TreeSet<String> fkTableNames = tableNames == null ? null : new TreeSet<String>(tableNames);
+ TreeSet<String> indexTableNames = tableNames == null ? null : new TreeSet<String>(tableNames);
+
+ if (tableNames == null) {
+ String message = "Could not get table name information from the database, aborting.";
+ if (messages != null) messages.add(message);
+ Debug.logError(message, module);
+ return;
+ }
+ timer.timerString("After Get All Table Names");
+
+ // get ALL column info, put into hashmap by table name
+ Map<String, Map<String, ColumnCheckInfo>> colInfo = this.getColumnInfo(tableNames, checkPks, messages);
+ if (colInfo == null) {
+ String message = "Could not get column information from the database, aborting.";
+ if (messages != null) messages.add(message);
+ Debug.logError(message, module);
+ return;
+ }
+ timer.timerString("After Get All Column Info");
+
+ // -make sure all entities have a corresponding table
+ // -list all tables that do not have a corresponding entity
+ // -display message if number of table columns does not match number of entity fields
+ // -list all columns that do not have a corresponding field
+ // -make sure each corresponding column is of the correct type
+ // -list all fields that do not have a corresponding column
+
+ timer.timerString("Before Individual Table/Column Check");
+
+ ArrayList<ModelEntity> modelEntityList = new ArrayList<ModelEntity>(modelEntities.values());
+ // sort using compareTo method on ModelEntity
+ Collections.sort(modelEntityList);
+ int curEnt = 0;
+ int totalEnt = modelEntityList.size();
+ List<ModelEntity> entitiesAdded = FastList.newInstance();
+ List<Future<TableFuture>> tableFutureFutures = FastList.newInstance();
+ ScheduledExecutorService threadPool = ExecutionPool.getNewOptimalExecutor("Databaseutil");
+
+ for (ModelEntity entity: modelEntityList) {
+ curEnt++;
+
+ // if this is a view entity, do not check it...
+ if (entity instanceof ModelViewEntity) {
+ String entMessage = "(" + timer.timeSinceLast() + "ms) NOT Checking #" + curEnt + "/" + totalEnt + " View Entity " + entity.getEntityName();
+ Debug.logVerbose(entMessage, module);
+ if (messages != null) messages.add(entMessage);
+ continue;
+ }
+
+ tableFutureFutures.add(threadPool.submit(new TableFuture(modelEntities, timer, tableNames, colInfo, curEnt, totalEnt, entity, checkPks, addMissing)));
}
+ for (TableFuture tableFuture: getAllFutures(tableFutureFutures)) {
+ tableFuture.updateData(messages, entitiesAdded);
+ }
timer.timerString("After Individual Table/Column Check");
// -list all tables that do not have a corresponding entity
@@ -388,13 +508,17 @@ public class DatabaseUtil {
// for each newly added table, add fk indices
if (datasourceInfo.useFkIndices) {
- int totalFkIndices = 0;
+ List<Future<ForeignKeyIndexFuture>> fkIndicesFutureFutures = FastList.newInstance();
+ AtomicInteger totalFkIndices = new AtomicInteger();
for (ModelEntity curEntity: entitiesAdded) {
if (curEntity.getRelationsOneSize() > 0) {
- totalFkIndices += this.createForeignKeyIndices(curEntity, datasourceInfo.constraintNameClipLength, messages);
+ fkIndicesFutureFutures.add(threadPool.submit(new ForeignKeyIndexFuture(totalFkIndices, curEntity, datasourceInfo.constraintNameClipLength)));
}
}
- if (totalFkIndices > 0) Debug.logImportant("==== TOTAL Foreign Key Indices Created: " + totalFkIndices, module);
+ for (ForeignKeyIndexFuture fkIndicesFuture: getAllFutures(fkIndicesFutureFutures)) {
+ fkIndicesFuture.updateData(messages);
+ }
+ if (totalFkIndices.get() > 0) Debug.logImportant("==== TOTAL Foreign Key Indices Created: " + totalFkIndices.get(), module);
}
// for each newly added table, add fks
@@ -408,13 +532,17 @@ public class DatabaseUtil {
// for each newly added table, add declared indexes
if (datasourceInfo.useIndices) {
- int totalDis = 0;
+ List<Future<DeclaredIndexFuture>> disFutureFutures = FastList.newInstance();
+ AtomicInteger totalDis = new AtomicInteger();
for (ModelEntity curEntity: entitiesAdded) {
if (curEntity.getIndexesSize() > 0) {
- totalDis += this.createDeclaredIndices(curEntity, messages);
+ disFutureFutures.add(threadPool.submit(new DeclaredIndexFuture(totalDis, curEntity)));
}
}
- if (totalDis > 0) Debug.logImportant("==== TOTAL Declared Indices Created: " + totalDis, module);
+ for (DeclaredIndexFuture disFuture: getAllFutures(disFutureFutures)) {
+ disFuture.updateData(messages);
+ }
+ if (totalDis.get() > 0) Debug.logImportant("==== TOTAL Declared Indices Created: " + totalDis.get(), module);
}
// make sure each one-relation has an FK
@@ -657,7 +785,7 @@ public class DatabaseUtil {
/** Creates a list of ModelEntity objects based on meta data from the database */
public List<ModelEntity> induceModelFromDb(Collection<String> messages) {
// get ALL tables from this database
- TreeSet<String> tableNames = this.getTableNames(messages);
+ SortedSet<String> tableNames = this.getTableNames(messages);
// get ALL column info, put into hashmap by table name
Map<String, Map<String, ColumnCheckInfo>> colInfo = this.getColumnInfo(tableNames, true, messages);
@@ -700,7 +828,7 @@ public class DatabaseUtil {
List<String> messages = new ArrayList<String>();
// get ALL tables from this database
- TreeSet<String> tableNames = this.getTableNames(messages);
+ SortedSet<String> tableNames = this.getTableNames(messages);
// get ALL column info, put into hashmap by table name
Map<String, Map<String, ColumnCheckInfo>> colInfo = this.getColumnInfo(tableNames, true, messages);
@@ -1019,7 +1147,7 @@ public class DatabaseUtil {
}
}
- public TreeSet<String> getTableNames(Collection<String> messages) {
+ public SortedSet<String> getTableNames(Collection<String> messages) {
Connection connection = getConnectionLogged(messages);
if (connection == null) {
@@ -1038,7 +1166,7 @@ public class DatabaseUtil {
if (Debug.infoOn()) Debug.logInfo("Getting Table Info From Database", module);
// get ALL tables from this database
- TreeSet<String> tableNames = new TreeSet<String>();
+ SortedSet<String> tableNames = new ConcurrentSkipListSet<String>();
ResultSet tableSet = null;
String lookupSchemaName = null;
@@ -1163,7 +1291,7 @@ public class DatabaseUtil {
if (Debug.infoOn()) Debug.logInfo("Getting Column Info From Database", module);
- Map<String, Map<String, ColumnCheckInfo>> colInfo = FastMap.newInstance();
+ Map<String, Map<String, ColumnCheckInfo>> colInfo = new ConcurrentHashMap<String, Map<String, ColumnCheckInfo>>();
String lookupSchemaName = null;
try {
if (dbData.supportsSchemasInTableDefinitions()) {