You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2010/12/21 01:42:18 UTC
svn commit: r1051347 [2/2] - in /oodt/trunk: ./
catalog/src/main/java/org/apache/oodt/cas/catalog/query/parser/
catalog/src/main/java/org/apache/oodt/cas/catalog/struct/
catalog/src/main/java/org/apache/oodt/cas/catalog/struct/impl/index/
catalog/src/m...
Modified: oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java
URL: http://svn.apache.org/viewvc/oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java?rev=1051347&r1=1051346&r2=1051347&view=diff
==============================================================================
--- oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java (original)
+++ oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocal.java Tue Dec 21 00:42:17 2010
@@ -23,11 +23,13 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.Vector;
+import java.util.Map.Entry;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
@@ -83,8 +85,10 @@ public class CatalogServiceLocal impleme
protected File pluginStorageDir;
protected boolean oneCatalogFailsAllFail;
protected boolean simplifyQueries;
+ protected boolean disableIntersectingCrossCatalogQueries;
+ protected int crossCatalogResultSortingThreshold;
- public CatalogServiceLocal(CatalogRepository catalogRepository, IngestMapper ingestMapper, File pluginStorageDir, TransactionIdFactory transactionIdFactory, boolean restrictQueryPermissions, boolean restrictIngestPermissions, boolean oneCatalogFailsAllFail, boolean simplifyQueries) throws InstantiationException {
+ public CatalogServiceLocal(CatalogRepository catalogRepository, IngestMapper ingestMapper, File pluginStorageDir, TransactionIdFactory transactionIdFactory, boolean restrictQueryPermissions, boolean restrictIngestPermissions, boolean oneCatalogFailsAllFail, boolean simplifyQueries, boolean disableIntersectingCrossCatalogQueries, int crossCatalogResultSortingThreshold) throws InstantiationException {
try {
this.catalogs = new HashSet<Catalog>();
this.catalogsLock = new ReentrantReadWriteLock();
@@ -97,6 +101,8 @@ public class CatalogServiceLocal impleme
this.setCatalogRepository(catalogRepository);
this.oneCatalogFailsAllFail = oneCatalogFailsAllFail;
this.simplifyQueries = simplifyQueries;
+ this.disableIntersectingCrossCatalogQueries = disableIntersectingCrossCatalogQueries;
+ this.crossCatalogResultSortingThreshold = crossCatalogResultSortingThreshold;
}catch (Exception e) {
e.printStackTrace();
throw new InstantiationException(e.getMessage());
@@ -664,9 +670,68 @@ public class CatalogServiceLocal impleme
}
public Page getPage(PageInfo pageInfo, QueryExpression queryExpression, Set<String> catalogIds) throws CatalogServiceException {
- QueryPager queryPager = new QueryPager(this._query(queryExpression, catalogIds));
- queryPager.setPageInfo(pageInfo);
- return this.getPage(queryExpression, catalogIds, queryPager);
+ if (this.disableIntersectingCrossCatalogQueries) {
+ try {
+ int totalResults = 0;
+ LinkedHashMap<String, Integer> catalogToSizeOfMap = new LinkedHashMap<String, Integer>();
+ for (String catalogId : catalogIds) {
+ Catalog catalog = this.getCatalog(catalogId);
+ QueryExpression qe = this.reduceToUnderstoodExpressions(catalog, queryExpression);
+ if (qe != null) {
+ int catalogResultSize = catalog.sizeOf(qe);
+ totalResults += catalogResultSize;
+ catalogToSizeOfMap.put(catalogId, catalogResultSize);
+ }
+ }
+
+ LOG.log(Level.INFO, "Routing query to catalogs as non-cross catalog intersecting queries . . .");
+ if (totalResults <= this.crossCatalogResultSortingThreshold) {
+ List<CatalogReceipt> catalogReceipts = new Vector<CatalogReceipt>();
+ for (String catalogId : catalogToSizeOfMap.keySet()) {
+ Catalog catalog = this.getCatalog(catalogId);
+ QueryExpression qe = this.reduceToUnderstoodExpressions(catalog, queryExpression);
+ if (qe != null)
+ catalogReceipts.addAll(catalog.query(qe));
+ }
+ List<TransactionReceipt> transactionReceipts = this.getPossiblyUnindexedTransactionReceipts(catalogReceipts);
+ LOG.log(Level.INFO, "Sorting Query Results . . . ");
+ Collections.sort(transactionReceipts, new Comparator<TransactionReceipt>() {
+ public int compare(TransactionReceipt o1,
+ TransactionReceipt o2) {
+ return o2.getTransactionDate().compareTo(o1.getTransactionDate());
+ }
+ });
+ QueryPager queryPager = new QueryPager(transactionReceipts);
+ queryPager.setPageInfo(pageInfo);
+ return this.getPage(queryExpression, catalogIds, queryPager);
+ }else {
+ int currentIndex = 0;
+ int desiredStartingIndex = pageInfo.getPageNum() * pageInfo.getPageSize();
+ List<CatalogReceipt> pageOfReceipts = new Vector<CatalogReceipt>();
+ for (Entry<String, Integer> entry : catalogToSizeOfMap.entrySet()) {
+ if (desiredStartingIndex - currentIndex <= entry.getValue()) {
+ Catalog catalog = this.getCatalog(entry.getKey());
+ QueryExpression qe = this.reduceToUnderstoodExpressions(catalog, queryExpression);
+ if (qe != null) {
+ List<CatalogReceipt> receipts = catalog.query(qe, desiredStartingIndex - currentIndex, Math.min((desiredStartingIndex - currentIndex) + pageInfo.getPageSize(), entry.getValue()));
+ pageOfReceipts.addAll(receipts);
+ if (pageOfReceipts.size() >= pageInfo.getPageSize())
+ break;
+ }
+ }else {
+ currentIndex += entry.getValue();
+ }
+ }
+ return new Page(new ProcessedPageInfo(pageInfo.getPageSize(), pageInfo.getPageNum(), totalResults), queryExpression, catalogIds, this.indexReceipts(this.getPossiblyUnindexedTransactionReceipts(pageOfReceipts)));
+ }
+ }catch (Exception e) {
+ throw new CatalogServiceException(e.getMessage(), e);
+ }
+ }else {
+ QueryPager queryPager = new QueryPager(this._query(queryExpression, catalogIds));
+ queryPager.setPageInfo(pageInfo);
+ return this.getPage(queryExpression, catalogIds, queryPager);
+ }
}
public QueryPager query(QueryExpression queryExpression) throws CatalogServiceException {
@@ -1061,6 +1126,10 @@ TR: for (CatalogReceipt catalogReceip
List<QueryExpression> restrictedExpressions = new Vector<QueryExpression>();
for (QueryExpression qe : queryLogicalGroup.getExpressions()) {
QueryExpression restrictedQE = this.reduceToUnderstoodExpressions(catalog, qe);
+ if (restrictedQE == null && queryLogicalGroup.getOperator().equals(QueryLogicalGroup.Operator.AND) && this.disableIntersectingCrossCatalogQueries) {
+ restrictedExpressions.clear();
+ break;
+ }
if (restrictedQE != null)
restrictedExpressions.add(restrictedQE);
}
Modified: oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java
URL: http://svn.apache.org/viewvc/oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java?rev=1051347&r1=1051346&r2=1051347&view=diff
==============================================================================
--- oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java (original)
+++ oodt/trunk/catalog/src/main/java/org/apache/oodt/cas/catalog/system/impl/CatalogServiceLocalFactory.java Tue Dec 21 00:42:17 2010
@@ -52,12 +52,14 @@ public class CatalogServiceLocalFactory
protected String pluginStorageDir = null;
protected boolean oneCatalogFailsAllFail = false;
protected boolean simplifyQueries = false;
+ protected boolean disableIntersectingCrossCatalogQueries = false;
+ protected int crossCatalogResultSortingThreshold = 200;
public CatalogServiceLocalFactory() {}
public CatalogServiceLocal createCatalogService() {
try {
- return new CatalogServiceLocal(this.catalogRepositoryFactory.createRepository(), this.ingestMapperFactory.createMapper(), new File(this.pluginStorageDir), this.transactionIdFactory, this.restrictQueryPermissions, this.restrictIngestPermissions, this.oneCatalogFailsAllFail, this.simplifyQueries);
+ return new CatalogServiceLocal(this.catalogRepositoryFactory.createRepository(), this.ingestMapperFactory.createMapper(), new File(this.pluginStorageDir), this.transactionIdFactory, this.restrictQueryPermissions, this.restrictIngestPermissions, this.oneCatalogFailsAllFail, this.simplifyQueries, this.disableIntersectingCrossCatalogQueries, this.crossCatalogResultSortingThreshold);
}catch (Exception e) {
LOG.log(Level.SEVERE, "Failed to create CatalogServiceLocal : " + e.getMessage(), e);
return null;
@@ -104,4 +106,15 @@ public class CatalogServiceLocalFactory
this.simplifyQueries = simplifyQueries;
}
+ @Required
+ public void setDisableIntersectingCrossCatalogQueries(boolean disableIntersectingCrossCatalogQueries) {
+ this.disableIntersectingCrossCatalogQueries = disableIntersectingCrossCatalogQueries;
+ }
+
+ @Required
+ public void setCrossCatalogResultSortingThreshold(
+ int crossCatalogResultSortingThreshold) {
+ this.crossCatalogResultSortingThreshold = crossCatalogResultSortingThreshold;
+ }
+
}
Modified: oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml?rev=1051347&r1=1051346&r2=1051347&view=diff
==============================================================================
--- oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml (original)
+++ oodt/trunk/catalog/src/main/resources/policy/catserv-beans.xml Tue Dec 21 00:42:17 2010
@@ -32,6 +32,8 @@
<property name="restrictIngestPermissions" value="false"/>
<property name="oneCatalogFailsAllFail" value="true"/>
<property name="simplifyQueries" value="true"/>
+ <property name="disableIntersectingCrossCatalogQueries" value="false"/>
+ <property name="crossCatalogResultSortingThreshold" value="200"/>
</bean>
<bean id="CatalogServiceClientFactory" lazy-init="true" class="org.apache.oodt.cas.catalog.system.impl.CatalogServiceClientFactory">
<property name="communicationChannelClientFactory" ref="XmlRpcClientFactory"/>