You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2011/03/26 23:32:37 UTC
svn commit: r1085819 - in /oodt/branches/wengine-branch/filemgr/src/main:
java/org/apache/oodt/cas/filemgr/catalog/ resources/
Author: bfoster
Date: Sat Mar 26 22:32:37 2011
New Revision: 1085819
URL: http://svn.apache.org/viewvc?rev=1085819&view=rev
Log:
- continued updates to column-based datasource catalog
---------------------
Modified:
oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalog.java
oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalogFactory.java
oodt/branches/wengine-branch/filemgr/src/main/resources/filemgr.properties
Modified: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalog.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalog.java?rev=1085819&r1=1085818&r2=1085819&view=diff
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalog.java (original)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalog.java Sat Mar 26 22:32:37 2011
@@ -37,6 +37,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.Vector;
import java.util.logging.Level;
@@ -66,12 +67,16 @@ public class ColumnBasedDataSourceCatalo
/* our validation layer */
private ValidationLayer validationLayer = null;
-
- private Set<String> dbVectorElements;
- private Set<String> dbIntegerTypes;
/* size of pages of products within the catalog */
protected int pageSize = -1;
+
+ public static final String VECTOR = "VECTOR";
+ public static final String STRING = "STRING";
+ public static final String NUMBER = "NUMBER";
+ public static final String DATE = "DATE";
+ public static final String TIMESTAMP = "TIMESTAMP";
+ public static final String TIMESTAMP_TZ = "TIMESTAMP_TZ";
/**
* <p>
@@ -79,12 +84,10 @@ public class ColumnBasedDataSourceCatalo
* </p>.
* @throws
*/
- public ColumnBasedDataSourceCatalog(DataSource ds, ValidationLayer valLayer, int pageSize, Set<String> dbIntegerTypes, Set<String> dbVectorElements) {
+ public ColumnBasedDataSourceCatalog(DataSource ds, ValidationLayer valLayer, int pageSize) {
this.dataSource = ds;
this.validationLayer = valLayer;
this.pageSize = pageSize;
- this.dbIntegerTypes = dbIntegerTypes;
- this.dbVectorElements = dbVectorElements;
}
public int getPageSize() {
@@ -93,46 +96,40 @@ public class ColumnBasedDataSourceCatalo
protected Statement setDateFormats(Statement statement) throws Exception {
statement.execute("alter session set NLS_DATE_FORMAT = 'YYYY-MM-DD'");
- statement.execute("alter session set NLS_TIME_FORMAT = 'HH24:MI:SS.FF3'");
- statement.execute("alter session set NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3'");
- statement.execute("alter session set NLS_TIME_TZ_FORMAT = 'HH24:MI:SS.FF3TZH:TZM'");
statement.execute("alter session set NLS_TIMESTAMP_FORMAT='YYYY-MM-DD\"T\"HH24:MI:SS.FF3\"Z\"'");
statement.execute("alter session set NLS_TIMESTAMP_TZ_FORMAT='YYYY-MM-DD\"T\"HH24:MI:SS.FF3TZH:TZM'");
return statement;
}
protected boolean isVector(Element element) {
- return this.dbVectorElements.contains(element.getElementId());
+ return element.getDCElement() != null ? element.getDCElement().toUpperCase().contains(VECTOR) : false;
}
-
- protected boolean isString(Element element, ProductType productType) throws Exception {
- Connection conn = null;
- ResultSet tables = null;
- ResultSet columns = null;
- try {
- conn = this.dataSource.getConnection();
- DatabaseMetaData metaData = conn.getMetaData();
-
- if (this.isVector(element))
- tables = metaData.getTables(null, null, element.getElementName().toUpperCase() + "_XREF", new String[] { "TABLE" });
- else
- tables = metaData.getTables(null, null, productType.getName().toUpperCase() + "_METADATA", new String[] { "TABLE" });
-
- if (tables.next()) {
- columns = metaData.getColumns(null, null, tables.getString("TABLE_NAME"), null);
- while (columns.next())
- if (columns.getString("COLUMN_NAME").equalsIgnoreCase(element.getElementName()))
- return !dbIntegerTypes.contains(columns.getString("TYPE_NAME").toLowerCase());
- }
- throw new Exception("Failed to determine type for element '" + element.getElementName() + "'");
- }catch (Exception e) {
- throw new Exception("Failed to determine if element is string for element '" + element.getElementName() + "' : " + e.getMessage(),e);
- }finally {
- try { conn.close(); } catch (Exception e) {}
- try { tables.close(); } catch (Exception e) {}
- try { columns.close(); } catch (Exception e) {}
- }
+ protected String getType(Element element) {
+ if (this.isVector(element))
+ return element.getDCElement().replaceFirst(VECTOR + "\\<", "").replaceAll("\\>", "");
+ else
+ return element.getDCElement() != null ? element.getDCElement().toUpperCase() : STRING;
+ }
+
+ protected String getDateFormat(String type) {
+ if (type.equalsIgnoreCase(DATE)) {
+ return "YYYY-MM-DD";
+ }else if (type.equalsIgnoreCase(TIMESTAMP)) {
+ return "YYYY-MM-DD\"T\"HH24:MI:SS.FF3\"Z\"";
+ }else if (type.equalsIgnoreCase(TIMESTAMP_TZ)) {
+ return "YYYY-MM-DD\"T\"HH24:MI:SS.FF3TZH:TZM";
+ }else {
+ return null;
+ }
+ }
+
+ protected boolean isDate(String type) {
+ return type.equalsIgnoreCase(DATE) || type.equalsIgnoreCase(TIMESTAMP) || type.equalsIgnoreCase(TIMESTAMP_TZ);
+ }
+
+ protected boolean isString(String type) {
+ return type.equalsIgnoreCase(STRING);
}
/*
@@ -162,7 +159,7 @@ public class ColumnBasedDataSourceCatalo
if (metadata.getMetadata(element.getElementName()) != null) {
if (!this.isVector(element)) {
scalarElementNames.add(element.getElementName());
- if (this.isString(element, product.getProductType()))
+ if (this.isString(this.getType(element)))
scalarElementValues.add("'" + metadata.getMetadata(element.getElementName()) + "'");
else
scalarElementValues.add(metadata.getMetadata(element.getElementName()));
@@ -172,7 +169,7 @@ public class ColumnBasedDataSourceCatalo
element.getElementName(),
element.getElementName(),
product.getProductId(),
- (this.isString(element, product.getProductType()) ? "'" + value + "'" : value));
+ (this.isString(this.getType(element)) ? "'" + value + "'" : value));
LOG.log(Level.FINE, "addMetadata Executing: " + sqlInsert);
statement.execute(sqlInsert);
}
@@ -852,8 +849,20 @@ public class ColumnBasedDataSourceCatalo
conn = dataSource.getConnection();
statement = this.setDateFormats(conn.createStatement());
- String metadataSql = "SELECT * FROM "
- + product.getProductType().getName() + "_vw "
+ Vector<String> selectElements = new Vector<String>();
+ for (Element element : this.validationLayer.getElements(product.getProductType())) {
+ try {
+ String type = this.getType(element);
+ if (this.isDate(type))
+ selectElements.add("to_char(" + element.getElementName() + ", '" + this.getDateFormat(type) + "') as " + element.getElementName());
+ else
+ selectElements.add(element.getElementName());
+ }catch (Exception e) {
+ LOG.log(Level.WARNING, "Element '" + element.getElementName() + "' not found : " + e.getMessage());
+ }
+ }
+ String metadataSql = "SELECT " + StringUtils.join(selectElements, ",") + " FROM "
+ + product.getProductType().getName() + "_VW"
+ " WHERE ProductId = " + product.getProductId();
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
@@ -862,7 +871,8 @@ public class ColumnBasedDataSourceCatalo
Metadata metadata = new Metadata();
List<Element> elements = this.validationLayer.getElements(product.getProductType());
if (rs.next()) {
- for (Element element : elements) {
+
+ for (Element element : elements) {
try {
String value = rs.getString(element.getElementName());
if (value == null)
@@ -893,9 +903,24 @@ public class ColumnBasedDataSourceCatalo
try {
conn = dataSource.getConnection();
statement = this.setDateFormats(conn.createStatement());
-
- String metadataSql = "SELECT * FROM "
- + product.getProductType().getName() + "_vw "
+
+ Vector<String> selectElements = new Vector<String>();
+ for (Element element : this.validationLayer.getElements(product.getProductType())) {
+ try {
+ if (elems.contains(element.getElementName())) {
+ String type = this.getType(element);
+ if (this.isDate(type))
+ selectElements.add("to_char(" + element.getElementName() + ", '" + this.getDateFormat(type) + "') as " + element.getElementName());
+ else
+ selectElements.add(element.getElementName());
+ }
+ }catch (Exception e) {
+ LOG.log(Level.WARNING, "Element '" + element.getElementName() + "' not found : " + e.getMessage());
+ }
+ }
+
+ String metadataSql = "SELECT " + StringUtils.join(selectElements, ",") + " FROM "
+ + product.getProductType().getName() + "_VW"
+ " WHERE ProductId = " + product.getProductId();
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
@@ -920,7 +945,7 @@ public class ColumnBasedDataSourceCatalo
}
}
- public List<Metadata> getReducedMetadata(Query query, ProductType type,
+ public List<Metadata> getReducedMetadata(Query query, ProductType productType,
List<String> elementNames) throws CatalogException {
Connection conn = null;
Statement statement = null;
@@ -930,14 +955,29 @@ public class ColumnBasedDataSourceCatalo
statement = this.setDateFormats(conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
ResultSet.CONCUR_READ_ONLY));
- String getProductSql = "SELECT * FROM " + type.getName();
+ Vector<String> selectElements = new Vector<String>();
+ for (Element element : this.validationLayer.getElements(productType)) {
+ try {
+ if (elementNames.contains(element.getElementName())) {
+ String type = this.getType(element);
+ if (this.isDate(type))
+ selectElements.add("to_char(" + element.getElementName() + ", '" + this.getDateFormat(type) + "') as " + element.getElementName());
+ else
+ selectElements.add(element.getElementName());
+ }
+ }catch (Exception e) {
+ LOG.log(Level.WARNING, "Element '" + element.getElementName() + "' not found : " + e.getMessage());
+ }
+ }
+
+ String getProductSql = "SELECT " + StringUtils.join(selectElements, ",") + " FROM " + productType.getName() + "_VW";
if (query.getCriteria() != null)
getProductSql += " WHERE " + SqlParser.getInfixCriteriaString(query.getCriteria()).replaceAll("==", "=");;
rs = statement.executeQuery(getProductSql);
Vector<Metadata> metadatas = new Vector<Metadata>();
- List<Element> elements = this.validationLayer.getElements(type);
+ List<Element> elements = this.validationLayer.getElements(productType);
while (rs.next()) {
Metadata metadata = new Metadata();
for (Element element : elements)
Modified: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalogFactory.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalogFactory.java?rev=1085819&r1=1085818&r2=1085819&view=diff
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalogFactory.java (original)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/ColumnBasedDataSourceCatalogFactory.java Sat Mar 26 22:32:37 2011
@@ -55,9 +55,6 @@ public class ColumnBasedDataSourceCatalo
protected String user;
protected String pass;
protected String driver;
-
- protected Set<String> dbIntegerTypes;
- protected Set<String> dbVectorElements;
protected String validationLayerFactoryClass;
@@ -84,16 +81,6 @@ public class ColumnBasedDataSourceCatalo
String dbIntTypes = PathUtils
.replaceEnvVariables(System
.getProperty("org.apache.oodt.cas.filemgr.catalog.column.based.datasource.db.int.types"));
- if (dbIntTypes != null)
- this.dbIntegerTypes = new HashSet<String>(Arrays.asList(dbIntTypes.toLowerCase()
- .split(",")));
-
- String dbVecElems = PathUtils
- .replaceEnvVariables(System
- .getProperty("org.apache.oodt.cas.filemgr.catalog.column.based.datasource.db.vector.elements"));
- if (dbVecElems != null)
- this.dbVectorElements = new HashSet<String>(Arrays.asList(dbVecElems
- .split(",")));
validationLayerFactoryClass = System
.getProperty("filemgr.validationLayer.factory",
@@ -163,22 +150,6 @@ public class ColumnBasedDataSourceCatalo
public void setValidationLayer(ValidationLayer validationLayer) {
this.validationLayer = validationLayer;
}
-
- public Set<String> getDbIntegerTypes() {
- return dbIntegerTypes;
- }
-
- public void setDbIntegerTypes(Set<String> dbIntegerTypes) {
- this.dbIntegerTypes = dbIntegerTypes;
- }
-
- public Set<String> getDbVectorElements() {
- return dbVectorElements;
- }
-
- public void setDbVectorElements(Set<String> dbVectorElements) {
- this.dbVectorElements = dbVectorElements;
- }
/*
* (non-Javadoc)
@@ -190,7 +161,7 @@ public class ColumnBasedDataSourceCatalo
this.validationLayer = GenericFileManagerObjectFactory.getValidationLayerFromFactory(validationLayerFactoryClass);
if (this.ds == null)
this.ds = DatabaseConnectionBuilder.buildDataSource(user, pass,driver, jdbcUrl);
- return new ColumnBasedDataSourceCatalog(this.ds, validationLayer, pageSize, this.dbIntegerTypes, this.dbVectorElements);
+ return new ColumnBasedDataSourceCatalog(this.ds, validationLayer, pageSize);
}
}
Modified: oodt/branches/wengine-branch/filemgr/src/main/resources/filemgr.properties
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/resources/filemgr.properties?rev=1085819&r1=1085818&r2=1085819&view=diff
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/resources/filemgr.properties (original)
+++ oodt/branches/wengine-branch/filemgr/src/main/resources/filemgr.properties Sat Mar 26 22:32:37 2011
@@ -38,8 +38,6 @@ org.apache.oodt.cas.filemgr.catalog.colu
org.apache.oodt.cas.filemgr.catalog.column.based.datasource.jdbc.pass=pass
org.apache.oodt.cas.filemgr.catalog.column.based.datasource.jdbc.driver=driver.class.name
org.apache.oodt.cas.filemgr.catalog.column.based.datasource.pageSize=20
-org.apache.oodt.cas.filemgr.catalog.column.based.datasource.db.int.types=number,int
-org.apache.oodt.cas.filemgr.catalog.column.based.datasource.db.vector.elements=urn:cas:InputFiles
# mapped data source catalog configuration
org.apache.oodt.cas.filemgr.catalog.mappeddatasource.mapFile=/path/to/ops.catalog.typemap.properties