You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by lu...@apache.org on 2013/04/26 19:28:02 UTC
svn commit: r1476304 [1/2] - in /oodt/trunk/filemgr/src/main: assembly/
java/org/apache/oodt/cas/filemgr/catalog/solr/ resources/
Author: luca
Date: Fri Apr 26 17:28:01 2013
New Revision: 1476304
URL: http://svn.apache.org/r1476304
Log:
Inserting the first version of Solr back-end for the CAS File Manager.
Added:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/CompleteProduct.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/NameProductIdGenerator.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/OODT_SOLR_README.TXT
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/Parameters.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductIdGenerator.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductSerializer.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QUICK_START_GUIDE.TXT
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QueryResponse.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalogFactory.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/UUIDProductIdGenerator.java (with props)
oodt/trunk/filemgr/src/main/resources/schema.xml (with props)
Modified:
oodt/trunk/filemgr/src/main/assembly/assembly.xml
oodt/trunk/filemgr/src/main/resources/filemgr.properties
Modified: oodt/trunk/filemgr/src/main/assembly/assembly.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/assembly/assembly.xml?rev=1476304&r1=1476303&r2=1476304&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/assembly/assembly.xml (original)
+++ oodt/trunk/filemgr/src/main/assembly/assembly.xml Fri Apr 26 17:28:01 2013
@@ -62,6 +62,7 @@ $Id$
<outputDirectory>etc</outputDirectory>
<includes>
<include>mime-types.xml</include>
+ <include>schema.xml</include>
</includes>
</fileSet>
<fileSet>
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/CompleteProduct.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/CompleteProduct.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/CompleteProduct.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/CompleteProduct.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * Class that combines a Product and its associated Metadata.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public class CompleteProduct {
+
+ private Product product;
+
+ private Metadata metadata;
+
+ public CompleteProduct() {
+
+ product = new Product();
+ product.setProductType(new ProductType());
+ metadata = new Metadata();
+
+ }
+
+ public Product getProduct() {
+ return product;
+ }
+
+ public void setProduct(Product product) {
+ this.product = product;
+ }
+
+ public Metadata getMetadata() {
+ return metadata;
+ }
+
+ public void setMetadata(Metadata metadata) {
+ this.metadata = metadata;
+ }
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/CompleteProduct.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,514 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+/**
+ * Default implementation of {@link ProductSerializer}
+ * that transforms a CAS product into a single Solr document based on the following rules:
+ * o) the core product attributes are used to generate Solr fields starting with "CAS...."
+ * o) the product references are converted to Solr fields starting with "CAS.Reference..." or "CAS.RootReference..."
+ * o) all other metadata fields are converted into Solr fields with the same name and number of values.
+ * Note that the field multiplicity must be consistent with its definition in the Solr schema.xml.
+ *
+ * This class generates all Solr documents in XML format.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public class DefaultProductSerializer implements ProductSerializer {
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getMimeType() {
+ return Parameters.MIME_TYPE_XML;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<String> serialize(Product product, boolean create) {
+
+ Map<String, List<String>> fields = new HashMap<String, List<String>>();
+ List<String> docs = new ArrayList<String>();
+
+ // add core product attributes to map
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_ID, product.getProductId());
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_NAME, product.getProductName());
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_STRUCTURE, product.getProductStructure());
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_TRANSFER_STATUS, product.getTransferStatus());
+ ProductType productType = product.getProductType();
+ if (productType!=null) {
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_NAME, productType.getName());
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId());
+ }
+ if (create) {
+ // only insert date/time when product is first created
+ Date productDateTime = new Date(); // current datetime
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_RECEIVED_TIME, Parameters.SOLR_DATE_TIME_FORMATTER.format(productDateTime));
+ }
+
+ // create new product: use Solr id == CAS id
+ if (create) {
+ docs.add( this.generateInsertDocuments(product.getProductId(), fields) );
+
+ // update existing product
+ } else {
+ docs.addAll( this.generateUpdateDocuments(product.getProductId(), fields, true) ); // replace=true
+
+ }
+
+ return docs;
+
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public List<String> serialize(String productId, Reference rootReference, List<Reference> references, boolean replace) {
+
+ Map<String, List<String>> fields = new HashMap<String, List<String>>();
+
+ // product root reference
+ if (rootReference!=null) {
+
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, rootReference.getOrigReference());
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, rootReference.getDataStoreReference());
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_FILESIZE, ""+rootReference.getFileSize());
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, rootReference.getMimeType().toString());
+
+ }
+
+ // all other product references
+ // note that Solr will preserve the indexing order.
+ for (Reference reference : references) {
+
+ addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, reference.getOrigReference());
+ addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, reference.getDataStoreReference());
+ addKeyValueToMap(fields, Parameters.REFERENCE_FILESIZE, ""+reference.getFileSize());
+ addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, reference.getMimeType().toString());
+
+ }
+
+ return generateUpdateDocuments(productId, fields, replace);
+
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public QueryResponse deserialize(String xml) throws CatalogException {
+
+ try {
+
+ QueryResponse queryResponse = new QueryResponse();
+
+ // parse XML into DOM
+ Document document = parseXml(xml);
+
+ // extract information from DOM to Product
+ Element response = document.getDocumentElement();
+ Node result = response.getElementsByTagName("result").item(0);
+ queryResponse.setNumFound( Integer.parseInt( ((Element)result).getAttribute("numFound") ) );
+ queryResponse.setStart( Integer.parseInt( ((Element)result).getAttribute("start") ) );
+ NodeList docs = result.getChildNodes();
+ for (int i=0; i< docs.getLength(); i++) {
+ Node node = docs.item(i);
+ if (node.getNodeName().equals("doc")) {
+ Element doc = (Element)node;
+ CompleteProduct cp = this.deserialize(doc);
+ queryResponse.getCompleteProducts().add(cp);
+ }
+ }
+ return queryResponse;
+
+ } catch(Exception e) {
+ e.printStackTrace();
+ throw new CatalogException(e.getMessage());
+ }
+
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public List<String> serialize(String productId, Metadata metadata, boolean replace) {
+
+ Map<String, List<String>> fields = new HashMap<String, List<String>>();
+
+ for (String key : metadata.getKeys()) {
+ if (! (key.startsWith(Parameters.NS) // skip metadata keys starting with reserved namespace
+ || Parameters.PRODUCT_TYPE_NAME.indexOf(key)>=0 // skip 'ProductType' as already stored as 'CAS.ProductTypeName'
+ || Parameters.PRODUCT_STRUCTURE.indexOf(key)>=0)) { // skip 'ProductType' as already stored as 'CAS.ProductStructure'
+ for (String value : metadata.getAllMetadata(key)) {
+ this.addKeyValueToMap(fields, key, value);
+ }
+ }
+ }
+
+ return this.generateUpdateDocuments(productId, fields, replace);
+
+ }
+
+ /**
+ * Method to add a (key, value) to a multi-valued map with appropriate checks.
+ * @param map
+ * @param key
+ * @param value
+ */
+ protected void addKeyValueToMap(Map<String, List<String>> map, String key, String value) {
+
+ if (!map.containsKey(key)) {
+ map.put(key, new ArrayList<String>());
+ }
+ if (value!=null) {
+ map.get(key).add(value);
+ } else {
+ // use special value to trigger key removal
+ map.get(key).add(Parameters.NULL);
+ }
+ }
+
+ /**
+ * Utility method to generate a Solr insert document.
+ *
+ * @param productId
+ * @param fields
+ * @return
+ */
+ protected String generateInsertDocuments(String productId, Map<String,List<String>> fields) {
+
+ StringBuilder doc = new StringBuilder();
+ doc.append("<doc>");
+
+ // product Solr id field
+ doc.append( encodeIndexField(Parameters.ID, productId) );
+
+ // all other fields
+ for (String key : fields.keySet()) {
+ for (String value : fields.get(key)) {
+ doc.append( encodeIndexField(key, value) );
+ }
+ }
+
+ doc.append("</doc>");
+ return doc.toString();
+ }
+
+ /**
+ * Utility method to generate Solr update documents.
+ * Note that the requests for setting/adding/deleting fields must be sent as separate documents to Solr
+ * @param productId
+ * @param fields
+ * @param replace
+ * @return
+ */
+ protected List<String> generateUpdateDocuments(String productId, Map<String,List<String>> fields, boolean replace) {
+
+ // list for different instruction types
+ List<String> setFields = new ArrayList<String>();
+ List<String> addFields = new ArrayList<String>();
+ List<String> delFields = new ArrayList<String>();
+
+ // encode update instructions
+ for (String key : fields.keySet()) {
+
+ List<String> values = fields.get(key);
+
+ if (replace) {
+
+ if (values.isEmpty()) {
+ // use special value to flag removal
+ delFields.add( this.encodeUpdateField(key, Parameters.NULL, replace) );
+
+ } else {
+ for (String value : values) {
+ setFields.add( this.encodeUpdateField(key, value, replace) );
+ }
+ }
+
+ } else {
+ for (String value : values) {
+ addFields.add( this.encodeUpdateField(key, value, replace) );
+ }
+ }
+
+ }
+
+ List<String> docs = new ArrayList<String>();
+ if (!delFields.isEmpty()) docs.add( toDoc(productId, delFields) );
+ if (!setFields.isEmpty()) docs.add( toDoc(productId, setFields) );
+ if (!addFields.isEmpty()) docs.add( toDoc(productId, addFields) );
+ return docs;
+
+ }
+
+ /**
+ * Utility method to merge field update instructions into a single document.
+ * @param productId
+ * @param updates
+ * @return
+ */
+ private String toDoc(String productId, List<String> updates) {
+
+ StringBuilder doc = new StringBuilder();
+ doc.append("<doc>");
+
+ // reference product record id
+ doc.append( encodeIndexField(Parameters.ID, productId) );
+
+ // loop over field update instructions
+ for (String update : updates) {
+ doc.append(update);
+ }
+
+ doc.append("</doc>");
+
+ return doc.toString();
+
+ }
+
+ /**
+ * Method to encode a Solr field indexing instruction.
+ * If the value is null, the empty string is returned.
+ * @param key
+ * @param value
+ * @return
+ */
+ protected String encodeIndexField(String key, String value) {
+ if (value==null || value.equals(Parameters.NULL)) {
+ return "";
+ } else {
+ return "<field name=\""+key+"\">" + value + "</field>";
+ }
+ }
+
+ /**
+ * Method to encode a field update instruction for the three possible cases:
+ * add new values to a key (1), replace current values for a key (2), remove all values for a key (3).
+ *
+ * @param key
+ * @param value
+ * @param replace
+ * @return
+ */
+ protected String encodeUpdateField(String key, String value, boolean replace) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<field name=\""+key+"\"");
+
+ if (replace) {
+
+ if (value==null || value.equals(Parameters.NULL)) {
+
+ // (3) remove all values for given key
+ sb.append(" update=\"set\" null=\"true\" />");
+
+ } else {
+
+ // (2) replace existing values with new values
+ sb.append(" update=\"set\">" + value + "</field>");
+ }
+
+ } else {
+
+ // (1) add new values to existing values
+ sb.append(" update=\"add\">"+value+"</field>");
+
+ }
+
+ return sb.toString();
+ }
+
+
+
+ /**
+ * Method that parses a single Solr document snippet
+ * to extract Product and Metadata attributes.
+ *
+ * @param doc
+ * @return
+ */
+ private CompleteProduct deserialize(Element doc) {
+
+ CompleteProduct cp = new CompleteProduct();
+ Product product = cp.getProduct();
+ ProductType productType = product.getProductType();
+ Metadata metadata = cp.getMetadata();
+ List<Reference> references = product.getProductReferences();
+ Reference rootReference = product.getRootRef();
+
+ NodeList children = doc.getChildNodes();
+ for (int j=0; j<children.getLength(); j++) {
+
+ Node child = children.item(j);
+ Element element = (Element)child;
+ String name = element.getAttribute("name");
+
+ /**
+ *<arr name="ScanPointingSource">
+ * <str>G073.65+0.19</str>
+ * <str>J2015+3410</str>
+ * ..........
+ */
+ if (child.getNodeName().equals("arr")) {
+
+ NodeList values = element.getChildNodes();
+ List<String> vals = new ArrayList<String>();
+ for (int k=0; k<values.getLength(); k++) {
+ String value = ((Element)values.item(k)).getTextContent();
+ vals.add(value);
+ }
+ // CAS.reference.... fields
+ if (name.startsWith(Parameters.NS)) {
+ for (int k=0; k<values.getLength(); k++) {
+ // create this reference
+ if (references.size()<=k) references.add(new Reference());
+ if (name.equals(Parameters.REFERENCE_ORIGINAL)) {
+ references.get(k).setOrigReference(vals.get(k));
+ } else if (name.equals(Parameters.REFERENCE_DATASTORE)) {
+ references.get(k).setDataStoreReference(vals.get(k));
+ } else if (name.equals(Parameters.REFERENCE_FILESIZE)) {
+ references.get(k).setFileSize(Long.parseLong(vals.get(k)));
+ } else if (name.equals(Parameters.REFERENCE_MIMETYPE)) {
+ references.get(k).setMimeType(vals.get(k));
+ }
+ }
+ // all other multi-valued fields
+ } else {
+ this.deserializeMultiValueField(name, vals, metadata);
+ }
+
+ /**
+ * <str name="id">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
+ * <str name="CAS.ProductId">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
+ * <str name="CAS.ProductName">tns_br145x4_20</str>
+ * <str name="FileLocation">/usr/local/ska-dc/data/archive</str>
+ * ...........
+ */
+ } else {
+
+ String value = element.getTextContent();
+
+ // core CAS fields
+ if (name.startsWith(Parameters.NS)) {
+ if (name.equals(Parameters.PRODUCT_ID)) {
+ product.setProductId(value);
+ } else if (name.equals(Parameters.PRODUCT_NAME)) {
+ product.setProductName(value);
+ } else if (name.equals(Parameters.PRODUCT_STRUCTURE)) {
+ product.setProductStructure(value);
+ } else if (name.equals(Parameters.PRODUCT_TRANSFER_STATUS)) {
+ product.setTransferStatus(value);
+ } else if (name.equals(Parameters.PRODUCT_TYPE_NAME)) {
+ productType.setName(value);
+ } else if (name.equals(Parameters.PRODUCT_TYPE_ID)) {
+ productType.setProductTypeId(value);
+ } else if (name.equals(Parameters.PRODUCT_RECEIVED_TIME)) {
+ // ignore ?
+
+ // CAS root reference
+ } else if (name.startsWith(Parameters.NS+Parameters.ROOT)) {
+ if (rootReference==null) rootReference = new Reference();
+ if (name.equals(Parameters.ROOT_REFERENCE_ORIGINAL)) {
+ rootReference.setOrigReference(value);
+ } else if (name.equals(Parameters.ROOT_REFERENCE_DATASTORE)) {
+ rootReference.setDataStoreReference(value);
+ } else if (name.equals(Parameters.ROOT_REFERENCE_FILESIZE)) {
+ rootReference.setFileSize(Long.parseLong(value));
+ } else if (name.equals(Parameters.ROOT_REFERENCE_MIMETYPE)) {
+ rootReference.setMimeType(value);
+ }
+
+ }
+
+ // non core single-valued fields
+ } else {
+ this.deserializeSingleValueField(name, value, metadata);
+ } // "CAS".... or not
+
+ } // "arr" or anything else
+
+ } // loop over <doc> children
+
+ return cp;
+
+ }
+
+ private Document parseXml(String xml) throws Exception {
+
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder parser = factory.newDocumentBuilder();
+ Document document = parser.parse( new InputSource(new StringReader(xml)) );
+ return document;
+
+ }
+
+ /**
+ * Method that deserializes a single-valued Solr field into a Metadata element.
+ * This method can be overridden by sub-classes to provide custom behavior.
+ *
+ * @param name : the Solr field name
+ * @param value : the Solr field single value
+ * @param metadata : the metadata container
+ */
+ protected void deserializeSingleValueField(String name, String value, Metadata metadata) {
+
+ if (name.equals(Parameters.ID)) {
+ // ignore Solr internal identifier (as it is duplicate information of CAS.ProductId)
+ } else {
+ metadata.addMetadata(name, value);
+ }
+
+ }
+
+ /**
+ * Method that deserializes a multi-valued Solr field into a Metadata element.
+ * This method can be overridden by sub-classes to provide custom behavior.
+ *
+ * @param name : the Solr field name
+ * @param values : the Solr field multiple values
+ * @param metadata : the metadata container
+ */
+ protected void deserializeMultiValueField(String name, List<String> values, Metadata metadata) {
+ metadata.addMetadata(name, values);
+ }
+
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/NameProductIdGenerator.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/NameProductIdGenerator.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/NameProductIdGenerator.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/NameProductIdGenerator.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+
+/**
+ * Implementation of {@link ProductIdGenerator}
+ * that sets the Product identifier equal to the product name.
+ * This generator will cause a second submission of the same named Product
+ * to override the previous record inside the Solr index.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public class NameProductIdGenerator implements ProductIdGenerator {
+
+ @Override
+ public String generateId(Product product) {
+ return product.getProductName();
+ }
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/NameProductIdGenerator.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/OODT_SOLR_README.TXT
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/OODT_SOLR_README.TXT?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/OODT_SOLR_README.TXT (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/OODT_SOLR_README.TXT Fri Apr 26 17:28:01 2013
@@ -0,0 +1,170 @@
+NOTE: THIS FILE WILL NOT BE CHECKED IN, RATHER TRANSFORMED INTO AN APACHE WIKI PAGE
+===================================================================================
+
+
+GUIDE: USING OODT FILE MANAGER WITH A SOLR METADATA STORE BACK-END
+==================================================================
+
+Introduction
+------------
+
+The Apache OODT File Manager and Apache Solr are in many respects two complementary technologies
+that can be combined to offer a very attractive solution for managing scientific data and metadata.
+The File Manager is a service for archiving and retrieving data products (files and directories),
+typically used as part of a data distribution service or data processing pipeline.
+Solr is a scalable, high performance search engine that provides a web based API on top of the underlying
+Lucene index. By deploying an OODT File Manager backed-up by a Solr metadata store,
+a scientific project can leverage the product creation and ingestion functionality of the OODT framework,
+together with the fast and powerful query capabilities offered by Solr.
+
+Architecture
+------------
+
+When the File Manager is deployed with a Solr-based metadata catalog, the overall system architecture is composed of the following tiers:
+
+File Manager (archive) clients <--> File Manager <--> Solr <--> Solr (query) clients
+ (default port:9000) (default ports: 80, 8080 or 8983)
+
+File Manager clients send product archiving requests to the File Manager (typically running as a local service on default port 9000).
+This communication is carried over HTTP through XML-RPC encoded request/response pairs.
+As part of the archiving process, the File Manager transforms the available product metadata
+(either generated on the client side and sent as part of the archiving request, or generated by metadata extractors on the server side)
+into Solr documents, which are sent to the Solr server for indexing. The Solr server must be deployed as an external service, running
+within Tomcat (default ports: 80 or 8080) or Jetty (default port: 8983), and configured with a metadata schema that defines
+the project specific fields to be indexed. The File Manager sends documents to Solr as payloads of HTTP/POST requests.
+Product metadata can be queried either by the standard File Manager clients, but more efficiently by clients that interact directly
+with the Solr web services (as Solr is optimized for querying).
+In conclusion, in this architecture the OODT File Manager provides the data archiving functionality, while Solr is responsible for
+providing metadata querying services.
+
+
+Release Notes and Limitations
+-----------------------------
+
+The OODT File Manager is only compatible with a Solr version of 4.X or above - i.e. no support is provided for interacting with pre-existing Solr
+installations of 3.X or below. The technical motivation is that the Solr implementation of the OODT Catalog interface relies heavily on the
+"atomic update" functionality that was introduced in Solr 4 - i.e. the ability to update single parts of Solr document without the need to re-index the whole
+document. The logical rationale behind this decision is that this is a new functionality provided to the OODT framework, and consequently there is
+no need to support legacy deployments: a project that wishes to leverage this architecture might as well start with the most up to date version of Solr,
+instead of installing an older version.
+
+Furthermore, in order to leverage the out-of-the-box "atomic update" functionality, ALL fields to be indexed must be declared in the Solr schema.xml
+with "stored=true". This requirement comes with the price of a larger index, but will allow documents to be updated in place, as opposed to be fully replaced.
+
+Currently, multiple Solr cores are NOT supported: all documents sent by the File Manager are indexed and retrieved from the default Solr core.
+
+Extension Points
+----------------
+
+Currently, a File Manager deployment based on a Solr back-end can be customized in three possible ways,
+controlled by properties set in the File Manager filemgr.properties file. All extension points have reasonable
+default implementations, which should be adequate in many cases, or at least to start experiment with this architecture.
+
+o Generation of product unique identifiers.
+ When a product is first ingested, a unique identifier must be created to reference that product in all subsequent requests.
+ The particular algorithm that is used to generate the identifier has great consequences for the underlying semantics of the
+ metadata catalog. By default, the Solr File Manager will assign a newly generated UUID as the product unique identifier.
+ As a consequence, if the FM client sends the same physical product twice, two distinct records will be created in the Solr index.
+ Another possibility is to configure the Solr File Manager to use the product name as the unique identifier. In this case, if the
+ same product is ingested for a second time, the new Solr document will completely override the previous one, resulting in only
+ one product record in the catalog.
+ Alternatively, a project can provide a custom algorithm to generate product identifiers (for example, based on the system time
+ when the product is ingested) by implementing the ProductIdGenerator interface.
+
+o Generation of Solr documents from product metadata.
+ When a physical product is sent for archiving to the File Manager, the associated metadata must be transformed into query-able information
+ that is stored in the back-end Solr catalog. By default, the Solr File Manager will transform each product into one corresponding Solr
+ document, thus generating a single searchable record in the Solr index. Each product attribute is transformed into a corresponding Solr
+ field with the same name and value(s) (note that all fields MUST be defined in the project specific schema deployed with the Solr installation).
+ Alternatively, a project may provide its own algorithm for generating Solr records from a CAS product by implementing the ProductSerializer interface.
+ For example, a project that manages products composed of full directories may wish to create a "collection"-level Solr record for the enclosing directory,
+ and separate "file"-level Solr records for each file in the directory. These different record types could be stored in the same Solr core,
+ or sent to separate Solr cores.
+
+o Generation of product metadata from Solr documents.
+ Viceversa, when a client queries the File Manager for product information, result documents are retrieved from Solr and must be transformed
+ into product objects that are presented back to the client. By default, the Solr File Manager will generate a single product for each
+ Solr result document based on the inverse rules that were used to generate the Solr document in the first place. Alternatively, a project may provide its own
+ product generation algorithm by implementing the ProductDeserializer interface, in a way that is consistent with its own implementation of the
+ ProductSerializer interface.
+
+
+Configuration
+-------------
+
+The deployment of a Solr File Manager is controlled by two main files: the File Manager filemgr.properties, and the Solr schema.xml.
+
+>>> File Manager filemgr.properties
+
+To use a Solr-based metadata catalog, the File Manager file filemgr.properties must be edited as follows.
+At a minimum the following two properties must be defined:
+
+o filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.solr.SolrCatalogFactory
+ mandatory: instructs OODT to instantiate a Solr Catalog implementation at startup
+
+o org.apache.oodt.cas.filemgr.catalog.solr.url=http://<hostname>:<port>/solr
+ mandatory: points the File Manager to the base URL of the Solr server
+
+Additionally, the following properties control how products are ingested and extracted into/from the Solr server,
+i.e. the implementation used for the extension points described above. These properties have default values,
+and need to be set only when the default is not the desired behavior.
+
+o org.apache.oodt.cas.filemgr.catalog.solr.productIdGenerator=org.apache.oodt.cas.filemgr.catalog.solr.UUIDProductIdGenerator
+ org.apache.oodt.cas.filemgr.catalog.solr.productIdGenerator=org.apache.oodt.cas.filemgr.catalog.solr.NameProductIdGenerator
+ optional: controls the algorithm for generating the product unique identifier when it is first stored in the catalog.
+ default: UUIDProductIdGenerator: this class generates a new UUID every time a product is indexed.
+ alternative out of the box implementation: NameProductIdGenerator: this class will assign the product an identifier equal to the product name.
+ alternatively: provide any custom implementation of the ProductIdGenerator interface.
+
+o org.apache.oodt.cas.filemgr.catalog.solr.productSerializer=org.apache.oodt.cas.filemgr.catalog.solr.DefaultProductSerializer
+ optional: controls the format of the documents ingested into Solr, i.e. how a CAS product object is transformed into one (or more) Solr records.
+ default: DefaultProductSerializer: creates one Solr record for each incoming CAS product:
+ -) the product core attributes (id, name, type) are converted to Solr fields starting with "CAS." ("CAS.ProductId", "CAS.ProductName", ....)
+ -) the product identifier is used again to assign the Solr record identifier (i.e. "id" and "CAS.ProductId" have the same value)
+ -) the product references are converted into Solr fields starting with ("CAS.reference..." or "CAS.root.reference...")
+ -) the product metadata attributes are converted into Solr fields with the same name and number of values
+ Alternative: any custom implementation of the ProductSerializer interface can be used.
+ Note that each specific implementation of ProductSerializer must declare the format of the generated Solr documents (XML, JSON, etc.),
+ so each implementation is free to generate Solr documents in the document rormat of choice.
+
+o org.apache.oodt.cas.filemgr.catalog.solr.productDeserializer=org.apache.oodt.cas.filemgr.catalog.solr.DefaultProductDeserializer
+ optional: controls how CAS products are queried back from the Solr index.
+ default: DefaultProductDeserializer: creates one CAS product for each returned Solr record, based on reversing the rules that created that record
+ in the first place (see DefaultProductSerializer).
+ Alternative: any custom implementation of the ProductDeserializer interface can be used,
+ but it should be consistent with the specific implementation used for the ProductSerializer implementation.
+ Note that each specific implementation of ProductDeserializer must declare the format of the Solr response documents it can parse (XML, JSON, etc.),
+ so each implementation is free to parse the document format of choice.
+
+Example of full Solr File Manager configuration (with default behavior):
+
+filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.solr.SolrCatalogFactory
+org.apache.oodt.cas.filemgr.catalog.solr.url=http://localhost:8983/solr
+#org.apache.oodt.cas.filemgr.catalog.solr.productIdGenerator=org.apache.oodt.cas.filemgr.catalog.solr.NameProductIdGenerator
+#org.apache.oodt.cas.filemgr.catalog.solr.productSerializer=org.apache.oodt.cas.filemgr.catalog.solr.DefaultProductSerializer
+#org.apache.oodt.cas.filemgr.catalog.solr.productDeserializer=org.apache.oodt.cas.filemgr.catalog.solr.DefaultProductDeserializer
+
+>>> Solr schema.xml
+
+The file schema.xml, part of each specific Solr deployment, defines which metadata fields are stored in the Solr index, and can consequently be queried
+and retrieved by clients. Note that no metadata field can be ingested in Solr unless it is defined (explicitely or implicitely) in schema.xml.
+Additionally, a specific requirement of the File Manager - Solr integration is that each metadata field included in schema.xml must be "stored" (i.e.
+defined with stored="true"), so that it can be retrieved and re-inserted during partial document updates.
+
+Each project using a Solr File Manager is responsible for creating and deploying a schema.xml file that is consistent with its own algorithms
+for generating Solr documents from product metadata, and viceversa (as defined by the specific implementations of the ProductSerializer and
+ProductDeserializer interfaces). An example schema.xml is provided as part of the File Manager distribution in the resourced/ sub-directory. This
+schema is compatible with the default implementations DefaultProductSerializer and DefaultProductDeserializer, and contains the following defninitions:
+
+o The field "id" is used as the unique Solr documents identifier (Solr default).
+
+o Solr fields corresponding to the core CAS product attributes (id, name, type, references etc.) are explicitely defined, either as single or multi-valued fields.
+
+o All other incoming fields ending in '*Date' or '*Time' are stored as single-valued Solr fields with the same name.
+
+o All other incoming metadata fields are stored as multi-valued Solr fields of type string with the same name.
+
+o All field values are copied into a catch-all field called 'text' to allow for free-text searching over the full profuct metadata.
+
+Each project is free to use the default schema.xml provided as part of the File Manager distribution, or to change it in full or in part,
+compatibly with the project algorithm for creating and retrieving Solr records.
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/Parameters.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/Parameters.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/Parameters.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/Parameters.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+
+
+/**
+ * Class holding parameters for Solr implementation of CAS File Manager.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public class Parameters {
+
+ // the Solr unique identifier field
+ public final static String ID = "id";
+
+ public final static String NS = "CAS.";
+
+ public final static String PRODUCT_ID = NS+"ProductId";
+ public final static String PRODUCT_NAME = NS+"ProductName";
+ public final static String PRODUCT_STRUCTURE = NS+"ProductStructure";
+ public final static String PRODUCT_TRANSFER_STATUS = NS+"ProductTransferStatus";
+ public final static String PRODUCT_RECEIVED_TIME = NS+"ProductReceivedTime";
+ public final static String PRODUCT_TYPE_NAME = NS+"ProductTypeName";
+ public final static String PRODUCT_TYPE_ID = NS+"ProductTypeId";
+
+ public final static String REFERENCE_ORIGINAL = NS+"ReferenceOriginal";
+ public final static String REFERENCE_DATASTORE = NS+"ReferenceDatastore";
+ public final static String REFERENCE_FILESIZE = NS+"ReferenceFileSize";
+ public final static String REFERENCE_MIMETYPE = NS+"ReferenceMimeType";
+
+ public final static String ROOT = "Root";
+ public final static String ROOT_REFERENCE_ORIGINAL = NS+ROOT+REFERENCE_ORIGINAL;
+ public final static String ROOT_REFERENCE_DATASTORE = NS+ROOT+REFERENCE_DATASTORE;
+ public final static String ROOT_REFERENCE_FILESIZE = NS+ROOT+REFERENCE_FILESIZE;
+ public final static String ROOT_REFERENCE_MIMETYPE = NS+ROOT+REFERENCE_MIMETYPE;
+
+ // required date/time format for Solr documents
+ public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+ public static final DateFormat SOLR_DATE_TIME_FORMATTER = new SimpleDateFormat(SOLR_DATE_FORMAT);
+
+ // possible mime types for communication with Solr
+ public static final String MIME_TYPE_XML = "application/xml";
+ public static final String MIME_TYPE_JSON = "application/json";
+
+ // page size for pagination
+ public static final int PAGE_SIZE = 20;
+
+ // special value that indicates that the metadata field must be deleted
+ public final static String NULL = "__NULL__";
+
+ private Parameters() {}
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/Parameters.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductIdGenerator.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductIdGenerator.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductIdGenerator.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductIdGenerator.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+
+/**
+ * API for generating a unique identifier for a CAS Product.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public interface ProductIdGenerator {
+
+ public String generateId(Product product);
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductIdGenerator.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductSerializer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductSerializer.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductSerializer.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductSerializer.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import java.util.List;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * API for serializing CAS objects (Products, Metadata and References) into documents to be indexed by Solr,
+ * and vice-versa deserializing Solr response documents into CAS objects.
+ *
+ * Implementing classes are responsible for converting the CAS fields
+ * into corresponding Solr fields as defined in their specific Solr schema.xml,
+ * and vice-versa for transforming the Solr fields into corresponding CAS fields.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public interface ProductSerializer {
+
+ /**
+ * Method to transform a CAS Product into one or more documents to be indexed by Solr.
+ *
+ * @param product : the product to be indexed
+ * @param create: true to create a new product, false to update an existing one.
+ * @return
+ */
+ List<String> serialize(Product product, boolean create);
+
+ /**
+ * Method to transform a CAS Metadata object into one or more Solr "update" documents
+ * (i.e. documents containing attributes of an existing product).
+ *
+ * @param productId : the product unique identifier
+ * @param metadata : the CAS metadata hash table
+ * @param replace : true to override the existing metadata values, false to add new values to the existing ones.
+ * Existing metadata fields not contained in the hash are left unchanged.
+ * Note: the special value Parameters.NULL is used to indicate that the corresponding key must be removed
+ * (as the Metadata object cannot store null or empty lists).
+ * @return
+ */
+ List<String> serialize(String productId, Metadata metadata, boolean replace);
+
+ /**
+ * Method to transform a CAS Product References object into one or more Solr "update" documents
+ * (i.e. documents containing attributes of an existing product).
+ *
+ * @param productId
+ * @param rootReference : the product root reference, may be null
+ * @param references : list of product references, may be empty
+ * @param replace : true to replace the existing references, false to add new references to the existing ones
+ * @return
+ */
+ List<String> serialize(String productId, Reference rootReference, List<Reference> references, boolean replace);
+
+ /**
+ * Method to parse a full XML response document into a list of Product, References and Metadata objects.
+ *
+ * @param xml : the Solr XML document to parse (input)
+ * @return
+ * @throws Exception
+ */
+ QueryResponse deserialize(String xml) throws CatalogException;
+
+ /**
+ * Mime type of the documents generated by the specific implementation (XML, JSON etc.)
+ * @return
+ */
+ String getMimeType();
+
+}
\ No newline at end of file
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/ProductSerializer.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QUICK_START_GUIDE.TXT
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QUICK_START_GUIDE.TXT?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QUICK_START_GUIDE.TXT (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QUICK_START_GUIDE.TXT Fri Apr 26 17:28:01 2013
@@ -0,0 +1,258 @@
+NOTE: THIS FILE WILL NOT BE CHECKED IN, RATHER TRANSFORMED INTO AN APACHE WIKI PAGE
+===================================================================================
+
+SOLR FILE MANAGER QUICK START GUIDE
+===================================
+
+This document is intended as a guide to quickly deploy and test a system composed of the OODT File Manager and Solr.
+For a more in-depth discussion of the Solr File manager architecture and customization, see the complete OODT Solr guide.
+
+We assume the system is deployed on a Unix platform (including Mac-OSX), and that the user has basic familiarity with the Unix operating system.
+The installation can take place in any directory on the system to where the user has access, for example under '/usr/local/oodt'. To install under any other
+directory, simply replace '/usr/local/oodt' with your custom path in the instructions below.
+
+
+Step 1: Deploy Solr within Tomcat or Jetty
+--------------------------------------------------------
+
+Detailed instructions on how to deploy Solr within a servlet container such as Tomcat or Jetty can be found on the Solr web site.
+In short, choosing to install Solr within a Tomcat servlet container:
+
+o Download and install a fairly recent version of Tomcat (at the time of this writing, apache-tomcat-7.0.34.tar):
+
+cp apache-tomcat-7.0.34.tar /usr/local/oodt/.
+cd /usr/local/oodt
+tar xvf apache-tomcat-7.0.34.tar
+ln -s ln -s ./apache-tomcat-7.0.34 ./apache-tomcat
+
+o Download and install a Solr version of 4.X or above:
+
+cp solr-4.2.0.tar /usr/local/oodt/.
+cd /usr/local/oodt
+tar xvf solr-4.2.0.tar
+ln -s ./solr-4.2.0 ./solr
+
+o install the Solr war file into the Tomcat directory:
+cp /usr/local/oodt/solr/dist/solr-4.2.0.war /usr/local/oodt/apache-tomcat/webapps/solr.war
+
+(IMPORTANT: the Solr distribution MUST be renamed to solr.war as in the example command above)
+
+o copy the example Solr configuration directory to another location for subsequent modifications:
+cp -R /usr/local/oodt/solr/example/solr /usr/local/oodt/solr-home
+
+o test your installation by starting Tomcat with a pointer to the Solr home directory:
+
+export CATALINA_OPTS='-Dsolr.solr.home=/usr/local/oodt/solr-home'
+cd /usr/local/oodt/apache-tomcat/bin
+./catalina.sh start
+
+The Solr admin home page should be available at the following URL:
+http://localhost:8080/solr
+
+o stop Tomcat
+./catalina.sh stop
+
+
+Step 2: Deploy and configure the OODT File manager
+------------------------------------
+
+o Download and install the latest version of the OODT File Manager:
+
+cp cas-filemgr-0.6-SNAPSHOT-dist.tar.gz /usr/local/oodt/.
+cd /usr/local/oodt
+tar xvfz cas-filemgr-0.6-SNAPSHOT-dist.tar.gz
+ln -s cas-filemgr-0.6-SNAPSHOT ./cas-filemgr
+
+o Configure the File Manager installation to use a Solr back-end:
+
+edit the file /usr/local/oodt/cas-filemgr/etc/filemgr.properties and make the following changes:
+
+# use a SolrCatalogFactory
+filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.solr.SolrCatalogFactory
+
+# point to the base URL of the Solr web application within the Tomcat container
+org.apache.oodt.cas.filemgr.catalog.solr.url=http://localhost:8080/solr
+
+o Configure the File Manager for ingesting products of type 'GenericFile':
+
+Still inside filemgr.properties, define the following properties:
+
+# point to the policy and mime-type specificiations for a 'GenericFile' type
+org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:///usr/local/oodt/cas-filemgr/policy/core
+org.apache.oodt.cas.filemgr.validation.dirs=file:///usr/local/oodt/cas-filemgr/policy/core
+org.apache.oodt.cas.filemgr.mime.type.repository=file:///usr/local/oodt/cas-filemgr/etc/mime-types.xml
+
+Additionally, edit the file /usr/local/oodt/cas-filemgr/policy/core/product-types.xml and set the repository
+path for 'GenericFile' products to an existing location on your system, for example:
+
+<repository path="file:///usr/local/oodt/archive"/>
+
+Create the directory if not existing:
+
+mkdir -p /usr/local/oodt/archive
+
+o start the File manager
+
+cd /usr/local/oodt/cas-filemgr/bin
+./filemgr start
+
+Step 3: Configure Solr to use the default File Manager metadata schema
+----------------------------------------------------------------------
+
+o stop Tomcat if not done already:
+
+cd /usr/local/apache-tomcat/bin
+./catalina.sh stop
+
+o copy the OODT default schema.xml to the Solr configuration directory:
+cp /usr/local/oodt/cas-filemgr/etc/schema.xml /usr/local/oodt/solr-home/collection1/conf/schema.xml
+
+o restart Tomcat:
+
+export CATALINA_OPTS='-Dsolr.solr.home=/usr/local/oodt/solr-home'
+cd /usr/local/apache-tomcat/bin
+./catalina.sh start
+
+
+Step 4: Archive a test product
+------------------------------
+
+o Generate a sample file and associated metadata:
+
+echo 'Test File' > /tmp/test.txt
+touch /tmp/test.txt.met
+edit the file /tmp/test.txt.met and insert the following content:
+
+<cas:metadata xmlns:cas="http://oodt.jpl.nasa.gov/1.0/cas">
+ <keyval>
+ <key>topic</key>
+ <val>computer science</val>
+ </keyval>
+</cas:metadata>
+
+o Request the File Manager to archive the file and associated metadata:
+
+cd /usr/local/oodt/cas-filemgr/bin
+./filemgr-client --url http://localhost:9000 --operation --ingestProduct --productName test.txt --productStructure Flat --productTypeName GenericFile --metadataFile file:///tmp/test.txt.met --refs file:///tmp/test.txt
+
+o Verify that the product has been archived:
+
+list the content of the archive directory:
+ls -lR /usr/local/oodt/archive/
+there should be a single sub-directory 'test.txt' containing a single file 'test.txt'
+
+in a browser, issue a query to Solr for all records in the index:
+http://localhost:8080/solr/select?q=*:*
+a single document should be returned, containing all the available metadata for the product just archived.
+Note how the metadata fields include the product id, the core CAS fields (starting with 'CAS....'), and additional fields parsed from the metadata file such as 'topic'.
+
+The response XML after ingesting one such document is reported below as an example:
+
+<?xml version="1.0" encoding="UTF-8"?>
+<response>
+
+<lst name="responseHeader">
+ <int name="status">0</int>
+ <int name="QTime">62</int>
+ <lst name="params">
+ <str name="indent">true</str>
+ <str name="q">*:*</str>
+ <str name="wt">xml</str>
+ </lst>
+</lst>
+<result name="response" numFound="1" start="0">
+ <doc>
+ <str name="id">137b60bb-da75-404b-a790-1f5adbd65324</str>
+ <str name="CAS.ProductId">137b60bb-da75-404b-a790-1f5adbd65324</str>
+ <str name="CAS.ProductTypeName">GenericFile</str>
+ <date name="CAS.ProductReceivedTime">2013-04-12T07:57:16Z</date>
+ <str name="CAS.ProductTransferStatus">RECEIVED</str>
+ <str name="CAS.ProductName">test.txt</str>
+ <str name="CAS.ProductTypeId">urn:oodt:GenericFile</str>
+ <str name="CAS.ProductStructure">Flat</str>
+ <arr name="topic">
+ <str>computer science</str>
+ </arr>
+ <arr name="FileLocation">
+ <str>/usr/local/oodt/archive/test.txt</str>
+ </arr>
+ <arr name="MimeType">
+ <str>text/plain</str>
+ <str>text</str>
+ <str>plain</str>
+ </arr>
+ <arr name="Filename">
+ <str>test.txt</str>
+ </arr>
+ <arr name="CAS.ReferenceFileSize">
+ <long>10</long>
+ </arr>
+ <arr name="CAS.ReferenceMimeType">
+ <str>text/plain</str>
+ </arr>
+ <arr name="CAS.ReferenceDatastore">
+ <str>file:/usr/local/oodt/archive/test.txt/test.txt</str>
+ </arr>
+ <arr name="CAS.ReferenceOriginal">
+ <str>file:///tmp/test.txt</str>
+ </arr>
+ <long name="_version_">1432118925052608512</long></doc>
+</result>
+</response>
+
+Step 5: Additional client commands examples
+-------------------------------------------
+
+o Querying the CAS catalog direcly through Solr with a browser:
+
+http://localhost:8080/solr/select/?q=*:*
+
+http://localhost:8080/solr/select/?q=science
+
+http://localhost:8080/solr/select/?q=CAS.ProductTypeName:GenericFile
+
+http://localhost:8080/solr/select/?q=CAS.ProductName:test.txt
+
+o Using the OODT query tool:
+
+cd /usr/local/oodt/cas-filemgr/bin
+
+./query_tool --url http://localhost:9000 --lucene -query 'CAS.ProductStructure:Flat'
+
+./query_tool --url http://localhost:9000 --lucene -query 'CAS.ProductTypeName:GenericFile'
+
+./query_tool --url http://localhost:9000 --sql -query 'SELECT * FROM GenericFile'
+
+./query_tool --url http://localhost:9000 --lucene -query 'CAS.ProductStructure:Flat AND CAS.ProductTransferStatus:RECEIVED'
+
+./query_tool --url http://localhost:9000 --lucene -query 'CAS.ProductName:test.txt'
+
+o Using the cas-filemgr-client:
+
+cd /usr/local/oodt/cas-filemgr/bin
+
+./filemgr-client --url http://localhost:9000 --operation --ingestProduct --productName test.txt --productStructure Flat --productTypeName GenericFile --metadataFile file:///tmp/test.txt.met --refs file:///tmp/test.txt
+
+./filemgr-client -op --deleteProductByName --productName test.txt --url http://localhost:9000
+
+./filemgr-client -op --deleteProductById --productId 14324704-2afe-4267-acad-503baf3af1d4 --url http://localhost:9000
+
+./filemgr-client -op --hasProduct --productName test.txt --url http://localhost:9000
+
+./filemgr-client -op --getProductTypeByName --productTypeName GenericFile --url http://localhost:9000
+
+./filemgr-client -op --getNumProducts --productTypeName GenericFile --url http://localhost:9000
+
+./filemgr-client -op --getProductByName --productName test.txt --url http://localhost:9000
+
+./filemgr-client -op --getProductById --productId d749a67d-d012-4f55-b5ec-b53b6a91c676 --url http://localhost:9000
+
+./filemgr-client -op --dumpMetadata --productId d749a67d-d012-4f55-b5ec-b53b6a91c676 --url http://localhost:9000
+
+./filemgr-client -op --getFirstPage --productTypeName GenericFile --url http://localhost:9000
+
+./filemgr-client -op --getNextPage --curPage 1 --productTypeName GenericFile --url http://localhost:9000
+
+./filemgr-client -op --getPrevPage --curPage 2 --productTypeName GenericFile --url http://localhost:9000
+
+./filemgr-client -op --getLastPage --productTypeName GenericFile --url http://localhost:9000
\ No newline at end of file
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QueryResponse.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QueryResponse.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QueryResponse.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QueryResponse.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+
+/**
+ * Class holding information returned from querying the Solr server.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public class QueryResponse {
+
+ /**
+ * Total number of results matching the query
+ * (NOT the number of results returned, which is typically less).
+ */
+ private int numFound = 0;
+
+ /**
+ * Offset into returned results.
+ */
+ private int start = 0;
+
+ /**
+ * List of products matching the query,
+ * with associated metadata and references.
+ */
+ private List<CompleteProduct> completeProducts;
+
+ public QueryResponse() {
+ completeProducts = new ArrayList<CompleteProduct>();
+ }
+
+ public void setNumFound(int numFound) {
+ this.numFound = numFound;
+ }
+
+ public void setStart(int start) {
+ this.start = start;
+ }
+
+ public void setResults(List<CompleteProduct> results) {
+ this.completeProducts = results;
+ }
+
+ public int getNumFound() {
+ return numFound;
+ }
+
+ public int getStart() {
+ return start;
+ }
+
+ public List<CompleteProduct> getCompleteProducts() {
+ return completeProducts;
+ }
+
+ /**
+ * Utility method to return only the 'product' part of the complete results.
+ * @return
+ */
+ public List<Product> getProducts() {
+ List<Product> products = new ArrayList<Product>();
+ for (CompleteProduct cp : completeProducts) {
+ products.add(cp.getProduct());
+ }
+ return products;
+ }
+
+
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/QueryResponse.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,543 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.oodt.cas.filemgr.catalog.Catalog;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.ProductPage;
+import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.oodt.cas.filemgr.structs.Query;
+import org.apache.oodt.cas.filemgr.structs.QueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.filemgr.validation.ValidationLayer;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.commons.pagination.PaginationUtils;
+import org.springframework.util.StringUtils;
+
+/**
+ * Implementation of the CAS {@link Catalog} interface
+ * that uses a Solr back-end metadata store.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public class SolrCatalog implements Catalog {
+
+ // Class responsible for serializing/deserializing CAS products into Solr documents
+ ProductSerializer productSerializer;
+
+ // Class responsible for generating unique identifiers for incoming Products
+ ProductIdGenerator productIdGenerator;
+
+ // Class responsible for interacting with the Solr server
+ SolrClient solrClient;
+
+ private static final Logger LOG = Logger.getLogger(SolrCatalog.class.getName());
+
+ public SolrCatalog(String solrUrl, ProductIdGenerator productIdGenerator, ProductSerializer productSerializer) {
+ this.productIdGenerator = productIdGenerator;
+ this.productSerializer = productSerializer;
+ this.solrClient = new SolrClient(solrUrl);
+ }
+
+ @Override
+ public void addMetadata(Metadata metadata, Product product) throws CatalogException {
+
+ LOG.info("Adding metadata for product:"+product.getProductName());
+
+ // serialize metadadta to Solr document(s)
+ // replace=false i.e. add metadata to existing values
+ List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false);
+
+ // send documents to Solr server
+ solrClient.index(docs, true, productSerializer.getMimeType());
+
+ }
+
+ /**
+ * This method implementation will remove the specified keys and values from the product metadata,
+ * leaving all other metadata keys and values unchanged.
+ * Note: ALL occurrences of each specified (key, value) pair will be removed, not just the first.
+ */
+ @Override
+ public void removeMetadata(Metadata metadata, Product product) throws CatalogException {
+
+ // retrieve full existing metadata
+ Metadata currentMetadata = getMetadata(product);
+
+ // metadata to be updated
+ Metadata updateMetadata = new Metadata();
+
+ // loop over keys of metadata be updated
+ for (String key : metadata.getKeys()) {
+
+ // list of values remaining after removal
+ List<String> values = new ArrayList<String>();
+ if (currentMetadata.containsKey(key)) {
+ for (String value : currentMetadata.getAllMetadata(key)) {
+ if (!metadata.getAllMetadata(key).contains(value)) {
+ values.add(value);
+ }
+ }
+
+ // add remaining values to updated metadata
+ if (values.isEmpty()) {
+ // special value because Metadata will NOT store an empty list
+ values.add(Parameters.NULL);
+ }
+ updateMetadata.addMetadata(key, values);
+
+ }
+ }
+
+ // generate Solr update documents
+ // replace=true to override existing values
+ List<String> docs = productSerializer.serialize(product.getProductId(), updateMetadata, true);
+
+ // send documents to Solr server
+ solrClient.index(docs, true, productSerializer.getMimeType());
+
+ }
+
+ /**
+ * Method that adds a Product to the Catalog,
+ * persisting its fundamental CAS attributes (id, name, type, ingestion time, etc.).
+ * This method assigns the product a unique identifier, if not existing already.
+ */
+ @Override
+ public void addProduct(Product product) throws CatalogException {
+
+ LOG.info("Adding product:"+product.getProductName());
+
+ // generate product identifier if not existing already
+ if (!StringUtils.hasText(product.getProductId())) {
+ String productId = this.productIdGenerator.generateId(product);
+ product.setProductId(productId);
+ }
+
+ // serialize product for ingestion into Solr
+ List<String> docs = productSerializer.serialize(product, true); // create=true
+
+ // send records to Solr
+ solrClient.index(docs, true, productSerializer.getMimeType());
+
+ }
+
+ @Override
+ public void modifyProduct(Product product) throws CatalogException {
+
+ LOG.info("Modifying product:"+product.getProductName());
+
+ // serialize the update product information to Solr document(s)
+ List<String> docs = productSerializer.serialize(product, false); // create=false
+
+ // send records to Solr
+ solrClient.index(docs, true, productSerializer.getMimeType());
+
+
+ }
+
+ @Override
+ public void removeProduct(Product product) throws CatalogException {
+
+ // send message to Solr server
+ solrClient.delete(product.getProductId(), true);
+
+ }
+
+ @Override
+ public void setProductTransferStatus(Product product) throws CatalogException {
+
+ this.modifyProduct(product);
+
+ }
+
+ @Override
+ public void addProductReferences(Product product) throws CatalogException {
+
+ // generate update documents (with replace=true)
+ List<String> docs = productSerializer.serialize(product.getProductId(), product.getRootRef(), product.getProductReferences(), true);
+
+ // send documents to Solr server
+ solrClient.index(docs, true, productSerializer.getMimeType());
+
+ }
+
+ @Override
+ public Product getProductById(String productId) throws CatalogException {
+
+ CompleteProduct cp = getCompleteProductById(productId);
+ return cp.getProduct();
+
+ }
+
+ @Override
+ public Product getProductByName(String productName) throws CatalogException {
+
+ CompleteProduct cp = getCompleteProductByName(productName);
+ LOG.info("Found product name="+productName+" id="+cp.getProduct().getProductId());
+ return cp.getProduct();
+
+ }
+
+ @Override
+ public List<Reference> getProductReferences(Product product) throws CatalogException {
+
+ CompleteProduct cp = getCompleteProductById(product.getProductId());
+ return cp.getProduct().getProductReferences();
+
+ }
+
+ @Override
+ public List<Product> getProducts() throws CatalogException {
+
+ // build query parameters
+ Map<String, String[]> params = new HashMap<String, String[]>();
+ params.put("q", new String[] { "*:*" } );
+ //params.put("rows", new String[] { "20" } ); // control pagination ?
+
+ // query Solr for all matching products
+ return getProducts(params, 0, -1).getProducts(); // get ALL products
+
+ }
+
+ /**
+ * Common utility to retrieve a range of products matching the specified {@link Query} and {@link ProductType}.
+ * This method transforms the given constraints in a map of HTTP (name, value) pairs and delegates to the following method.
+ *
+ * @param query
+ * @param type
+ * @param offset
+ * @param limit
+ * @return
+ * @throws CatalogException
+ */
+ private QueryResponse getProducts(Query query, ProductType type, int offset, int limit) throws CatalogException {
+
+ // build HTTP request
+ HashMap<String, String[]> params = new HashMap<String, String[]>();
+ // product type constraint
+ params.put("q", new String[]{Parameters.PRODUCT_TYPE_NAME+":"+type.getName()} );
+ // convert filemgr query into a Solr query
+ List<String> qc = new ArrayList<String>();
+ for (QueryCriteria queryCriteria : query.getCriteria()) {
+ LOG.info("Query criteria="+queryCriteria.toString());
+ qc.add(queryCriteria.toString());
+ }
+ params.put("fq", qc.toArray( new String[ qc.size() ] ));
+ // sort
+ params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
+
+ return this.getProducts(params, offset, limit);
+
+ }
+
+ /**
+ * Common utility to retrieve a range of products matching the specified query parameters
+ * @param params : HTTP query parameters used for query to Solr
+ * @param offset : the index of the first result, starting at 0
+ * @param limit : the maximum number of returned results (use -1 to return ALL results)
+ * @return
+ */
+ private QueryResponse getProducts(Map<String, String[]> params, int offset, int limit) throws CatalogException {
+
+ // combined results from pagination
+ QueryResponse queryResponse = new QueryResponse();
+ queryResponse.setStart(offset);
+ int start = offset;
+ while (queryResponse.getCompleteProducts().size()<limit || limit<0) {
+
+ params.put("start", new String[] { ""+start } );
+ String response = solrClient.query(params, productSerializer.getMimeType());
+ QueryResponse qr = productSerializer.deserialize(response);
+
+ for (CompleteProduct cp : qr.getCompleteProducts()) {
+ if (queryResponse.getCompleteProducts().size()<limit) {
+ queryResponse.getCompleteProducts().add( cp );
+ }
+ }
+
+ queryResponse.setNumFound( qr.getNumFound() );
+ start = offset+queryResponse.getCompleteProducts().size();
+ if (limit<0) limit = queryResponse.getNumFound(); // retrieve ALL results
+ if (start>=queryResponse.getNumFound()) break; // don't query any longer
+
+ }
+
+ LOG.info("Total number of products found="+queryResponse.getNumFound());
+ LOG.info("Total number of products returned="+queryResponse.getCompleteProducts().size());
+ return queryResponse;
+
+ }
+
+ @Override
+ public List<Product> getProductsByProductType(ProductType type) throws CatalogException {
+
+ // build query parameters
+ Map<String, String[]> params = new HashMap<String, String[]>();
+ params.put("q", new String[] { "*:*" } );
+ // use the product type name as query parameter
+ params.put("fq", new String[] { Parameters.PRODUCT_TYPE_NAME+":"+type.getName() } );
+
+ // query Solr for all matching products
+ return getProducts(params, 0, -1).getProducts(); // get ALL products
+
+ }
+
+ @Override
+ public Metadata getMetadata(Product product) throws CatalogException {
+
+ CompleteProduct cp = getCompleteProductById(product.getProductId());
+ return cp.getMetadata();
+
+ }
+
+ @Override
+ public Metadata getReducedMetadata(Product product, List<String> elements) throws CatalogException {
+
+ // build HTTP request
+ HashMap<String, String[]> params = new HashMap<String, String[]>();
+ params.put("q", new String[]{Parameters.PRODUCT_ID+":"+product.getProductId()} );
+ // request metadata elements explicitly
+ params.put("fl", elements.toArray(new String[elements.size()]) );
+
+ // execute request
+ String doc = solrClient.query(params, productSerializer.getMimeType());
+
+ // parse response
+ CompleteProduct cp = extractCompleteProduct(doc);
+ return cp.getMetadata();
+
+ }
+
+ /**
+ * {@inheritDoc}
+ * Note that this method implementation will return ALL products in the catalog matching the query criteria.
+ */
+ @Override
+ public List<String> query(Query query, ProductType type) throws CatalogException {
+
+ // execute request for ALL results
+ QueryResponse queryResponse = this.getProducts(query, type, 0, -1); // get ALL products
+
+ // extract ids from products
+ List<String> ids = new ArrayList<String>();
+ for (CompleteProduct cp : queryResponse.getCompleteProducts()) {
+ ids.add(cp.getProduct().getProductId());
+ }
+
+ return ids;
+
+ }
+
+ @Override
+ public List<Product> getTopNProducts(int n) throws CatalogException {
+
+ // retrieve most recent n products from Solr
+ String doc = solrClient.queryProductsByDate(n, productSerializer.getMimeType());
+
+ // parse Solr response into Product objects
+ return this.getProductsFromDocument(doc);
+
+ }
+
+ @Override
+ public ProductPage pagedQuery(Query query, ProductType type, int pageNum) throws CatalogException {
+
+
+ // execute request for one page of results
+ int offset = (pageNum-1)*Parameters.PAGE_SIZE;
+ int limit = Parameters.PAGE_SIZE;
+ QueryResponse queryResponse = this.getProducts(query, type, offset, limit);
+
+ // build product page from query response
+ return newProductPage(pageNum, queryResponse);
+
+ }
+
+ @Override
+ public ProductPage getFirstPage(ProductType type) {
+ try {
+ return this.pagedQuery(new Query(), type, 1);
+
+ } catch(CatalogException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ }
+
+ @Override
+ public ProductPage getLastProductPage(ProductType type) {
+
+ try {
+
+ // query for total number of products of this type
+ int numTotalResults = this.getNumProducts(type);
+
+ // compute last page number
+ int numOfPages = PaginationUtils.getTotalPage(numTotalResults, Parameters.PAGE_SIZE);
+
+ // request last page
+ return pagedQuery(new Query(), type, numOfPages);
+
+ } catch(CatalogException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ }
+
+ @Override
+ public ProductPage getNextPage(ProductType type, ProductPage currentPage) {
+
+ int nextPageNumber = currentPage.getPageNum()+1;
+ if (nextPageNumber>currentPage.getTotalPages()) throw new RuntimeException("Invalid next page number: "+nextPageNumber);
+
+ try {
+ return this.pagedQuery(new Query(), type, currentPage.getPageNum()+1);
+
+ } catch(CatalogException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ }
+
+ @Override
+ public ProductPage getPrevPage(ProductType type, ProductPage currentPage) {
+
+ int prevPageNumber = currentPage.getPageNum()-1;
+ if (prevPageNumber<=0) throw new RuntimeException("Invalid previous page number: "+prevPageNumber);
+
+ try {
+ return this.pagedQuery(new Query(), type, prevPageNumber);
+
+ } catch(CatalogException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ }
+
+ /**
+ * Common functionality for extracting products from a Solr response document.
+ * @param doc
+ * @return
+ */
+ private List<Product> getProductsFromDocument(String doc) throws CatalogException {
+
+ // extract full product information from Solr response
+ QueryResponse queryResponse = productSerializer.deserialize(doc);
+
+ // return products only
+ return queryResponse.getProducts();
+
+ }
+
+ @Override
+ public List<Product> getTopNProducts(int n, ProductType type) throws CatalogException {
+
+ // retrieve most recent n products from Solr
+ String doc = solrClient.queryProductsByDateAndType(n, type, productSerializer.getMimeType());
+
+ // parse Solr response into Product objects
+ return this.getProductsFromDocument(doc);
+
+ }
+
+ @Override
+ public ValidationLayer getValidationLayer() throws CatalogException {
+ // FIXME: must parse Solr schema.xmnl from:
+ // http://localhost:8080/solr/admin/file/?contentType=text/xml;charset=utf-8&file=schema.xml
+ throw new RuntimeException("Method 'getValidationLayer' not yet implemented");
+ }
+
+ @Override
+ public int getNumProducts(ProductType type) throws CatalogException {
+
+ // build query parameters
+ Map<String, String[]> params = new HashMap<String, String[]>();
+ params.put("q", new String[] { "*:*" } );
+ params.put("rows", new String[] { "0" } ); // don't return any results
+
+ // execute query
+ String response = solrClient.query(params, productSerializer.getMimeType());
+
+ // parse response
+ QueryResponse queryResponse = productSerializer.deserialize(response);
+ return queryResponse.getNumFound();
+
+ }
+
+ private CompleteProduct getCompleteProductById(String productId) throws CatalogException {
+
+ // request document with given id
+ String doc = solrClient.queryProductById(productId, productSerializer.getMimeType());
+
+ // parse document into complete product
+ return extractCompleteProduct(doc);
+
+ }
+
+ private CompleteProduct getCompleteProductByName(String productName) throws CatalogException {
+
+ // request document with given id
+ String doc = solrClient.queryProductByName(productName, productSerializer.getMimeType());
+
+ // parse document into complete product
+ return extractCompleteProduct(doc);
+
+ }
+
+ private CompleteProduct extractCompleteProduct(String doc) throws CatalogException {
+
+ // deserialize document into Product
+ LOG.info("Parsing Solr document: "+doc);
+ QueryResponse queryResponse = productSerializer.deserialize(doc);
+ int numFound = queryResponse.getNumFound();
+ if (numFound>1) {
+ throw new CatalogException("Product query returned "+numFound+" results instead of 1!");
+ } else if (numFound==0) {
+ throw new CatalogException("No products found in response document");
+ } else {
+ return queryResponse.getCompleteProducts().get(0);
+ }
+
+ }
+
+ /**
+ * Factory method to create a {@link ProductPage} from a {@link QueryResponse}.
+ * @param pageNumber
+ * @param queryResponse
+ * @return
+ */
+ private ProductPage newProductPage(int pageNum, QueryResponse queryResponse) {
+
+ ProductPage page = new ProductPage();
+ page.setPageNum(pageNum);
+ page.setPageSize(queryResponse.getProducts().size());
+ page.setNumOfHits(queryResponse.getNumFound());
+ page.setPageProducts(queryResponse.getProducts());
+ page.setTotalPages(PaginationUtils.getTotalPage(queryResponse.getNumFound(), Parameters.PAGE_SIZE));
+ return page;
+
+ }
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalogFactory.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalogFactory.java?rev=1476304&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalogFactory.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalogFactory.java Fri Apr 26 17:28:01 2013
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.catalog.solr;
+
+import java.util.logging.Logger;
+
+import org.apache.oodt.cas.filemgr.catalog.Catalog;
+import org.apache.oodt.cas.filemgr.catalog.CatalogFactory;
+import org.apache.oodt.cas.metadata.util.PathUtils;
+
+/**
+ * Factory class that creates a {@link SolrCatalog} instance
+ * configured with the properties specified in filemgr.properties.
+ *
+ * @author Luca Cinquini
+ *
+ */
+public class SolrCatalogFactory implements CatalogFactory {
+
+ private String solrUrl;
+ private ProductIdGenerator productIdGenerator;
+ private ProductSerializer productSerializer;
+
+ private static final Logger LOG = Logger.getLogger(SolrCatalogFactory.class.getName());
+
+ public SolrCatalogFactory() throws IllegalArgumentException {
+
+ // base Solr URL
+ String solrUrl = System.getProperty("org.apache.oodt.cas.filemgr.catalog.solr.url");
+ if (solrUrl==null) {
+ throw new IllegalArgumentException("Invalid Solr URL specified from property 'org.apache.oodt.cas.filemgr.catalog.solr.url'");
+ }
+
+ this.solrUrl = PathUtils.replaceEnvVariables( solrUrl );
+ this.configure();
+
+ }
+
+ /**
+ * Method to configure the specific flavor of the Solr catalog implementation
+ * before the catalog is instantiated.
+ */
+ private void configure() {
+
+ // product serializer
+ String productSerializerClass = System.getProperty("org.apache.oodt.cas.filemgr.catalog.solr.productSerializer");
+ if (productSerializerClass!=null) {
+ try {
+ productSerializer = (ProductSerializer)Class.forName( PathUtils.replaceEnvVariables(productSerializerClass) ).newInstance();
+ } catch(Exception e) {
+ LOG.severe(e.getMessage());
+ System.exit(-1);
+ }
+ } else {
+ productSerializer = new DefaultProductSerializer();
+ }
+
+ // product id generator
+ String productIdGeneratorClass = System.getProperty("org.apache.oodt.cas.filemgr.catalog.solr.productIdGenerator");
+ if (productIdGeneratorClass!=null) {
+ try {
+ productIdGenerator = (ProductIdGenerator)Class.forName( PathUtils.replaceEnvVariables(productIdGeneratorClass) ).newInstance();
+ } catch(Exception e) {
+ LOG.severe(e.getMessage());
+ System.exit(-1);
+ }
+ } else {
+ productIdGenerator = new UUIDProductIdGenerator();
+ }
+
+ }
+
+ @Override
+ public Catalog createCatalog() {
+ LOG.info("Creating Solr Catalog for URL="+this.solrUrl);
+ return new SolrCatalog(solrUrl, productIdGenerator, productSerializer);
+ }
+
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalogFactory.java
------------------------------------------------------------------------------
svn:mime-type = text/plain