You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/09/03 18:57:02 UTC
oodt git commit: Merge master versions of files.
Repository: oodt
Updated Branches:
refs/heads/development 835f16d08 -> 9b0b61ec0
Merge master versions of files.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9b0b61ec
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9b0b61ec
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9b0b61ec
Branch: refs/heads/development
Commit: 9b0b61ec0e1fd5c36a1ea9c9902a947ba0d20ba1
Parents: 835f16d
Author: Chris Mattmann <ma...@apache.org>
Authored: Sun Sep 3 11:56:59 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sun Sep 3 11:56:59 2017 -0700
----------------------------------------------------------------------
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 10 +-
.../oodt/cas/filemgr/tools/DeleteProduct.java | 303 ++++++++++---------
.../oodt/cas/filemgr/tools/QueryTool.java | 127 ++++----
.../oodt/cas/filemgr/tools/SolrIndexer.java | 228 +++++++-------
.../LuceneWorkflowInstanceRepository.java | 53 +++-
5 files changed, 384 insertions(+), 337 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index a8c0253..59ded25 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -902,7 +902,8 @@ public class LuceneCatalog implements Catalog {
*/
public ProductPage getNextPage(ProductType type, ProductPage currentPage) {
if(type==null){
- return null;
+ LOG.warning("getNextPage: Provided type was null: Returning blank page.");
+ return ProductPage.blankPage();
}
if (currentPage == null) {
return getFirstPage(type);
@@ -926,11 +927,11 @@ public class LuceneCatalog implements Catalog {
"CatalogException getting next page for product type: ["
+ type.getProductTypeId()
+ "] from catalog: Message: " + e.getMessage());
- return null;
+ return ProductPage.blankPage();
}
// There are no products and thus no next page
if (products == null || (products.size() == 0)) {
- return null;
+ return ProductPage.blankPage();
}
nextPage.setPageProducts(products);
@@ -945,7 +946,8 @@ public class LuceneCatalog implements Catalog {
*/
public ProductPage getPrevPage(ProductType type, ProductPage currentPage) {
if(type==null){
- return null;
+ LOG.warning("getPrevPage: Provided type was null: Returning blank page.");
+ return ProductPage.blankPage();
}
if (currentPage == null) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
index 42b07e9..654c85a 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
@@ -22,9 +22,7 @@ import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
-import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
//JDK imports
import java.io.BufferedReader;
@@ -43,180 +41,193 @@ import java.util.Vector;
* @author woollard
* @version $Revision$
*
- * <p>
- * A utility class that deletes products in the File Manager Catalog based on
- * productID.
- * </p>
+ * <p>
+ * A utility class that deletes products in the File Manager Catalog
+ * based on productID.
+ * </p>
*
*/
-@Deprecated
public class DeleteProduct {
- /* our log stream */
- private static final Logger LOG = Logger.getLogger(DeleteProduct.class.getName());
+ /* our log stream */
+ private static final Logger LOG = Logger
+ .getLogger(DeleteProduct.class.getName());
- /* our File Manager client */
- private FileManagerClient client = null;
+ /* our File Manager client */
+ private XmlRpcFileManagerClient client = null;
- /* whether or not we should commit our deletions */
- private boolean commit = true;
+ /* whether or not we should commit our deletions */
+ private boolean commit = true;
- public DeleteProduct(String fileManagerUrl, boolean commit) {
- try {
- client = RpcCommunicationFactory.createClient(new URL(fileManagerUrl));
- } catch (Exception e) {
- LOG.log(Level.SEVERE,
- "Unable to create file manager client: Message: "
- + e.getMessage() + ": errors to follow");
- }
+ public DeleteProduct(String fileManagerUrl, boolean commit) {
+ try {
+ client = new XmlRpcFileManagerClient(new URL(fileManagerUrl));
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "Unable to create file manager client: Message: "
+ + e.getMessage() + ": errors to follow");
+ }
- this.commit = commit;
+ this.commit = commit;
- if (!this.commit) {
- LOG.log(Level.INFO, "Commit disabled.");
- } else {
- LOG.log(Level.INFO, "Commit enabled.");
- }
+ if (!this.commit) {
+ LOG.log(Level.INFO, "Commit disabled.");
+ } else {
+ LOG.log(Level.INFO, "Commit enabled.");
}
+ }
- /**
- * @param args
- */
- public static void main(String[] args) {
- String productId = null;
- String fileManagerUrl = null;
- boolean commitChanges = true;
- boolean readFromStdIn = false;
-
- String usage = "DeleteProduct --productID <product id> "
- + "--fileManagerUrl <url to file manager> [--read] [--nocommit]\n";
-
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("--productID")) {
- productId = args[++i];
- } else if (args[i].equals("--fileManagerUrl")) {
- fileManagerUrl = args[++i];
- } else if (args[i].equals("--read")) {
- readFromStdIn = true;
- } else if (args[i].equals("--nocommit")) {
- commitChanges = false;
- }
- }
-
- if ((productId == null && !readFromStdIn) || fileManagerUrl == null) {
- System.err.println(usage);
- System.exit(1);
- }
+ public void remove(String productId) {
+ Product target = null;
- DeleteProduct remover = new DeleteProduct(fileManagerUrl, commitChanges);
- if (readFromStdIn) {
- List prodIds = readProdIdsFromStdin();
- for (Object prodId1 : prodIds) {
- String prodId = (String) prodId1;
- remover.remove(prodId);
- }
- } else {
- remover.remove(productId);
- }
+ try {
+ target = client.getProductById(productId);
+ } catch (CatalogException e) {
+ LOG.log(Level.WARNING,
+ "Unable to obtain product : [" + productId + "] from file manager: ["
+ + client.getFileManagerUrl() + "]: Message: " + e.getMessage());
+ }
+ if (target == null) {
+ // could not file product
+ return;
}
- private static List readProdIdsFromStdin() {
- List prodIds = new Vector();
- BufferedReader br;
+ // delete references first
+ Vector refs = new Vector();
- br = new BufferedReader(new InputStreamReader(System.in));
+ try {
+ refs = (Vector) client.getProductReferences(target);
+ } catch (CatalogException e) {
+ LOG.log(Level.WARNING,
+ "Unable to obtain references for product : [" + productId
+ + "] from file manager: [" + client.getFileManagerUrl()
+ + "]: Message: " + e.getMessage());
+ }
- String line = null;
+ for (Object ref1 : refs) {
+ Reference ref = (Reference) ref1;
+ if (commit) {
try {
- while ((line = br.readLine()) != null) {
- prodIds.add(line);
- }
- } catch (IOException e) {
- LOG.log(Level.WARNING, "Error reading prod id: line: [" + line
- + "]: Message: " + e.getMessage(), e);
- } finally {
- try {
- br.close();
- } catch (Exception ignore) {
- }
-
+ client.removeFile(
+ new File(new URI(ref.getDataStoreReference())).getAbsolutePath());
+ } catch (DataTransferException e) {
+ LOG.log(Level.WARNING,
+ "Unable to delete reference : [" + ref.getDataStoreReference()
+ + "] for product : [" + productId + "] from file manager : ["
+ + client.getFileManagerUrl() + "]: Message: "
+ + e.getMessage());
+ } catch (URISyntaxException e) {
+ LOG.log(Level.WARNING,
+ "uri syntax exception getting file absolute path from URI: ["
+ + ref.getDataStoreReference() + "]: Message: "
+ + e.getMessage());
}
+ } else {
+ LOG.log(Level.INFO,
+ "Delete file: [" + ref.getDataStoreReference() + "]");
+ }
+
+ }
- return prodIds;
+ if (commit) {
+
+ // now delete product
+ try {
+ client.removeProduct(target);
+ } catch (CatalogException e) {
+ LOG.log(Level.WARNING,
+ "Unable to remove product : [" + productId
+ + "] from file manager: [" + client.getFileManagerUrl()
+ + "]: Message: " + e.getMessage());
+ }
+ } else {
+ LOG.log(Level.INFO, "Remote catalog entry for product: ["
+ + target.getProductName() + "]");
}
- private void remove(String productId) {
- Product target = null;
+ }
+
+ /**
+ * @return the commit
+ */
+ public boolean isCommit() {
+ return commit;
+ }
+
+ /**
+ * @param commit
+ * the commit to set
+ */
+ public void setCommit(boolean commit) {
+ this.commit = commit;
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ String productId = null;
+ String fileManagerUrl = null;
+ boolean commitChanges = true;
+ boolean readFromStdIn = false;
+
+ String usage = "DeleteProduct --productID <product id> "
+ + "--fileManagerUrl <url to file manager> [--read] [--nocommit]\n";
+
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("--productID")) {
+ productId = args[++i];
+ } else if (args[i].equals("--fileManagerUrl")) {
+ fileManagerUrl = args[++i];
+ } else if (args[i].equals("--read")) {
+ readFromStdIn = true;
+ } else if (args[i].equals("--nocommit")) {
+ commitChanges = false;
+ }
+ }
- try {
- target = client.getProductById(productId);
- } catch (CatalogException e) {
- LOG.log(Level.WARNING, "Unable to obtain product : [" + productId
- + "] from file manager: [" + client.getFileManagerUrl()
- + "]: Message: " + e.getMessage());
- }
+ if ((productId == null && !readFromStdIn) || fileManagerUrl == null) {
+ System.err.println(usage);
+ System.exit(1);
+ }
- if (target == null) {
- // could not file product
- return;
- }
+ DeleteProduct remover = new DeleteProduct(fileManagerUrl, commitChanges);
+ if (readFromStdIn) {
+ List prodIds = readProdIdsFromStdin();
+ for (Object prodId1 : prodIds) {
+ String prodId = (String) prodId1;
+ remover.remove(prodId);
+ }
+ } else {
+ remover.remove(productId);
+ }
- // delete references first
- Vector refs = new Vector();
+ }
- try {
- refs = (Vector) client.getProductReferences(target);
- } catch (CatalogException e) {
- LOG.log(Level.WARNING,
- "Unable to obtain references for product : [" + productId
- + "] from file manager: ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
- }
+ private static List readProdIdsFromStdin() {
+ List prodIds = new Vector();
+ BufferedReader br;
- for (Object ref1 : refs) {
- Reference ref = (Reference) ref1;
-
- if (commit) {
- try {
- client.removeFile(new File(new URI(ref
- .getDataStoreReference())).getAbsolutePath());
- } catch (DataTransferException e) {
- LOG.log(Level.WARNING, "Unable to delete reference : ["
- + ref.getDataStoreReference() + "] for product : ["
- + productId + "] from file manager : ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING,
- "uri syntax exception getting file absolute path from URI: ["
- + ref.getDataStoreReference()
- + "]: Message: " + e.getMessage());
- }
- } else {
- LOG.log(Level.INFO, "Delete file: ["
- + ref.getDataStoreReference() + "]");
- }
+ br = new BufferedReader(new InputStreamReader(System.in));
- }
+ String line = null;
- if (commit) {
-
- // now delete product
- try {
- client.removeProduct(target);
- } catch (CatalogException e) {
- LOG.log(Level.WARNING, "Unable to remove product : ["
- + productId + "] from file manager: ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
- }
- } else {
- LOG.log(Level.INFO, "Remote catalog entry for product: ["
- + target.getProductName() + "]");
- }
+ try {
+ while ((line = br.readLine()) != null) {
+ prodIds.add(line);
+ }
+ } catch (IOException e) {
+ LOG.log(Level.WARNING, "Error reading prod id: line: [" + line
+ + "]: Message: " + e.getMessage(), e);
+ } finally {
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
}
+
+ return prodIds;
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
index 7b37f90..27e37d2 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
@@ -32,9 +32,7 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.QueryFormulationException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery;
import org.apache.oodt.cas.filemgr.structs.query.QueryResult;
-import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.util.SqlParser;
import java.net.MalformedURLException;
@@ -53,12 +51,11 @@ import java.util.logging.Logger;
* A tool to return product ids given a {@link Query} against the File Manager.
* </p>
*/
-@Deprecated
public final class QueryTool {
private static String freeTextBlock = "__FREE__";
- private FileManagerClient client = null;
+ private XmlRpcFileManagerClient client = null;
private enum QueryType { LUCENE, SQL }
@@ -67,26 +64,12 @@ public final class QueryTool {
public QueryTool(URL fmUrl) throws InstantiationException {
try {
- client = RpcCommunicationFactory.createClient(fmUrl);
+ client = new XmlRpcFileManagerClient(fmUrl);
} catch (ConnectionException e) {
throw new InstantiationException(e.getMessage());
}
}
- public static Query parseQuery(String query) {
- QueryParser parser;
- // note that "__FREE__" is a control work for free text searching
- parser = new QueryParser(freeTextBlock, new CASAnalyzer());
-
- Query luceneQ = null;
- try {
- luceneQ = (Query) parser.parse(query);
- } catch (ParseException e) {
- System.out.println("Error parsing query text.");
- System.exit(-1);
- }
- return luceneQ;
- }
public List query(org.apache.oodt.cas.filemgr.structs.Query query) {
List prodIds = new Vector();
@@ -118,55 +101,55 @@ public final class QueryTool {
return prodIds;
}
-
+
+
+ public static Query parseQuery(String query) {
+ QueryParser parser;
+ // note that "__FREE__" is a control work for free text searching
+ parser = new QueryParser(freeTextBlock, new CASAnalyzer());
+ Query luceneQ = null;
+ try {
+ luceneQ = (Query) parser.parse(query);
+ } catch (ParseException e) {
+ System.out.println("Error parsing query text.");
+ System.exit(-1);
+ }
+ return luceneQ;
+ }
+
public void generateCASQuery(
- org.apache.oodt.cas.filemgr.structs.Query casQuery,
- Query luceneQuery) {
- if (luceneQuery instanceof TermQuery) {
- Term t = ((TermQuery) luceneQuery).getTerm();
- if (!t.field().equals(freeTextBlock)) {
- casQuery.addCriterion(new TermQueryCriteria(t.field(),
- t.text()));
- }
- } else if (luceneQuery instanceof PhraseQuery) {
- Term[] t = ((PhraseQuery) luceneQuery).getTerms();
- if (!t[0].field().equals(freeTextBlock)) {
- for (Term aT : t) {
- casQuery.addCriterion(new TermQueryCriteria(
- aT.field(), aT.text()));
- }
- }
- } else if (luceneQuery instanceof TermRangeQuery) {
- BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
- BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
- casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
- } else if (luceneQuery instanceof BooleanQuery) {
- List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
- for (BooleanClause clause : clauses) {
- generateCASQuery(casQuery, (clause).getQuery());
+ org.apache.oodt.cas.filemgr.structs.Query casQuery,
+ Query luceneQuery) {
+ if (luceneQuery instanceof TermQuery) {
+ Term t = ((TermQuery) luceneQuery).getTerm();
+ if (!t.field().equals(freeTextBlock)) {
+ casQuery.addCriterion(new TermQueryCriteria(t.field(),
+ t.text()));
+ }
+ } else if (luceneQuery instanceof PhraseQuery) {
+ Term[] t = ((PhraseQuery) luceneQuery).getTerms();
+ if (!t[0].field().equals(freeTextBlock)) {
+ for (Term aT : t) {
+ casQuery.addCriterion(new TermQueryCriteria(
+ aT.field(), aT.text()));
}
- } else {
- throw new RuntimeException(
- "Error parsing query! Cannot determine clause type: ["
- + luceneQuery.getClass().getName() + "] !");
}
- }
-
- private List safeGetProductTypes() {
- List prodTypes = null;
-
- try {
- prodTypes = client.getProductTypes();
- } catch (RepositoryManagerException e) {
- LOG.log(Level.WARNING,
- "Error obtaining product types from file manager: ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+ casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
+ } else if (luceneQuery instanceof BooleanQuery) {
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
+ for (BooleanClause clause : clauses) {
+ generateCASQuery(casQuery, (clause).getQuery());
}
-
- return prodTypes;
+ } else {
+ throw new RuntimeException(
+ "Error parsing query! Cannot determine clause type: ["
+ + luceneQuery.getClass().getName() + "] !");
}
-
+}
+
public static void main(String[] args)
throws MalformedURLException, InstantiationException, CatalogException, QueryFormulationException,
ConnectionException {
@@ -181,7 +164,7 @@ public final class QueryTool {
+ " -query <query> \n"
+ " -sortBy <metadata-key> \n"
+ " -outputFormat <output-format-string> \n";
-
+
String fmUrlStr = null, queryStr = null, sortBy = null, outputFormat = null, delimiter = null;
QueryType queryType = null;
for (int i = 0; i < args.length; i++) {
@@ -241,6 +224,22 @@ public final class QueryTool {
}
}
+
+
+ private List safeGetProductTypes() {
+ List prodTypes = null;
+
+ try {
+ prodTypes = client.getProductTypes();
+ } catch (RepositoryManagerException e) {
+ LOG.log(Level.WARNING,
+ "Error obtaining product types from file manager: ["
+ + client.getFileManagerUrl() + "]: Message: "
+ + e.getMessage());
+ }
+
+ return prodTypes;
+ }
private static String performSqlQuery(String query, String sortBy, String outputFormat, String delimiter, String filemgrUrl)
throws MalformedURLException, CatalogException, ConnectionException, QueryFormulationException {
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
index 6d453bd..16ca7f1 100755
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
@@ -27,7 +27,6 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductPage;
@@ -39,8 +38,9 @@ import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.springframework.util.StringUtils;
@@ -73,31 +73,31 @@ public class SolrIndexer {
private final static String ACCESS_KEY = "access.key";
private final static String ACCESS_URL = "access.url";
private final static String PRODUCT_NAME = "CAS.ProductName";
- private final HttpSolrClient server;
private IndexerConfig config = null;
+ private final SolrServer server;
private String fmUrl;
private String solrUrl;
private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName());
private final static SimpleDateFormat solrFormat = new SimpleDateFormat(
- "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+ "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
/**
* Constructor reads in the configuration and initiates the connection to the
* Solr instance.
- *
+ *
* @param solrUrl
* URL for the Solr instance.
* @param fmUrl
* URL for the File Manager instance.
*/
public SolrIndexer(String solrUrl, String fmUrl)
- throws InstantiationException {
+ throws InstantiationException {
InputStream input = null;
String filename;
try {
LOG.info("System property " + SOLR_INDEXER_CONFIG + " set to "
- + System.getProperty(SOLR_INDEXER_CONFIG));
+ + System.getProperty(SOLR_INDEXER_CONFIG));
filename = System.getProperty(SOLR_INDEXER_CONFIG);
if (filename != null) {
LOG.info("Reading config from " + filename);
@@ -109,7 +109,7 @@ public class SolrIndexer {
config = new IndexerConfig(input);
} catch (IOException e) {
LOG
- .severe("Could not read in configuration for indexer from classpath or file");
+ .severe("Could not read in configuration for indexer from classpath or file");
throw new InstantiationException(e.getMessage());
} finally {
if (input != null) {
@@ -132,10 +132,14 @@ public class SolrIndexer {
}
LOG.info("Using Solr: " + this.solrUrl + " FileManager: " + this.fmUrl);
- SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
- server = new HttpSolrClient.Builder(this.solrUrl).build();
-// server = new SolrClient(this.solrUrl);
+
+ try {
+ server = new CommonsHttpSolrServer(this.solrUrl);
+ } catch (MalformedURLException e) {
+ LOG.severe("Could not connect to Solr server " + this.solrUrl);
+ throw new InstantiationException(e.getMessage());
+ }
}
@@ -162,7 +166,7 @@ public class SolrIndexer {
/**
* This method transforms the product metadata into a Solr document.
- *
+ *
* @param metadata
* The metadata object for the product to index.
* @return Returns the SolrInputDocument containing product metadata.
@@ -170,8 +174,6 @@ public class SolrIndexer {
private SolrInputDocument getSolrDocument(Metadata metadata) {
SolrInputDocument doc = new SolrInputDocument();
// Only grab metadata which have a mapping in the indexer.properties
- String official = "";
-
for (Object objKey : config.getMapProperties().keySet()) {
// The key in the metadata object
String key = (String) objKey;
@@ -182,21 +184,19 @@ public class SolrIndexer {
for (String value : values) {
// Add each metadata value into the
if (value != null && !config.getIgnoreValues().contains(value.trim())) {
- official += " Key: " + key + " Value: " + value;
+ LOG.fine("Adding field: " + fieldName + " value: " + value);
doc.addField(fieldName, value);
}
}
}
}
- // LOG.info("KEYS WERE:" +official);
-
return doc;
}
/**
* This method adds a single product extracted from a metadata file to the
* Solr index.
- *
+ *
* @param file
* The file containing product metadata.
* @param delete
@@ -206,24 +206,24 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexMetFile(File file, boolean delete)
- throws
- SolrServerException {
+ throws
+ SolrServerException {
LOG.info("Attempting to index product from metadata file.");
try {
SerializableMetadata metadata = new SerializableMetadata("UTF-8", false);
metadata.loadMetadataFromXmlStream(new FileInputStream(file));
metadata.addMetadata("id", metadata.getMetadata("CAS."
- + CoreMetKeys.PRODUCT_ID));
+ + CoreMetKeys.PRODUCT_ID));
metadata.addMetadata(config.getProperty(ACCESS_KEY), config
- .getProperty(ACCESS_URL)
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .getProperty(ACCESS_URL)
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
if (delete) {
server
- .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
}
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: "
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
} catch (InstantiationException e) {
LOG.severe("Could not instantiate metadata object: " + e.getMessage());
} catch (FileNotFoundException e) {
@@ -236,7 +236,7 @@ public class SolrIndexer {
/**
* This method indexes all product types retrieved from the File Manager to
* the Solr index.
- *
+ *
* @param delete
* Flag indicating whether each product type retrieved from the File
* Manager should be deleted from the index.
@@ -247,7 +247,7 @@ public class SolrIndexer {
LOG.info("Indexing product types...");
try {
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
@@ -256,9 +256,9 @@ public class SolrIndexer {
metadata.addMetadata("id", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeId", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeDescription", type
- .getDescription());
+ .getDescription());
metadata.addMetadata("CAS.ProductTypeRepositoryPath", type
- .getProductRepositoryPath());
+ .getProductRepositoryPath());
metadata.addMetadata("CAS.ProductTypeVersioner", type.getVersioner());
metadata.addMetadata("CAS.ProductTypeName", type.getName());
metadata.addMetadata("ProductType", "ProductType");
@@ -268,7 +268,7 @@ public class SolrIndexer {
server.deleteById(type.getProductTypeId());
} catch (Exception e) {
LOG.severe("Could not delete product type " + type.getName()
- + " from index: " + e.getMessage());
+ + " from index: " + e.getMessage());
}
}
try {
@@ -277,7 +277,7 @@ public class SolrIndexer {
LOG.info("Indexed product type: " + type.getName());
} catch (Exception e) {
LOG.severe("Could not index " + type.getName() + ": "
- + e.getMessage());
+ + e.getMessage());
}
} else {
LOG.info("Ignoring product type: " + type.getName());
@@ -289,13 +289,13 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
LOG.info("Finished indexing product types.");
}
-
+
/**
- * Suppresses exception that occurred with older file managers.
+ * Suppresses exception that occurred with older file managers.
*/
private ProductPage safeFirstPage(XmlRpcFileManagerClient fmClient, ProductType type) {
ProductPage page = null;
@@ -311,7 +311,7 @@ public class SolrIndexer {
* This method indexes all products retrieved from the File Manager to the
* Solr index. Metadata from the product's associated ProductType is also
* included.
- *
+ *
* @param delete
* Flag indicating whether each product retrieved from the File
* Manager should be deleted from the index.
@@ -322,29 +322,28 @@ public class SolrIndexer {
LOG.info("Indexing products...");
try {
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
if (!config.getIgnoreTypes().contains(type.getName().trim())) {
- ProductPage page = safeFirstPage(fmClient, type);
LOG.info("Paging through products for product type: "
- + type.getName()+" Total pages are "+page.getTotalPages());
+ + type.getName());
+ ProductPage page = safeFirstPage(fmClient, type);
while (page != null) {
- for (Product product : page.getPageProducts()) {
+ for (Product product : page.getPageProducts()) {
try {
- String p = product.getProductId();
- Metadata m = fmClient.getMetadata(product);
- this.indexProduct(p, m, type.getTypeMetadata());
+ this.indexProduct(product.getProductId(), fmClient
+ .getMetadata(product), type.getTypeMetadata());
} catch (Exception e) {
LOG.severe("Could not index " + product.getProductId() + ": "
- + e.getLocalizedMessage());
+ + e.getMessage());
}
}
- if (page.getPageNum() >= page.getTotalPages() || page.isLastPage()) {
- break;
- }
- page = fmClient.getNextPage(type, page);
+ if (page.isLastPage()) {
+ break;
+ }
+ page = fmClient.getNextPage(type, page);
}
}
}
@@ -355,10 +354,10 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve products from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
}
@@ -366,50 +365,39 @@ public class SolrIndexer {
* This method adds a single product retrieved from the File Manager by its
* product identifier to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productId
* The identifier of the product (CAS.ProductId).
* @throws SolrServerException
* When an error occurs communicating with the Solr server instance.
*/
public void indexProduct(String productId)
- throws SolrServerException {
+ throws SolrServerException {
LOG.info("Attempting to index product: " + productId);
try {
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
Product product = fmClient.getProductById(productId);
Metadata productMetadata = fmClient.getMetadata(product);
- List<String> keys = productMetadata.getAllKeys();
- String logger = "";
- for(String k :keys){
- logger+=" key: "+keys;
- List<String> values = productMetadata.getAllValues(k);
-
- for(String v : values){
- logger+=" value: "+v + ", ";
- }
- }
- LOG.info("Metadata: "+ logger);
indexProduct(product.getProductId(), productMetadata, product
- .getProductType().getTypeMetadata());
+ .getProductType().getTypeMetadata());
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
-
+
/**
* This method adds a single product retrieved from the File Manager by its
* product name to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productName
* The identifier of the product (CAS.ProductId).
* @param delete
@@ -419,7 +407,7 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexProductByName(String productName, boolean delete) throws SolrServerException {
-
+
LOG.info("Attempting to index product: " + productName);
try {
@@ -439,27 +427,27 @@ public class SolrIndexer {
}
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
Product product = fmClient.getProductByName(productName);
Metadata productMetadata = fmClient.getMetadata(product);
// NOTE: delete (by id) is now false
indexProduct(product.getProductId(), productMetadata, product.getProductType().getTypeMetadata());
-
+
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
private void indexProduct(String productId, Metadata productMetadata,
- Metadata typeMetadata) throws SolrServerException,
- java.text.ParseException {
+ Metadata typeMetadata) throws SolrServerException,
+ java.text.ParseException {
Metadata metadata = new Metadata();
metadata.addMetadata("id", productId);
// Add in product type metadata
@@ -485,7 +473,7 @@ public class SolrIndexer {
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: " + productId);
} catch (IOException e) {
- LOG.severe("Could not index product: " + productId+ "Exception:"+e.getLocalizedMessage());
+ LOG.severe("Could not index product: " + productId);
}
} else {
LOG.info("Could not find metadata for product: " + productId);
@@ -494,7 +482,7 @@ public class SolrIndexer {
/**
* This method deletes a single product identified by a productID from the Solr index
- *
+ *
* @param productId
* @throws IOException
* @throws SolrServerException
@@ -506,7 +494,7 @@ public class SolrIndexer {
/**
* This method deletes a product(s) from the Solr index identified by a given name
- *
+ *
* @param productName
* @throws IOException
* @throws SolrServerException
@@ -530,16 +518,16 @@ public class SolrIndexer {
private void deleteProductFromIndex(String productId) throws IOException, SolrServerException {
server.deleteById(productId);
}
-
+
/**
* Quick helper method to do substitution on the keys specified in the config
- *
+ *
* @param metadata
* to substitute on
* @throws java.text.ParseException
*/
private void performSubstitution(Metadata metadata)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Do metadata replacement
for (String key : config.getReplacementKeys()) {
List<String> values = metadata.getAllMetadata(key);
@@ -559,12 +547,10 @@ public class SolrIndexer {
List<String> values = metadata.getAllMetadata(keyString);
if (values != null) {
List<String> newValues = new ArrayList<String>();
-
- /* SimpleDateFormat format = new SimpleDateFormat(config
- .getFormatProperties().getProperty(keyString).trim());*/
+ SimpleDateFormat format = new SimpleDateFormat(config
+ .getFormatProperties().getProperty(keyString).trim());
for (String value : values) {
- String d = value.trim();
- newValues.add(d);
+ newValues.add(formatDate(format, value));
}
metadata.removeMetadata(keyString);
metadata.addMetadata(keyString, newValues);
@@ -574,17 +560,17 @@ public class SolrIndexer {
}
private String formatDate(SimpleDateFormat format, String value)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Ignore formating if its an ignore value
if (config.getIgnoreValues().contains(value.trim())) {
- return value;
+ return value;
}
return solrFormat.format(format.parse(value));
}
/**
* This method builds the command-line options.
- *
+ *
* @return Returns the supported Options.
*/
@SuppressWarnings("static-access")
@@ -593,31 +579,31 @@ public class SolrIndexer {
options.addOption(new Option("h", "help", false, "Print this message"));
options.addOption(new Option("o", "optimize", false,
- "Optimize the Solr index"));
+ "Optimize the Solr index"));
options.addOption(new Option("d", "delete", false,
- "Delete item before indexing"));
+ "Delete item before indexing"));
options.addOption(OptionBuilder.withArgName("Solr URL").hasArg()
- .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
- .create("su"));
+ .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
+ .create("su"));
options.addOption(OptionBuilder.withArgName("Filemgr URL").hasArg()
- .withDescription("URL to the File Manager").withLongOpt("fmUrl")
- .create("fmu"));
+ .withDescription("URL to the File Manager").withLongOpt("fmUrl")
+ .create("fmu"));
OptionGroup group = new OptionGroup();
Option all = new Option("a", "all", false,
- "Index all products from the File Manager");
+ "Index all products from the File Manager");
Option product = OptionBuilder.withArgName("productId").hasArg()
- .withDescription("Index the product from the File Manager")
- .withLongOpt("product").create("p");
+ .withDescription("Index the product from the File Manager")
+ .withLongOpt("product").create("p");
Option met = OptionBuilder.withArgName("file").hasArg().withDescription(
- "Index the product from a metadata file").withLongOpt("metFile")
- .create("mf");
+ "Index the product from a metadata file").withLongOpt("metFile")
+ .create("mf");
Option read = new Option("r", "read", false,
- "Index all products based on a list of product identifiers passed in");
+ "Index all products based on a list of product identifiers passed in");
Option types = new Option("t", "types", false,
- "Index all product types from the File Manager");
+ "Index all product types from the File Manager");
Option deleteAll = new Option("da", "deleteAll", false,
- "Delete all products/types from the Solr index");
+ "Delete all products/types from the Solr index");
group.addOption(all);
group.addOption(product);
@@ -632,7 +618,7 @@ public class SolrIndexer {
/**
* The main method. Execution without argument displays help message.
- *
+ *
* @param args
* Command-line arguments.
*/
@@ -651,8 +637,8 @@ public class SolrIndexer {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("java " + SolrIndexer.class.getName(), options);
} else if (line.hasOption("all") || line.hasOption("product")
- || line.hasOption("metFile") || line.hasOption("read")
- || line.hasOption("types") || line.hasOption("deleteAll")) {
+ || line.hasOption("metFile") || line.hasOption("read")
+ || line.hasOption("types") || line.hasOption("deleteAll")) {
SolrIndexer indexer;
String solrUrl = null;
String fmUrl = null;
@@ -670,7 +656,7 @@ public class SolrIndexer {
indexer.indexProduct(line.getOptionValue("product"));
} else if (line.hasOption("metFile")) {
indexer.indexMetFile(new File(line.getOptionValue("metFile")), line
- .hasOption("delete"));
+ .hasOption("delete"));
} else if (line.hasOption("read")) {
for (String productId : readProductIdsFromStdin()) {
indexer.indexProduct(productId);
@@ -689,14 +675,14 @@ public class SolrIndexer {
} catch (Exception e) {
LOG.severe("An error occurred indexing: " + e.getMessage());
LOG
- .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
+ .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
}
}
}
/**
* This method reads product identifiers from the standard input.
- *
+ *
* @return Returns a List of product identifiers.
*/
private static List<String> readProductIdsFromStdin() {
@@ -712,12 +698,12 @@ public class SolrIndexer {
}
} catch (IOException e) {
LOG.severe("Error reading product id: line: [" + line + "]: Message: "
- + e.getMessage());
+ + e.getMessage());
} finally {
- try {
- br.close();
- } catch (Exception ignore) {
- }
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
}
return productIds;
}
@@ -749,32 +735,32 @@ public class SolrIndexer {
String key = (String) objKey;
if (key.startsWith(PREFIX_CONFIG)) {
properties.put(key.substring(PREFIX_CONFIG.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_MET)) {
mapProperties.put(key.substring(PREFIX_MET.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_FORMAT)) {
formatProperties.put(key.substring(PREFIX_FORMAT.length()), props
- .getProperty(key));
+ .getProperty(key));
}
}
if (properties.getProperty(IGNORE_TYPES) != null) {
String[] values = properties.getProperty(IGNORE_TYPES).trim()
- .split(",");
- Collections.addAll(ignoreTypes, values);
+ .split(",");
+ Collections.addAll(ignoreTypes, values);
}
if (properties.getProperty(IGNORE_VALUES) != null) {
String[] values = properties.getProperty(IGNORE_VALUES).trim().split(
- ",");
- Collections.addAll(ignoreValues, values);
+ ",");
+ Collections.addAll(ignoreValues, values);
}
if (properties.getProperty(REPLACEMENT_KEYS) != null) {
String[] values = properties.getProperty(REPLACEMENT_KEYS).trim()
- .split(",");
- Collections.addAll(replacementKeys, values);
+ .split(",");
+ Collections.addAll(replacementKeys, values);
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
index c3bf291..93f3afe 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
@@ -23,8 +23,21 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
@@ -308,6 +321,42 @@ public class LuceneWorkflowInstanceRepository extends
return wInsts;
}
+
+ @Override
+ public synchronized boolean clearWorkflowInstances() throws InstanceRepositoryException {
+ IndexWriter writer = null;
+ try {
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
+
+ writer = new IndexWriter(indexDir, config);
+ LOG.log(Level.FINE,
+ "LuceneWorkflowEngine: remove all workflow instances");
+ writer.deleteDocuments(new Term("myfield", "myvalue"));
+ } catch (IOException e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ LOG
+ .log(Level.WARNING,
+ "Exception removing workflow instances from index: Message: "
+ + e.getMessage());
+ throw new InstanceRepositoryException(e.getMessage());
+ } finally {
+ if (writer != null){
+ try{
+ writer.close();
+ }
+ catch(Exception ignore){}
+
+ writer = null;
+ }
+
+ }
+
+ return true;
+ }
/*
* (non-Javadoc)