You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drat.apache.org by GitBox <gi...@apache.org> on 2018/05/21 05:09:31 UTC
[GitHub] chrismattmann closed pull request #129: Replaced Processwrapper
indexing method with SolrIndexer
chrismattmann closed pull request #129: Replaced Processwrapper indexing method with SolrIndexer
URL: https://github.com/apache/drat/pull/129
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/proteus/src/main/java/backend/FileConstants.java b/proteus/src/main/java/backend/FileConstants.java
index f5a69d7..651060e 100644
--- a/proteus/src/main/java/backend/FileConstants.java
+++ b/proteus/src/main/java/backend/FileConstants.java
@@ -20,6 +20,8 @@
import org.apache.oodt.cas.metadata.util.PathUtils;
import java.io.File;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* Created by stevenfrancus on 10/13/15.
@@ -32,7 +34,15 @@
public static final String DRAT_PATH = buildDratSubdirectoryPath("/deploy/bin/drat");
public static final String DRAT_TEMP_UNZIPPED_PATH = buildDratSubdirectoryPath("/deploy/data/staging");
public static final String DRAT_TEMP_LOG_OUTPUT = buildDratSubdirectoryPath("/deploy/data/drat_output.log");
+ public static final String SOLR_INDEXER_CONFIG_PATH = buildDratSubdirectoryPath("/deploy/filemgr/etc/indexer.properties");
+ public static final String FILEMGR_URL=PathUtils.replaceEnvVariables("[FILEMGR_URL]");
+ public static final String SOLR_DRAT_URL=PathUtils.replaceEnvVariables("[SOLR_DRAT_URL]");
+ public static final String CLIENT_URL=PathUtils.replaceEnvVariables("[WORKFLOW_URL]");
+ public static final String OPSUI_URL=PathUtils.replaceEnvVariables("[OPSUI_URL]");
+
+ public static final String SOLR_INDEXER_CONFIG = "SOLR_INDEXER_CONFIG";
+
private static String getDratDirectory() {
final String DRAT_HOME = PathUtils.replaceEnvVariables("[DRAT_HOME]");
return DRAT_HOME.substring(0, DRAT_HOME.lastIndexOf(DRAT) + DRAT.length());
@@ -41,4 +51,6 @@ private static String getDratDirectory() {
public static String buildDratSubdirectoryPath(String additionalPath) {
return DRAT_SUPER_DIR + additionalPath;
}
+
+
}
diff --git a/proteus/src/main/java/backend/ProcessDratWrapper.java b/proteus/src/main/java/backend/ProcessDratWrapper.java
index e2c4188..10f1e20 100644
--- a/proteus/src/main/java/backend/ProcessDratWrapper.java
+++ b/proteus/src/main/java/backend/ProcessDratWrapper.java
@@ -22,11 +22,17 @@
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.python.modules.synchronize;
+import org.apache.oodt.cas.crawl.CrawlerLauncher;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductPage;
import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.tools.DeleteProduct;
+import org.apache.oodt.cas.filemgr.tools.SolrIndexer;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient;
import org.apache.oodt.pcs.util.FileManagerUtils;
@@ -36,9 +42,14 @@
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -104,8 +115,20 @@ public void crawl() throws Exception {
}
@Override
- public void index() throws IOException, DratWrapperException {
- simpleDratExec(INDEX_CMD, this.path);
+ public void index() throws IOException, DratWrapperException, InstantiationException, SolrServerException {
+ solrIndex();
+ }
+
+ private synchronized void solrIndex() throws InstantiationException, SolrServerException, IOException {
+ DratLog idl = new DratLog("INDEXING");
+ idl.logInfo("Starting", null);
+ System.setProperty(FileConstants.SOLR_INDEXER_CONFIG,FileConstants.SOLR_INDEXER_CONFIG_PATH);
+ SolrIndexer sIndexer = new SolrIndexer(FileConstants.SOLR_DRAT_URL,FileConstants.FILEMGR_URL);
+ sIndexer.indexAll(false);
+ sIndexer.commit();
+ sIndexer.optimize();
+ idl.logInfo("Completed",null);
+
}
@Override
@@ -173,6 +196,7 @@ public void reset() throws Exception {
public void go() throws Exception {
// before go, always reset
+
this.reset();
this.crawl();
this.index();
@@ -422,5 +446,62 @@ private synchronized void wipeSolrCore(String coreName) {
+ e.getLocalizedMessage());
}
}
+
+ private class DratLog{
+ private static final String MODULE = "DRAT_LOG";
+ long startTime =0;
+ private long lastActionTime=-1L;
+ private long timeDiff =-1L;
+ private ZonedDateTime zdt;
+ private String action;
+ public DratLog(String action) {
+ this.action = action;
+
+ }
+
+ private void logWarning(String status,String desc) {
+ LOG.warning(getMsg(status,desc));
+ }
+
+ private void logWarning(String desc) {
+ LOG.warning(MODULE+" : "+desc);
+ }
+
+ private void logInfo(String status,String desc) {
+ LOG.info(getMsg(status,desc));
+ }
+
+ private void logInfo(String desc) {
+ LOG.info(MODULE+" : "+desc);
+ }
+
+ private void logSevere(String status,String desc) {
+ LOG.fine(getMsg(status,desc));
+ }
+
+ private String getMsg(String status,String desc) {
+ String basic = "";
+ if(startTime==0) {
+ startTime = System.currentTimeMillis();
+ zdt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTime), ZoneId.systemDefault());
+ basic = String.format("%1$s : %2$s : %3$s, at time %4$s", MODULE,action,status,
+ zdt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME));
+ }else {
+ lastActionTime = System.currentTimeMillis();
+ timeDiff = lastActionTime - startTime;
+ zdt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTime), ZoneId.systemDefault());
+ basic = String.format("%1$s : %2$s : %3$s, at time %4$s with duration %5$s", MODULE,action,status,
+ zdt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME),DurationFormatUtils.formatDuration(timeDiff,"MM-dd T HH-mm-ss"));
+ }
+
+ if(desc==null) {
+ return basic;
+ }else {
+ return String.format("%1$s : %2$s", basic,desc);
+ }
+ }
+
+
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services