You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/30 13:54:06 UTC
svn commit: r1642540 - in /manifoldcf/trunk: ./
connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
Author: kwright
Date: Sun Nov 30 12:54:06 2014
New Revision: 1642540
URL: http://svn.apache.org/r1642540
Log:
Fix for CONNECTORS-969.
Removed:
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputSpecs.java
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1642540&r1=1642539&r2=1642540&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Nov 30 12:54:06 2014
@@ -3,6 +3,9 @@ $Id$
======================= 2.0-dev =====================
+CONNECTORS-969: Don't use JSON as an output specification format.
+(Karl Wright)
+
CONNECTORS-974: Make SharePoint 2010 be the default selection.
(Karl Wright)
Modified: manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1642540&r1=1642539&r2=1642540&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java (original)
+++ manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java Sun Nov 30 12:54:06 2014
@@ -24,6 +24,10 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.StringReader;
+
+import org.apache.commons.io.IOUtils;
import java.net.URI;
import java.net.URISyntaxException;
@@ -35,6 +39,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.HashMap;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.agents.output.BaseOutputConnector;
@@ -142,7 +148,7 @@ public class FileOutputConnector extends
@Override
public VersionContext getPipelineDescription(Specification spec) throws ManifoldCFException, ServiceInterruption {
FileOutputSpecs specs = new FileOutputSpecs(getSpecNode(spec));
- return new VersionContext(specs.toJson().toString(),params,spec);
+ return new VersionContext(specs.toVersionString(),params,spec);
}
/** Add (or replace) a document in the output data store using the connector.
@@ -686,4 +692,98 @@ public class FileOutputConnector extends
}
return sb.toString();
}
+
+ protected static class FileOutputSpecs extends FileOutputParam {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1859652730572662025L;
+
+ final public static ParameterEnum[] SPECIFICATIONLIST = {
+ ParameterEnum.ROOTPATH
+ };
+
+ private final String rootPath;
+
+ /** Build a set of ElasticSearch parameters by reading an instance of
+ * SpecificationNode.
+ *
+ * @param node
+ * @throws ManifoldCFException
+ */
+ public FileOutputSpecs(ConfigurationNode node) throws ManifoldCFException {
+ super(SPECIFICATIONLIST);
+ String rootPath = null;
+ for (ParameterEnum param : SPECIFICATIONLIST) {
+ String value = null;
+ if (node != null) {
+ value = node.getAttributeValue(param.name());
+ }
+ if (value == null) {
+ value = param.defaultValue;
+ }
+ put(param, value);
+ }
+ rootPath = getRootPath();
+ this.rootPath = rootPath;
+ }
+
+ /**
+ * @param variableContext
+ * @param specNode
+ */
+ public static void contextToSpecNode(IPostParameters variableContext, ConfigurationNode specNode, int sequenceNumber) {
+ for (ParameterEnum param : SPECIFICATIONLIST) {
+ String p = variableContext.getParameter("s"+sequenceNumber+"_"+param.name().toLowerCase());
+ if (p != null) {
+ specNode.setAttribute(param.name(), p);
+ }
+ }
+ }
+
+ /** @return a version string representation of the parameter list */
+ public String toVersionString() {
+ StringBuilder sb = new StringBuilder();
+ pack(sb,rootPath,'+');
+ return sb.toString();
+ }
+
+ /**
+ * @return
+ */
+ public String getRootPath() {
+ return get(ParameterEnum.ROOTPATH);
+ }
+
+ /**
+ * @param content
+ * @return
+ * @throws ManifoldCFException
+ */
+ private final static TreeSet<String> createStringSet(String content) throws ManifoldCFException {
+ TreeSet<String> set = new TreeSet<String>();
+ BufferedReader br = null;
+ StringReader sr = null;
+ try {
+ sr = new StringReader(content);
+ br = new BufferedReader(sr);
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ line = line.trim();
+ if (line.length() > 0) {
+ set.add(line);
+ }
+ }
+ return set;
+ } catch (IOException e) {
+ throw new ManifoldCFException(e);
+ } finally {
+ if (br != null) {
+ IOUtils.closeQuietly(br);
+ }
+ }
+ }
+
+ }
+
}
Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1642540&r1=1642539&r2=1642540&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (original)
+++ manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Sun Nov 30 12:54:06 2014
@@ -25,12 +25,18 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.StringReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.HashMap;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -252,7 +258,7 @@ public class HDFSOutputConnector extends
@Override
public VersionContext getPipelineDescription(Specification spec) throws ManifoldCFException, ServiceInterruption {
HDFSOutputSpecs specs = new HDFSOutputSpecs(getSpecNode(spec));
- return new VersionContext(specs.toJson().toString(),params,spec);
+ return new VersionContext(specs.toVersionString(),params,spec);
}
/** Add (or replace) a document in the output data store using the connector.
@@ -323,9 +329,6 @@ public class HDFSOutputConnector extends
Long startTime = new Long(System.currentTimeMillis());
deleteFile(path,activities,documentURI);
activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, "OK", null);
- } catch (JSONException e) {
- activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed to delete document due to: " + e.getMessage());
- handleJSONException(e);
} catch (URISyntaxException e) {
activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed to delete document due to: " + e.getMessage());
handleURISyntaxException(e);
@@ -855,4 +858,112 @@ public class HDFSOutputConnector extends
}
}
+ public static class HDFSOutputSpecs extends HDFSOutputParam {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1145652730572662025L;
+
+ final public static ParameterEnum[] SPECIFICATIONLIST = {
+ ParameterEnum.rootpath
+ };
+
+ private final String rootPath;
+
+ /** Build a set of ElasticSearch parameters by reading an JSON object
+ *
+ * @param json
+ * @throws JSONException
+ * @throws ManifoldCFException
+ */
+ public HDFSOutputSpecs(String versionString) throws ManifoldCFException {
+ super(SPECIFICATIONLIST);
+ int index = 0;
+ StringBuilder rootPathBuffer = new StringBuilder();
+ index = unpack(rootPathBuffer,versionString,index,'+');
+ this.rootPath = rootPathBuffer.toString();
+ // MHL
+ }
+
+ /** Build a set of ElasticSearch parameters by reading an instance of
+ * SpecificationNode.
+ *
+ * @param node
+ * @throws ManifoldCFException
+ */
+ public HDFSOutputSpecs(ConfigurationNode node) throws ManifoldCFException {
+ super(SPECIFICATIONLIST);
+ String rootPath = null;
+ for (ParameterEnum param : SPECIFICATIONLIST) {
+ String value = null;
+ if (node != null) {
+ value = node.getAttributeValue(param.name());
+ }
+ if (value == null) {
+ value = param.defaultValue;
+ }
+ put(param, value);
+ }
+ rootPath = getRootPath();
+ this.rootPath = rootPath;
+ }
+
+ /**
+ * @param variableContext
+ * @param specNode
+ */
+ public static void contextToSpecNode(IPostParameters variableContext, ConfigurationNode specNode, int sequenceNumber) {
+ for (ParameterEnum param : SPECIFICATIONLIST) {
+ String p = variableContext.getParameter("s"+sequenceNumber+"_"+param.name().toLowerCase());
+ if (p != null) {
+ specNode.setAttribute(param.name(), p);
+ }
+ }
+ }
+
+ /** @return a JSON representation of the parameter list */
+ public String toVersionString() {
+ StringBuilder sb = new StringBuilder();
+ pack(sb,rootPath,'+');
+ return sb.toString();
+ }
+
+ /**
+ * @return
+ */
+ public String getRootPath() {
+ return get(ParameterEnum.rootpath);
+ }
+
+ /**
+ * @param content
+ * @return
+ * @throws ManifoldCFException
+ */
+ private final static TreeSet<String> createStringSet(String content) throws ManifoldCFException {
+ TreeSet<String> set = new TreeSet<String>();
+ BufferedReader br = null;
+ StringReader sr = null;
+ try {
+ sr = new StringReader(content);
+ br = new BufferedReader(sr);
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ line = line.trim();
+ if (line.length() > 0) {
+ set.add(line);
+ }
+ }
+ return set;
+ } catch (IOException e) {
+ throw new ManifoldCFException(e.getMessage(),e);
+ } finally {
+ if (br != null) {
+ IOUtils.closeQuietly(br);
+ }
+ }
+ }
+
+ }
+
}